diff --git a/.gdbinit b/.gdbinit index 457dc7b31..a9f898fb6 100644 --- a/.gdbinit +++ b/.gdbinit @@ -179,6 +179,25 @@ define ds dumpsxp $arg0 1 end +define refetch + shell /usr/bin/git fetch mine && /usr/bin/git reset --hard mine/$(/usr/bin/git rev-parse --abbrev-ref HEAD) && /usr/bin/ninja + python gdb.execute("file " + gdb.current_progspace().filename) + directory + run +end + +define rerun + python gdb.execute("file " + gdb.current_progspace().filename) + directory + run +end + +define ninja + shell ninja + python gdb.execute("file " + gdb.current_progspace().filename) + directory +end + # source .pirpp.py diff --git a/.gitignore b/.gitignore index 01d6a2c5b..0f3a213a9 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,7 @@ benchmarks/ *.DS_Store external/* !external/custom-r -.history \ No newline at end of file +!external/xxHash +.history +.cache +compile_commands.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9fdd2c9fe..fdb13d26f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -189,7 +189,7 @@ tests_fullverify: - make -j6 - bin/tests -# Test particular features, like deoptimization and serialization +# Test particular features, like deoptimization test_features_1: image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA variables: @@ -265,6 +265,73 @@ test_features_3: when: on_failure expire_in: 1 week +# Test serialization (no LLVM bitcode) +test_serialize_chaos: + image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA + variables: + GIT_STRATEGY: none + PIR_LLVM_OPT_LEVEL: 0 + stage: Run tests + needs: + - rir_container + except: + - schedules + script: + - /opt/rir/container/install-test-deps.sh + - cd /opt/rir/build/release + - RIR_SERIALIZE_CHAOS=5 FAST_TESTS=1 bin/tests + - PIR_WARMUP=2 RIR_SERIALIZE_CHAOS=50 bin/gnur-make-tests check || $SAVE_LOGS + - ../../tools/check-gnur-make-tests-error + - RIR_SERIALIZE_CHAOS=10 bin/tests + artifacts: + paths: + - logs + when: on_failure + expire_in: 1 week + +# Test LLVM bitcode serialization +test_serialize_llvm: + image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA + variables: + GIT_STRATEGY: none + PIR_LLVM_OPT_LEVEL: 0 + stage: Run tests + needs: + - rir_container + except: + - schedules + script: + - /opt/rir/container/install-test-deps.sh + - cd /opt/rir/build/release + - PIR_DEBUG_SERIALIZE_LLVM=1 bin/gnur-make-tests check || $SAVE_LOGS + - ../../tools/check-gnur-make-tests-error + artifacts: + paths: + - logs + when: on_failure + expire_in: 1 week + +# Test regular and LLVM bitcode serialization +test_serialize_both: + image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA + variables: + GIT_STRATEGY: none + PIR_LLVM_OPT_LEVEL: 0 + stage: Run tests + needs: + - rir_container + except: + - schedules + script: + - /opt/rir/container/install-test-deps.sh + - cd /opt/rir/build/release + - PIR_DEBUG_SERIALIZE_LLVM=1 FAST_TESTS=1 RIR_SERIALIZE_CHAOS=5 bin/tests + - PIR_DEBUG_SERIALIZE_LLVM=1 RIR_SERIALIZE_CHAOS=10 bin/tests + artifacts: + paths: + - logs + when: on_failure + expire_in: 1 week # Run ubsan and gc torture test_gctorture_1: @@ -334,6 +401,7 @@ test_big_inline: test_sanitize: image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA variables: + CLANG_DIR: /opt/clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04 GIT_STRATEGY: none stage: Run tests needs: @@ -341,18 +409,65 @@ test_sanitize: except: - schedules script: + # TODO: Store clang-16 and zlib on prl + # Install clang-16 + - apt install libtinfo5 + - wget https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz -O $CLANG_DIR.tar.xz + - tar -xf $CLANG_DIR.tar.xz -C /opt + # Manually add zlib (required when building on clang-16) + - wget https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.xz -O /opt/rir/external/zlib-1.3.tar.xz + - tar -xf /opt/rir/external/zlib-1.3.tar.xz -C /opt/rir/external + - cd /opt/rir/external/zlib-1.3 + - ./configure --prefix=/opt/rir/external/zlib + - make -j6 + - make install + # Rest - curl 10.200.14.25:8080/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz > /opt/rir/external/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz - mkdir /opt/rir/build/sanitize - cd /opt/rir/build/sanitize - /opt/rir/tools/fetch-llvm.sh - - CC=$(ls ../../external/clang*/bin/clang) CXX=$(ls ../../external/clang*/bin/clang) cmake -DCMAKE_BUILD_TYPE=sanitize ../.. + - cmake -DCMAKE_BUILD_TYPE=sanitize + -DCMAKE_C_COMPILER=$CLANG_DIR/bin/clang + -DCMAKE_CXX_COMPILER=$CLANG_DIR/bin/clang++ + -DZLIB_LIBRARY=/opt/rir/external/zlib/lib/libz.so + -DZLIB_INCLUDE_DIR=/opt/rir/external/zlib/include + ../.. - make -j6 # R_LD_PRELOAD is a feature of the test-runner. To repro this without the testrunner use LD_PRELOAD instead. # intercept_tls_get_addr=0 helps with leak sanitizer crashes - - ASAN_OPTIONS="intercept_tls_get_addr=0" LSAN_OPTIONS="symbolize=1" ASAN_SYMBOLIZER_PATH=$(ls /opt/rir/external/clang*/bin/llvm-symbolizer) R_LD_PRELOAD=$(ls /opt/rir/external/clang*/lib/clang/12.0.0/lib/linux/libclang_rt.asan-x86_64.so) bin/tests + - ASAN_OPTIONS="intercept_tls_get_addr=0" + LSAN_OPTIONS="symbolize=1" + ASAN_SYMBOLIZER_PATH=$CLANG_DIR/bin/llvm-symbolizer + R_LD_PRELOAD=$CLANG_DIR/lib/clang/16/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so + bin/tests # sometimes leak sanitizer segfaults retry: 2 +# Test the compiler server and client (on localhost) +test_compiler_server_client: + image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA + variables: + GIT_STRATEGY: none + PIR_LLVM_OPT_LEVEL: 0 + stage: Run tests + needs: + - rir_container + except: + - schedules + script: + - mkdir $CI_PROJECT_DIR/results && mkdir $CI_PROJECT_DIR/results/debug && mkdir $CI_PROJECT_DIR/results/release + - curl 10.200.14.25:8080/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz > /opt/rir/external/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz + - /opt/rir/tools/fetch-llvm.sh + - mkdir /opt/rir/build/debug && cd /opt/rir/build/debug && cmake -DCMAKE_BUILD_TYPE=Debug ../.. && make -j6 + - PIR_CLIENT_SKIP_DISCREPANCY_CHECK=1 ./bin/test-compiler-client-and-server; cp /tmp/test-compiler-server-actual.out /tmp/test-compiler-client-actual.out $CI_PROJECT_DIR/results/debug/ + - cd /opt/rir/build/release + - PIR_CLIENT_SKIP_DISCREPANCY_CHECK=1 ./bin/test-compiler-client-and-server; cp /tmp/test-compiler-server-actual.out /tmp/test-compiler-client-actual.out $CI_PROJECT_DIR/results/release/ + artifacts: + paths: + - results + when: always + expire_in: 1 week + # Test the benchmarks container before deploying test_benchmarks: image: registry.gitlab.com/rirvm/rir_mirror/benchmark:$CI_COMMIT_SHA diff --git a/.gitmodules b/.gitmodules index f21e7abfb..b18d454ab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = external/custom-r url = https://github.com/reactorlabs/gnur.git ignore = untracked +[submodule "external/xxHash"] + path = external/xxHash + url = https://github.com/Cyan4973/xxHash.git diff --git a/CMakeLists.txt b/CMakeLists.txt index f5436cefe..ead4d5ffb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,8 @@ set(R_LIBRARY_TREE ${CMAKE_SOURCE_DIR}/packages) set(R_ROOT_DIR ${R_HOME}) set(R_INCLUDE_DIR ${R_HOME}/include) set(LLVM_DIR ${CMAKE_SOURCE_DIR}/external/llvm-12) +set(ZEROMQ_DIR ${CMAKE_SOURCE_DIR}/external/zeromq) +set(XXHASH_DIR ${CMAKE_SOURCE_DIR}/external/xxHash) set(R_COMMAND ${R_HOME}/bin/R) @@ -22,29 +24,28 @@ endif () include_directories(SYSTEM ${LLVM_INCLUDE_DIRS}) add_definitions(${LLVM_DEFINITIONS}) -# use GCC 9 on macOS. Otherwise we use clang -if(APPLE) - option(MACOS_USE_GCC_9 "Use GCC 9 on macOS." FALSE) -endif(APPLE) +include_directories(${ZEROMQ_DIR}/include) +# Won't work on Linux if we try to link statically: +# /usr/bin/ld: ../external/zeromq/lib/libzmq.a(libzmq_la-zmq.o): relocation R_X86_64_PC32 against symbol +# `_ZSt7nothrow@@GLIBCXX_3.4' can not be used when making a shared object; recompile with -fPIC +if (${APPLE}) + link_libraries(${ZEROMQ_DIR}/lib/libzmq.dylib) +else () + link_libraries(${ZEROMQ_DIR}/lib/libzmq.so) +endif () -if(${MACOS_USE_GCC_9}) - set(CMAKE_C_COMPILER /usr/local/bin/gcc-9 CACHE PATH "" FORCE) - set(CMAKE_CXX_COMPILER /usr/local/bin/g++-9 CACHE PATH "" FORCE) -endif() +set(XXHASH_BUILD_XXHSUM OFF) +set(BUILD_SHARED_LIBS ON) +add_subdirectory(${XXHASH_DIR}/cmake_unofficial/ ${XXHASH_DIR}/build/ EXCLUDE_FROM_ALL) add_definitions(-g) set(CMAKE_CXX_FLAGS_RELEASE "-O2 -Werror -DSWITCH_TO_NAMED=1") set(CMAKE_CXX_FLAGS_RELEASENOASSERT "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG") set(CMAKE_CXX_FLAGS_FULLVERIFIER "${CMAKE_CXX_FLAGS_RELEASE} -DFULLVERIFIER") set(CMAKE_CXX_FLAGS_RELEASESLOWASSERT "${CMAKE_CXX_FLAGS_RELEASE} -DENABLE_SLOWASSERT") -set(CMAKE_CXX_FLAGS_DEBUG "-O0 -DSWITCH_TO_NAMED=1 -DENABLE_SLOWASSERT") +set(CMAKE_CXX_FLAGS_DEBUG "-O0 -DSWITCH_TO_NAMED=1 -DENABLE_SLOWASSERT -DDEBUG_DISASSEMBLY") set(CMAKE_CXX_FLAGS_DEBUGOPT "-Og -DSWITCH_TO_NAMED=1 -DENABLE_SLOWASSERT") -# with macOS GCC 9 we need to explicitly use libc++, since llvm does. See https://libcxx.llvm.org/docs/UsingLibcxx.html#using-libc-with-gcc -if(${MACOS_USE_GCC_9}) - set(CMAKE_CXX_FLAGS_LIBCXX "-nostdinc++ -nodefaultlibs -lc++ -lc++abi -lm -lc -lgcc_s.1 -lgcc") -else() - set(CMAKE_CXX_FLAGS_LIBCXX "") -endif() +set(CMAKE_CXX_FLAGS_LIBCXX "") set(CMAKE_CXX_FLAGS "${LLVM_CXX_FLAGS} ${CMAKE_CXX_FLAGS_LIBCXX} -Wall -Wuninitialized -Wundef -Winit-self -Wcast-align -Woverloaded-virtual -Wmissing-include-dirs -Wstrict-overflow=3 -std=c++14 -fno-rtti -fno-exceptions -Wimplicit-fallthrough -Wno-deprecated-declarations") set(CMAKE_C_FLAGS_RELEASE "-O2 -DSWITCH_TO_NAMED=1") set(CMAKE_C_FLAGS_RELEASENOASSERT "${CMAKE_C_FLAGS_RELEASE} -DNDEBUG") @@ -54,8 +55,16 @@ set(CMAKE_C_FLAGS_DEBUG "-O0 -DSWITCH_TO_NAMED=1 -DENABLE_SLOWASSERT") set(CMAKE_C_FLAGS_DEBUGOPT "-Og -DSWITCH_TO_NAMED=1 -DENABLE_SLOWASSERT") set(CMAKE_C_FLAGS "-std=gnu99") -set(CMAKE_CXX_FLAGS_SANITIZE "${CMAKE_CXX_FLAGS_RELEASE} -g2 -fno-omit-frame-pointer -fsanitize=undefined -fsanitize=address -fsanitize=leak -fno-sanitize=alignment -shared-libasan -fvisibility=default") -set(CMAKE_C_FLAGS_SANITIZE "${CMAKE_C_FLAGS_RELEASE} -g2 -fno-omit-frame-pointer -fsanitize=undefined -fsanitize=address -fsanitize=leak -fno-sanitize=alignment -shared-libasan -fvisibility=default") +if (${APPLE}) + set(SANITIZE_FLAGS "-g2 -fno-omit-frame-pointer -fsanitize=undefined -fsanitize=address -fno-sanitize=alignment -shared-libasan -fvisibility=default") +else() + set(SANITIZE_FLAGS "-g2 -fno-omit-frame-pointer -fsanitize=undefined -fsanitize=address -fsanitize=leak -fno-sanitize=alignment -shared-libasan -fvisibility=default") +endif() +set(CMAKE_CXX_FLAGS_SANITIZE "${CMAKE_CXX_FLAGS_RELEASE} ${SANITIZE_FLAGS}") +set(CMAKE_C_FLAGS_SANITIZE "${CMAKE_C_FLAGS_RELEASE} ${SANITIZE_FLAGS}") +set(CMAKE_CXX_FLAGS_DEBUG_SANITIZE "${CMAKE_CXX_FLAGS_DEBUG} ${SANITIZE_FLAGS}") +set(CMAKE_C_FLAGS_DEBUG_SANITIZE "${CMAKE_C_FLAGS_DEBUG} ${SANITIZE_FLAGS}") + MARK_AS_ADVANCED( CMAKE_CXX_FLAGS_SANITIZE @@ -96,12 +105,15 @@ endif() # Create proxy scripts for the scripts in /tools file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/.bin_create") -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/tests" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/tests \"$@\"") -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/R" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/R \"$@\"") -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/Rscript" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/Rscript \"$@\"") -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/Rgnu" "#!/bin/sh\n${CMAKE_SOURCE_DIR}/external/custom-r/bin/R \"$@\"") -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/gnur-make" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/gnur-make \"$@\"") -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/gnur-make-tests" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/gnur-make-tests \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/tests" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/tests \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/test-compiler-client-and-server" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/test-compiler-client-and-server \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/test-compiler-client-only" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/test-compiler-client-only \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/test-compiler-server-only" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/test-compiler-server-only \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/R" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/R \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/Rscript" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/Rscript \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/Rgnu" "#!/bin/sh\n${CMAKE_SOURCE_DIR}/external/custom-r/bin/R \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/gnur-make" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/gnur-make \"$@\"") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/gnur-make-tests" "#!/bin/sh\nRIR_BUILD=\"${CMAKE_CURRENT_BINARY_DIR}\" ${CMAKE_SOURCE_DIR}/tools/gnur-make-tests \"$@\"") file(GLOB BIN_IN "${CMAKE_CURRENT_BINARY_DIR}/.bin_create/*") file(INSTALL ${BIN_IN} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/bin" FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ GROUP_EXECUTE) @@ -118,36 +130,20 @@ endif(${NO_LOCAL_CONFIG}) include_directories(${R_INCLUDE_DIR}) include_directories(${CMAKE_SOURCE_DIR}/rir/src) -# again we need to explicitly use libc++ -if(${MACOS_USE_GCC_9}) - include_directories(/Library/Developer/CommandLineTools/usr/include/c++/v1) -endif(${MACOS_USE_GCC_9}) message(STATUS "Using R from ${R_HOME}") add_custom_target(setup-build-dir COMMAND ${CMAKE_SOURCE_DIR}/tools/setup-build-dir ${CMAKE_SOURCE_DIR} ${R_HOME} ) -if(${MACOS_USE_GCC_9}) - add_custom_target(dependencies - COMMAND ${CMAKE_SOURCE_DIR}/tools/build-gnur.sh --macos_gcc9 - COMMAND ${CMAKE_SOURCE_DIR}/tools/fetch-llvm.sh --macos_gcc9 - ) -else() - add_custom_target(dependencies - COMMAND ${CMAKE_SOURCE_DIR}/tools/build-gnur.sh - COMMAND ${CMAKE_SOURCE_DIR}/tools/fetch-llvm.sh - ) -endif() - -add_custom_target(default-gnur - DEPENDS dependencies - COMMAND ${CMAKE_SOURCE_DIR}/tools/build-gnur.sh custom-r +add_custom_target(dependencies + COMMAND ${CMAKE_SOURCE_DIR}/tools/build-gnur.sh + COMMAND ${CMAKE_SOURCE_DIR}/tools/build-zeromq.sh + COMMAND ${CMAKE_SOURCE_DIR}/tools/fetch-llvm.sh ) add_custom_target(setup DEPENDS dependencies - DEPENDS default-gnur ) add_custom_target(tests @@ -175,10 +171,17 @@ if (DEFINED LLVM_PACKAGE_VERSION) target_link_libraries(${PROJECT_NAME} ${LLVM_LIBS}) endif(DEFINED LLVM_PACKAGE_VERSION) +target_link_libraries(${PROJECT_NAME} xxHash::xxhash) + if(APPLE) set_target_properties(${PROJECT_NAME} PROPERTIES LINK_FLAGS "-L${R_HOME}/lib") target_link_libraries(${PROJECT_NAME} R) # to resolve build error from # https://www.gnu.org/software/gettext/FAQ.html#integrating_undefined - target_link_libraries(${PROJECT_NAME} -lintl) + # target_link_libraries(${PROJECT_NAME} -lintl) + include_directories(${LLVM_DIR}/include) + target_link_directories(${PROJECT_NAME} INTERFACE ${LLVM_DIR}/lib) + # Note: May need to update the version here + include_directories(/opt/homebrew/Cellar/gettext/0.21.1/include) + target_link_libraries(${PROJECT_NAME} /opt/homebrew/Cellar/gettext/0.21.1/lib/libintl.dylib) endif(APPLE) diff --git a/Dockerfile b/Dockerfile index 6131c1c48..5fe3dcfe9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,13 +5,16 @@ ENV LANG en_US.UTF-8 RUN echo $CI_COMMIT_SHA > /opt/rir_version && \ apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -qq && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl git gcc gfortran g++ libreadline-dev libx11-dev libxt-dev zlib1g-dev libbz2-dev liblzma-dev libpcre3-dev libcurl4-openssl-dev libcairo2-dev make libreadline8 libncurses-dev xz-utils cmake tcl-dev tk-dev locales rsync && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl git gcc gfortran g++ libreadline-dev libx11-dev libxt-dev zlib1g-dev libbz2-dev liblzma-dev libpcre3-dev libcurl4-openssl-dev libcairo2-dev make libreadline8 libncurses-dev xz-utils cmake tcl-dev tk-dev locales rsync wget && \ locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 && \ cd /opt/rir && \ tools/build-gnur.sh && \ rm -rf external/custom-r/cache_recommended.tar .git && \ find external -type f -name '*.o' -exec rm -f {} \; && \ apt-get clean +RUN cd /opt/rir && \ + USE_NINJA=0 tools/build-zeromq.sh && \ + rm -rf external/zeromq-* external/cppzmq-* RUN mkdir -p /opt/rir/build/release && \ cd /opt/rir && \ (curl 10.200.14.25:8080/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz > external/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz || true) && \ diff --git a/documentation/compiler-server.md b/documentation/compiler-server.md new file mode 100644 index 000000000..017a97c96 --- /dev/null +++ b/documentation/compiler-server.md @@ -0,0 +1,72 @@ +# Compiler Server and Client + +## How to use + +### Locally + +Start the compiler server + + PIR_SERVER_ADDR=tcp://*:5555 ./bin/R + +**In a separate terminal window**, start the client + + PIR_CLIENT_ADDR=tcp://localhost:5555 ./bin/R + +You can change the port if you'd like. You can also start multiple clients for one server. ~~And you can have one client connect to multiple servers separated by commas, e.g.:~~ + + PIR_CLIENT_ADDR=tcp://localhost:1234,tcp://localhost:5678 ./bin/R + +(multiple servers are currently only in statis and won't work, because retrieval for multiple servers isn't implemented) + +We use [ZeroMQ](https://zeromq.org) for communication. See the ZeroMQ docs for all supported address types and how to connect to a remote server. + +### Full configuration options + + PIR_CLIENT_ADDR= +
(on client) address of compiler server to connect to + (on client) comma-separated addresses of compiler servers to connect to + PIR_CLIENT_TIMEOUT= + (on client) how long to wait for a reply from the server before timing out. Default is 10000 (10 seconds) + PIR_CLIENT_COMPILE_SIZE_TO_HASH_ONLY= + (on client) the server memoizes compile requests from all clients. If the client is going to send a request that is larger than this size, it will only hash the request and send the hash first. Then if the server has already compiled the request, it will reply with the compiled code, and if not, the server will send a response causing the client to send the full request + PIR_CLIENT_DRY_RUN= + <0|1> (on client) whether to actually use the server's code, or compile locally and just use it for comparison. Default is false (actually use the code) + PIR_CLIENT_SKIP_DISCREPANCY_CHECK= + <0|1> (on client) whether to skip checking for discrepancies between local and remote compilation. Default is to not skip. + PIR_SERVER_ADDR= +
(on server) address to listen on + +#### Logging + + PIR_TRACE_COMPILER_PEER= + 1 log the contents of every request sent to and received by the compiler client or server + PIR_LOG_COMPILER_PEER= + 1 log every message sent from/to the compiler peer. Superseded by PIR_TRACE_COMPILER_PEER + PIR_WARN_COMPILER_PEER= + 1 warn when the compiler peer connection times out or closes. Superseded by PIR_LOG_COMPILER_PEER + +These options are also in [./debugging.md](./debugging.md). They can be applied to client or server, and will log on whatever peer they're applied but not affect connected peers. + +It's recommended to set `PIR_WARN_COMPILER_PEER` to see any issues. Try setting `PIR_LOG_COMPILER_PEER` on the server to see the requests and responses being made. + +## What is a compiler server? + +A separate process which JIT-compiles code while the local process interprets your program. It can be on the same or different machine. This reduces the overhead of compiling. + +## How it works + +Both the compiler client and server are Ř processes. The server starts with `PIR_SERVER_ADDR=`, which will cause the server to wait for compile requests instead of running a REPL like normal. The client starts with `PIR_CLIENT_ADDR=`, which will cause it to connect to `` and send future compile requests there. + +Whenever the compiler client attempts to compile a function (by default, this happens after running the function a few times), it sends a request to the compiler server containing the function's code along with context and speculation info such as runtime types. The compiler server processes the request and replies with the compiled (LLVM) code. The client inserts this into the function's **dispatch table**, and future calls trigger the compiled code. If there is a deoptimization or the function is called with a different context, the compiler client may request the server to compile the same function again, with new context and/or speculation info (there's no point in re-compiling the function with the exact same info). + +The compiler server also memoizes requests by hashing the request data including R bytecode and feedback, so if it's asked to recompile the same closure again, it will return the already-compiled version. + +### SEXP intern pool + +TODO: improve writing + +Separate from requests, the compiler server interns SEXPs and will send SEXPs to the client with connected SEXPs as hashes. If the client doesn't have the SEXP locally, it can send a `Retrieve` request to the server to get it from the intern pool, but if it does, it can skip this request. This prevents transmitting redundant SEXPs and more importantly, creating separate SEXPs on the client; this is not just bad for performance, but can be a semantic issue. + +The SEXPs are interned according to a hash computed from their immutable semantic data, using [xxHash](https://xxhash.com/). Data which is mutable but doesn't affect semantics, like feedback, isn't part of the hash. Environments are also not part of the hash since they are mutable and defined behavior shouldn't rely on their changes throughout the program's execution. + +The client will also intern SEXPs it retrieves from the server. However, it explicitly *doesn't* send connected SEXPs as hashes, instead sending the full SEXP even if redundant, because the client's intern pool is temporary. When the server interns SEXPs it also preserves them for future clients, and the server will presumably have more memory so it can handle this. Because the server has an intern pool, even though it will receive and deserialize redundant SEXPs, it won't actually store the duplicates, they'll simply be discarded. \ No newline at end of file diff --git a/documentation/debugging.md b/documentation/debugging.md index c44463199..bf8306e31 100644 --- a/documentation/debugging.md +++ b/documentation/debugging.md @@ -45,6 +45,44 @@ graphical representation of the code choose the GraphViz debug style. GraphViz print pir in GraphViz, displaying all instructions within BBs GraphVizBB print pir in GraphViz, displaying only BB names and connections + RIR_DEBUG_STYLE= + Standard print basic information in rir objects in human-readable format + Detailed print very detailed information in rir objects, useful for debugging or explaining unexpected semantic differences + PrettyGraph print in HTML which can be loaded with `tools/rirPrettyGraph` in the same location to display an interactive graph + + PIR_GRAPH_PRINT_RIR_OBJECTS= + <0|1|path> if set, folder to print pretty graphs of RIR objects which get compiled or interned. If set to 1, prints HTML to stdout. If set to 0 or unset (default), won't print. + + PIR_GRAPH_PRINT_RIR_OBJECTS_FREQUENCY= + n print pretty graphs of every nth RIR object which gets compiled or interned. Defaults to 10. Otherwise we print a lot more RIR objects than are necessary. + + PIR_TRACE_SERIALIZATION= + 1 log every serialized or deserialized piece of data + + PIR_TRACE_SERIALIZATION_MAX_RAW_PRINT_LENGTH= + unsigned max length we will print serialized raw data in the trace. Ignored unless PIR_TRACE_SERIALIZATION is set + + PIR_TRACE_SERIALIZATION_EXCLUDE= + regex exclude logging serialized data from matching flags. Ignored unless PIR_TRACE_SERIALIZATION is set + + PIR_TRACE_SERIALIZATION_MIN_SIZE= + size_t minimum length of individual pieces of data which will be logged. Ignored unless PIR_TRACE_SERIALIZATION is set + + PIR_LOG_INTERNING= + 1 log every new intern, reused intern, unintern, and other intern related events. + + PIR_WARN_INTERNING= + 1 warn when an interned object's UUID changes and other inconsistencies. Superseded by PIR_LOG_INTERNING + + PIR_TRACE_COMPILER_PEER= + 1 log the contents of every request sent to and received by the compiler client or server + + PIR_LOG_COMPILER_PEER= + 1 log every message sent from/to the compiler peer. Superseded by PIR_TRACE_COMPILER_PEER + + PIR_WARN_COMPILER_PEER= + 1 warn when the compiler peer connection times out or closes. Superseded by PIR_LOG_COMPILER_PEER + The following flags can be useful for profiling and finding out which passes take how much time to complete. @@ -57,6 +95,18 @@ complete. PIR_MEASURE_COMPILER_BACKEND= 1 print overall time spend in different phases in the backend + PIR_MEASURE_COMPILED_CLOSURES= + 1 print # of compiled closures and time it spends to compile each one + + PIR_MEASURE_SERIALIZATION= + 1 print detailed report on time spent in serialization + + PIR_MEASURE_INTERNING= + 1 print detailed report on time spent in interning + + PIR_MEASURE_CLIENT_SERVER= + 1 print time spent in client server communication (sending and receiving requests + processing) + #### Controlling compilation PIR_ENABLE= @@ -81,6 +131,9 @@ complete. PIR_DEBUG_DEOPTS= 1 show failing assumption when a deopt happens + R_DISABLE_GC= + 1 disable the garbage collector + #### Optimization heuristics For more flags see compiler/parameter.h. @@ -104,6 +157,12 @@ For more flags see compiler/parameter.h. n serialize and deserialize the dispatch table on every `n`th RIR call. WARNING: This sometimes prevents optimization + PIR_DEBUG_SERIALIZE_LLVM= + 1 serialize LLVM IR, and add metadata to make it patchable on + different sessions. This will be set regardless of the env + var if RIR_PRESERVE is set or the compiler server is running, + so the only time this is useful is when debugging. + ### Disassembly annotations #### Assumptions @@ -168,6 +227,9 @@ debugging: * `rir.eval`: evaluates the code in RIR * `rir.body`: returns the body of rir-compiled function. The body is the vector containing its ast maps and code objects +* `rir.serialize`: Serializes the SEXP, preserving RIR/PIR-compiled closures, to the given path +* `rir.deserialize`: Deserializes and returns the SEXP at the given path +* `rir.killCompilerServers`: (on client) send a special request to kill compiler servers connected to this client * `.printInvocation`: prints invocation during evaluation * `.int3`: breakpoint during evaluation @@ -405,3 +467,16 @@ In order to use rr inside a docker container, it is necessary to run it with som `docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it registry.gitlab.com/rirvm/rir_mirror/benchmark:SOME_COMMIT_ID` Recording Ř works just fine, with the usual `-d rr` . However, when running `rr replay`, it complains about not being able to find the debug symbols. To overcome this issue type in: `/opt/rir/external/custom-r/bin/exec/R` right after `rr replay` (within the *rr* prompt). + +## CLion + +You can create run/debug configurations for Ř in CLion. + +CLion should be smart enough to automatically generate a CMake configuration from our `CMakeLists.txt`, and thus have some preset run configurations. The one you will use is `rir`. This configuration will already build Ř in a folder called `cmake-build-debug`. However, you must do some manual configuration in order to get it to run properly: + +- Change `executable` to `/external/custom-r/bin/exec/R` (this is the path to the R executable that was built by Ř; replace `` with the actual repo path) +- Set the following environment variables (again, replace `` with the actual repo path): + - On Linux: `LD_LIBRARY_PATH=/external/custom-r/lib;EXTRA_LOAD_R=/rir/R/rir.R;EXTRA_LOAD_SO=/cmake-build-sanitize/Debug/librir.dylib;R_DOC_DIR=/external/custom-r/doc;R_HOME=/external/custom-r;R_HOME_DIR=/external/custom-r;R_INCLUDE_DIR=/external/custom-r/include;R_SHARE_DIR=/external/custom-r/share` + - On macOS: `DYLD_LIBRARY_PATH=/external/custom-r/lib;EXTRA_LOAD_R=/rir/R/rir.R;EXTRA_LOAD_SO=/cmake-build-sanitize/Debug/librir.dylib;R_DOC_DIR=/external/custom-r/doc;R_HOME=/external/custom-r;R_HOME_DIR=/external/custom-r;R_INCLUDE_DIR=/external/custom-r/include;R_SHARE_DIR=/external/custom-r/share` + +This should be enough to get run to start the REPL, and debug to work with breakpoints. You can also redirect input from files (e.g. one of the tests), or add extra environment variables like `PIR_DEBUG`. \ No newline at end of file diff --git a/external/custom-r b/external/custom-r index 6483fffd7..e6f10ecf0 160000 --- a/external/custom-r +++ b/external/custom-r @@ -1 +1 @@ -Subproject commit 6483fffd7edb49bac2e98479ac534cb7e8448475 +Subproject commit e6f10ecf04f60737fcc39f6f53b78be8c23ae296 diff --git a/external/xxHash b/external/xxHash new file mode 160000 index 000000000..35b0373c6 --- /dev/null +++ b/external/xxHash @@ -0,0 +1 @@ +Subproject commit 35b0373c697b5f160d3db26b1cbb45a0d5ba788c diff --git a/rir/R/rir.R b/rir/R/rir.R index 977a01e68..4501b2627 100644 --- a/rir/R/rir.R +++ b/rir/R/rir.R @@ -216,3 +216,16 @@ rir.annotateDepromised <- function(closure) { rir.markFunction(copy, DepromiseArgs=TRUE) copy } + +# Kill compiler servers connected to the client (this is the client) +rir.killCompilerServers <- function() { + .Call("rirKillCompilerServers") +} + +# We need to run this after all static C++ initializers are run +invisible(.Call("initializeUUIDPool")) +invisible(.Call("initializePrintPrettyGraphFromEnv")) +invisible(.Call("initPirTraceSerializationExcludeFlags")) + +# We need to ensure the compiler server starts after ALL code is loaded, so it can't be in initializeRuntime +invisible(.Call("tryToRunCompilerServer")) diff --git a/rir/src/R/Funtab.h b/rir/src/R/Funtab.h index fd609a1cd..ce125294e 100644 --- a/rir/src/R/Funtab.h +++ b/rir/src/R/Funtab.h @@ -24,6 +24,16 @@ static inline int getBuiltinArity(SEXP f) { static inline int getFlag(int i) { return ((R_FunTab[i].eval) / 100) % 10; } static inline int getFlag(SEXP f) { return getFlag(getBuiltinNr(f)); } +static inline SEXP getBuiltinFun(int id) { + assert(R_FunTab[id].eval % 10 == 1 && + "Only use for BUILTINSXP"); + if (R_FunTab[id].eval % 100 / 10 == 0) { + return Rf_install(getBuiltinName(id))->u.symsxp.value; + } else { + return Rf_install(getBuiltinName(id))->u.symsxp.internal; + } +} + static inline SEXP getBuiltinFun(char const* name) { assert(R_FunTab[rir::blt(name)].eval % 10 == 1 && "Only use for BUILTINSXP"); @@ -33,4 +43,20 @@ static inline SEXP getBuiltinFun(char const* name) { return Rf_install(name)->u.symsxp.internal; } +static inline SEXP getBuiltinOrSpecialFun(int id) { + if (R_FunTab[id].eval % 100 / 10 == 0) { + return Rf_install(getBuiltinName(id))->u.symsxp.value; + } else { + return Rf_install(getBuiltinName(id))->u.symsxp.internal; + } +} + +static inline SEXP getBuiltinOrSpecialFun(char const* name) { + if (R_FunTab[rir::blt(name)].eval % 100 / 10 == 0) { + return Rf_install(name)->u.symsxp.value; + } else { + return Rf_install(name)->u.symsxp.internal; + } +} + #endif diff --git a/rir/src/R/Printing.cpp b/rir/src/R/Printing.cpp index 7bb24d4b8..ccc7cce7f 100644 --- a/rir/src/R/Printing.cpp +++ b/rir/src/R/Printing.cpp @@ -8,6 +8,9 @@ #include "runtime/LazyArglist.h" #include "runtime/LazyEnvironment.h" #include "runtime/PirTypeFeedback.h" +#include "runtime/PoolStub.h" +#include "runtime/ProxyEnv.h" +#include "runtime/RirRuntimeObject.h" #include #include @@ -82,36 +85,36 @@ std::string Print::trim(std::string s, size_t n) { return s.substr(0, n - 4) + "|..."; } -std::string Print::dumpPROMSXP(SEXP s) { +std::string Print::dumpPROMSXP(SEXP s, size_t length) { std::stringstream ss; ss << ""; + ss << " env=" << dumpSexp(s->u.promsxp.env, length) << ">"; return ss.str(); } -std::string Print::dumpCLOSXP(SEXP s) { +std::string Print::dumpCLOSXP(SEXP s, size_t length) { std::stringstream ss; ss << "function("; auto f = FORMALS(s); while (f != R_NilValue) { if (TAG(f) != R_NilValue) - ss << dumpSexp(TAG(f)); + ss << dumpSexp(TAG(f), length); if (CAR(f) != R_MissingArg) - ss << "=" << dumpSexp(CAR(f)); + ss << "=" << dumpSexp(CAR(f), length); f = CDR(f); if (f != R_NilValue) ss << ", "; } - ss << ") " << dumpSexp(BODY(s)); - ss << " env=" << dumpSexp(CLOENV(s)); + ss << ") " << dumpSexp(BODY(s), length); + ss << " env=" << dumpSexp(CLOENV(s), length); return ss.str(); } -std::string Print::dumpLISTSXP(SEXP s, size_t limit) { +std::string Print::dumpLISTSXP(SEXP s, size_t limit, size_t length) { std::stringstream ss; ss << "<" << sexptype2char(TYPEOF(s)); @@ -124,11 +127,11 @@ std::string Print::dumpLISTSXP(SEXP s, size_t limit) { ss << " "; out++; if (TAG(s) != R_NilValue) { - auto e = dumpSexp(TAG(s)); + auto e = dumpSexp(TAG(s), length); ss << e << "="; out += 1 + e.length(); } - auto e = dumpSexp(CAR(s)); + auto e = dumpSexp(CAR(s), length); ss << e; out += e.length(); s = CDR(s); @@ -137,17 +140,17 @@ std::string Print::dumpLISTSXP(SEXP s, size_t limit) { return ss.str(); } -std::string Print::dumpLANGSXP(SEXP s) { +std::string Print::dumpLANGSXP(SEXP s, size_t length) { std::stringstream ss; if (s != R_NilValue) { - ss << dumpSexp(CAR(s)); + ss << dumpSexp(CAR(s), length); s = CDR(s); } ss << "("; while (s != R_NilValue) { if (TAG(s) != R_NilValue) - ss << dumpSexp(TAG(s)) << "="; - ss << dumpSexp(CAR(s)); + ss << dumpSexp(TAG(s), length) << "="; + ss << dumpSexp(CAR(s), length); s = CDR(s); if (s != R_NilValue) ss << ", "; @@ -156,7 +159,7 @@ std::string Print::dumpLANGSXP(SEXP s) { return ss.str(); } -std::string Print::dumpVector(SEXP s, size_t limit) { +std::string Print::dumpVector(SEXP s, size_t limit, size_t length) { std::stringstream ss; auto unsafe = unsafeTags(s); @@ -190,7 +193,7 @@ std::string Print::dumpVector(SEXP s, size_t limit) { break; } case STRSXP: { - ss << dumpSexp(STRING_PTR(s)[0]); + ss << dumpSexp(STRING_PTR(s)[0], length); break; } case RAWSXP: { @@ -250,13 +253,13 @@ std::string Print::dumpVector(SEXP s, size_t limit) { } case STRSXP: { // NA checked for CHARSXP in dumpSexp - auto e = dumpSexp(STRING_PTR(s)[i]); + auto e = dumpSexp(STRING_PTR(s)[i], length); ss << e; out += e.length(); break; } case VECSXP: { - auto e = dumpSexp(VECTOR_PTR(s)[i]); + auto e = dumpSexp(VECTOR_PTR(s)[i], length); ss << e; out += e.length(); break; @@ -305,7 +308,7 @@ std::string Print::dumpVector(SEXP s, size_t limit) { return ss.str(); } -std::string Print::dumpEXTERNALSXP(SEXP s) { +std::string Print::dumpEXTERNALSXP(SEXP s, size_t length) { std::stringstream ss; ss << "<"; if (auto p = Code::check(s)) { @@ -316,6 +319,9 @@ std::string Print::dumpEXTERNALSXP(SEXP s) { case Code::Kind::Native: ss << "n "; break; + case Code::Kind::Deserializing: + ss << "ds "; + break; } ss << "(rir::Code*)" << p; if (p->pendingCompilation()) @@ -334,6 +340,16 @@ std::string Print::dumpEXTERNALSXP(SEXP s) { ss << "(rir::LazyEnvironment*)" << p; } else if (auto p = PirTypeFeedback::check(s)) { ss << "(rir::PirTypeFeedback*)" << p; + } else if (auto p = TypeFeedback::check(s)) { + ss << "(rir::TypeFeedback*)" << p; + } else if (auto p = SerialModule::check(s)) { + ss << "(rir::SerialModule*)" << p << " (" << p->firstBitcodeBytes() << ")"; + } else if (auto p = PoolStub::check(s)) { + ss << "(rir::PoolStub*)"; + p->print(ss); + } else if (auto p = ProxyEnv::check(s)) { + ss << "(rir::ProxyEnv*)"; + p->print(ss); } else { assert(false && "missing RirRuntimeObject printing"); } @@ -380,12 +396,12 @@ std::string Print::dumpSexp(SEXP s, size_t length) { } case LISTSXP: { - ss << dumpLISTSXP(s, length); + ss << dumpLISTSXP(s, length, length); break; } case CLOSXP: { - ss << dumpCLOSXP(s); + ss << dumpCLOSXP(s, length); break; } @@ -403,18 +419,29 @@ std::string Print::dumpSexp(SEXP s, size_t length) { auto unsafe = unsafeTags(s); if (unsafe.length()) ss << " |" << unsafe; - ss << " " << s << ">"; + ss << " "; + if (R_IsPackageEnv(s) || R_IsNamespaceEnv(s)) { + ss << (R_IsPackageEnv(s) ? "pkg " : "ns "); + auto name = R_IsPackageEnv(s) ? R_PackageEnvName(s) : R_NamespaceEnvSpec(s); + if (name != R_NilValue) { + assert(TYPEOF(name) == STRSXP); + for (R_xlen_t i = 0; i < XLENGTH(name); i++) { + ss << CHAR(STRING_ELT(name, i)) << " "; + } + } + } + ss << s << ">"; } break; } case PROMSXP: { - ss << dumpPROMSXP(s); + ss << dumpPROMSXP(s, length); break; } case LANGSXP: { - ss << dumpLANGSXP(s); + ss << dumpLANGSXP(s, length); break; } @@ -439,12 +466,12 @@ std::string Print::dumpSexp(SEXP s, size_t length) { case STRSXP: case VECSXP: case RAWSXP: { - ss << dumpVector(s, length); + ss << dumpVector(s, length, length); break; } case EXTERNALSXP: { - ss << dumpEXTERNALSXP(s); + ss << dumpEXTERNALSXP(s, length); break; } diff --git a/rir/src/R/Printing.h b/rir/src/R/Printing.h index ab893c420..e66c432d2 100644 --- a/rir/src/R/Printing.h +++ b/rir/src/R/Printing.h @@ -16,12 +16,12 @@ class Print { static std::string sexptype2char(SEXPTYPE type); static std::string trim(std::string s, size_t n); static std::string unsafeTags(SEXP s); - static std::string dumpPROMSXP(SEXP s); - static std::string dumpCLOSXP(SEXP s); - static std::string dumpLISTSXP(SEXP s, size_t limit); - static std::string dumpLANGSXP(SEXP s); - static std::string dumpVector(SEXP s, size_t limit); - static std::string dumpEXTERNALSXP(SEXP s); + static std::string dumpPROMSXP(SEXP s, size_t length = 50); + static std::string dumpCLOSXP(SEXP s, size_t length = 50); + static std::string dumpLISTSXP(SEXP s, size_t limit, size_t length = 50); + static std::string dumpLANGSXP(SEXP s, size_t length = 50); + static std::string dumpVector(SEXP s, size_t limit, size_t length = 50); + static std::string dumpEXTERNALSXP(SEXP s, size_t length = 50); }; } // namespace rir diff --git a/rir/src/R/Protect.h b/rir/src/R/Protect.h index 550c05645..c5d723930 100644 --- a/rir/src/R/Protect.h +++ b/rir/src/R/Protect.h @@ -23,6 +23,14 @@ class Protect { return value; } + SEXP nullable(SEXP value) { + if (value) { + Rf_protect(value); + ++protectedValues_; + } + return value; + } + ~Protect() { Rf_unprotect(protectedValues_); } private: diff --git a/rir/src/R/Serialize.h b/rir/src/R/Serialize.h index 2080ac3b5..fdc78b06c 100644 --- a/rir/src/R/Serialize.h +++ b/rir/src/R/Serialize.h @@ -2,8 +2,6 @@ #include -#define REXPORT extern "C" - REXPORT SEXP R_serialize(SEXP object, SEXP icon, SEXP ascii, SEXP Sversion, SEXP fun); REXPORT SEXP R_unserialize(SEXP icon, SEXP fun); @@ -30,12 +28,12 @@ REXPORT void OutRefIndex(R_outpstream_t stream, int i); REXPORT int InRefIndex(R_inpstream_t stream, int flags); REXPORT void OutStringVec(R_outpstream_t stream, SEXP s, SEXP ref_table); -static inline void OutChar(R_outpstream_t stream, int chr) { - stream->OutChar(stream, chr); +static inline void OutChar(R_outpstream_t stream, char chr) { + stream->OutChar(stream, (int)chr); } -static inline int InChar(R_inpstream_t stream) { - return stream->InChar(stream); +static inline char InChar(R_inpstream_t stream) { + return (char)stream->InChar(stream); } static inline void OutBytes(R_outpstream_t stream, const void* buf, @@ -46,3 +44,54 @@ static inline void OutBytes(R_outpstream_t stream, const void* buf, static inline void InBytes(R_inpstream_t stream, void* buf, int length) { stream->InBytes(stream, buf, length); } + +static inline bool InBool(R_inpstream_t stream) { + return (bool)stream->InChar(stream); +} + +static inline void OutBool(R_outpstream_t stream, bool b) { + stream->OutChar(stream, (int)b); +} + +static inline void OutUInt(R_outpstream_t stream, unsigned int x) { + OutBytes(stream, &x, sizeof(x)); +} + +static inline unsigned int InUInt(R_inpstream_t stream) { + unsigned int x; + InBytes(stream, &x, sizeof(x)); + return x; +} + +static inline void OutU64(R_outpstream_t stream, uint64_t x) { + OutBytes(stream, &x, sizeof(x)); +} + +static inline uint64_t InU64(R_inpstream_t stream) { + uint64_t x; + InBytes(stream, &x, sizeof(x)); + return x; +} + +static inline void OutSize(R_outpstream_t stream, size_t x) { + OutU64(stream, (uint64_t)x); +} + +static inline size_t InSize(R_inpstream_t stream) { + return (size_t)InU64(stream); +} + +static inline void WriteNullableItem(SEXP s, SEXP ref_table, R_outpstream_t stream) { + OutBool(stream, s != nullptr); + if (s) { + WriteItem(s, ref_table, stream); + } +} + +static inline SEXP ReadNullableItem(SEXP ref_table, R_inpstream_t stream) { + if (InBool(stream)) { + return ReadItem(ref_table, stream); + } else { + return nullptr; + } +} \ No newline at end of file diff --git a/rir/src/R/disableGc.h b/rir/src/R/disableGc.h new file mode 100644 index 000000000..4ac5ba5db --- /dev/null +++ b/rir/src/R/disableGc.h @@ -0,0 +1,31 @@ +// +// Created by Jakob Hain on 7/22/23. +// + +#pragma once + +#include "R/r_incl.h" +#include + +template static ALWAYS_INLINE void disableGc(F f) { + auto gcEnabled = R_GCEnabled; + R_GCEnabled = 0; + f(); + R_GCEnabled = gcEnabled; +} + +template static ALWAYS_INLINE SEXP disableGc2(F f) { + auto gcEnabled = R_GCEnabled; + R_GCEnabled = 0; + auto res = f(); + R_GCEnabled = gcEnabled; + return res; +} + +template static ALWAYS_INLINE rir::UUID disableGc3(F f) { + auto gcEnabled = R_GCEnabled; + R_GCEnabled = 0; + auto res = f(); + R_GCEnabled = gcEnabled; + return res; +} diff --git a/rir/src/R/r.h b/rir/src/R/r.h index d7ee92003..381c9787a 100644 --- a/rir/src/R/r.h +++ b/rir/src/R/r.h @@ -51,6 +51,7 @@ extern FUNTAB R_FunTab[]; extern SEXP R_TrueValue; extern SEXP R_FalseValue; extern SEXP R_LogicalNAValue; +extern int R_GCEnabled; } // Performance critical stuff copied from Rinlinedfun.h @@ -102,6 +103,13 @@ inline R_xlen_t XLENGTH_EX(SEXP x) { return ALTREP(x) ? ALTREP_LENGTH(x) : STDVEC_LENGTH(x); } +/// This is semantically equivalent to LENGTH and XLENGTH, but necessary when +/// write barrier is enabled if x isn't necessarily an actual vector +/// TODO: technically UB so refactor to not rely on this behavior +inline R_xlen_t RAW_LENGTH(SEXP x) { + return ALTREP(x) ? ALTREP_LENGTH(x) : ((VECSEXP) (x))->vecsxp.length; +} + typedef struct { int ibeta, it, irnd, ngrd, machep, negep, iexp, minexp, maxexp; double eps, epsneg, xmin, xmax; diff --git a/rir/src/api.cpp b/rir/src/api.cpp index 205066999..8d5370d0a 100644 --- a/rir/src/api.cpp +++ b/rir/src/api.cpp @@ -4,23 +4,31 @@ #include "api.h" #include "R/Serialize.h" +#include "Rinternals.h" #include "bc/BC.h" #include "bc/Compiler.h" #include "compiler/backend.h" #include "compiler/compiler.h" #include "compiler/log/debug.h" +#include "compiler/native/lower_function_llvm.h" #include "compiler/parameter.h" #include "compiler/pir/closure.h" -#include "compiler/pir/type.h" #include "compiler/test/PirCheck.h" #include "compiler/test/PirTests.h" +#include "compilerClientServer/CompilerClient.h" +#include "compilerClientServer/CompilerServer.h" +#include "compilerClientServer/compiler_server_client_shared_utils.h" #include "interpreter/interp_incl.h" +#include "runtime/DispatchTable.h" +#include "runtime/log/printPrettyGraphFromEnv.h" +#include "serializeHash/hash/UUIDPool.h" +#include "serializeHash/serialize/traceSerialize.h" +#include "utils/ByteBuffer.h" #include "utils/measuring.h" #include #include #include -#include #include using namespace rir; @@ -35,6 +43,11 @@ static bool oldPreserve = false; static unsigned oldSerializeChaos = false; static size_t oldDeoptChaos = false; +bool pir::Parameter::PIR_MEASURE_COMPILED_CLOSURES = + getenv("PIR_MEASURE_COMPILED_CLOSURES") != nullptr && + strtol(getenv("PIR_MEASURE_COMPILED_CLOSURES"), nullptr, 10); + + bool parseDebugStyle(const char* str, pir::DebugStyle& s) { #define V(style) \ if (strcmp(str, #style) == 0) { \ @@ -56,13 +69,9 @@ REXPORT SEXP rirDisassemble(SEXP what, SEXP verbose) { if (!t) Rf_error("Not a rir compiled code (CLOSXP but not DispatchTable)"); - std::cout << "== closure " << what << " (dispatch table " << t << ", env " - << CLOENV(what) << ") ==\n"; - for (size_t entry = 0; entry < t->size(); ++entry) { - Function* f = t->get(entry); - std::cout << "= version " << entry << " (" << f << ") =\n"; - f->disassemble(std::cout); - } + std::cout << "== closure " << what << " (env " << CLOENV(what) << ") ==\n"; + + t->print(std::cout, Rf_asLogical(verbose)); return R_NilValue; } @@ -125,50 +134,50 @@ REXPORT SEXP rirMarkFunction(SEXP what, SEXP which, SEXP reopt_, Function* fun = dt->get(i); if (reopt != NA_LOGICAL) { if (reopt) { - fun->flags.set(Function::MarkOpt); - fun->flags.reset(Function::NotOptimizable); + fun->setFlag(Function::MarkOpt); + fun->resetFlag(Function::NotOptimizable); } else { - fun->flags.reset(Function::MarkOpt); + fun->resetFlag(Function::MarkOpt); } } if (forceInline != NA_LOGICAL) { if (forceInline) - fun->flags.set(Function::ForceInline); + fun->setFlag(Function::ForceInline); else - fun->flags.reset(Function::ForceInline); + fun->resetFlag(Function::ForceInline); } if (disableInline != NA_LOGICAL) { if (disableInline) - fun->flags.set(Function::DisableInline); + fun->setFlag(Function::DisableInline); else - fun->flags.reset(Function::DisableInline); + fun->resetFlag(Function::DisableInline); } if (disableSpecialization != NA_LOGICAL) { if (disableSpecialization) - fun->flags.set(Function::DisableAllSpecialization); + fun->setFlag(Function::DisableAllSpecialization); else - fun->flags.reset(Function::DisableAllSpecialization); + fun->resetFlag(Function::DisableAllSpecialization); } if (disableArgumentTypeSpecialization != NA_LOGICAL) { if (disableArgumentTypeSpecialization) - fun->flags.set(Function::DisableArgumentTypeSpecialization); + fun->setFlag(Function::DisableArgumentTypeSpecialization); else - fun->flags.reset(Function::DisableArgumentTypeSpecialization); + fun->resetFlag(Function::DisableArgumentTypeSpecialization); } if (disableNumArgumentSpecialization != NA_LOGICAL) { if (disableNumArgumentSpecialization) - fun->flags.set(Function::DisableNumArgumentsSpezialization); + fun->setFlag(Function::DisableNumArgumentsSpezialization); else - fun->flags.reset(Function::DisableNumArgumentsSpezialization); + fun->resetFlag(Function::DisableNumArgumentsSpezialization); } bool DISABLE_ANNOTATIONS = getenv("PIR_DISABLE_ANNOTATIONS") ? true : false; if (!DISABLE_ANNOTATIONS) { if (depromiseArgs != NA_LOGICAL) { if (depromiseArgs) - fun->flags.set(Function::DepromiseArgs); + fun->setFlag(Function::DepromiseArgs); else - fun->flags.reset(Function::DepromiseArgs); + fun->resetFlag(Function::DepromiseArgs); } } @@ -243,18 +252,18 @@ static pir::DebugOptions::DebugFlags getInitialDebugFlags() { return flags; } -static std::regex getInitialDebugPassFilter() { +static std::string getInitialDebugPassFilter() { auto filter = getenv("PIR_DEBUG_PASS_FILTER"); if (filter) - return std::regex(filter); - return std::regex(".*"); + return {filter}; + return {".*"}; } -static std::regex getInitialDebugFunctionFilter() { +static std::string getInitialDebugFunctionFilter() { auto filter = getenv("PIR_DEBUG_FUNCTION_FILTER"); if (filter) - return std::regex(filter); - return std::regex(".*"); + return {filter}; + return {".*"}; } static pir::DebugStyle getInitialDebugStyle() { @@ -274,8 +283,8 @@ static pir::DebugStyle getInitialDebugStyle() { return style; } -pir::DebugOptions pir::DebugOptions::DefaultDebugOptions = { - getInitialDebugFlags(), getInitialDebugPassFilter(), +pir::DebugOptions pir::DebugOptions::DefaultDebugOptions = + {getInitialDebugFlags(), getInitialDebugPassFilter(), getInitialDebugFunctionFilter(), getInitialDebugStyle()}; REXPORT SEXP pirSetDebugFlags(SEXP debugFlags) { @@ -288,7 +297,18 @@ REXPORT SEXP pirSetDebugFlags(SEXP debugFlags) { } SEXP pirCompile(SEXP what, const Context& assumptions, const std::string& name, - const pir::DebugOptions& debug) { + const pir::DebugOptions& debug, + std::string* closureVersionPirPrint) { + return pirCompile(what, assumptions, name, debug, + closureVersionPirPrint, SerialOptions::DeepCopy); +} + +SEXP pirCompile(SEXP what, const Context& assumptions, const std::string& name, + const pir::DebugOptions& debug, + std::string* closureVersionPirPrint, + const SerialOptions& serialOpts) { + Protect p(what); + if (!isValidClosureSEXP(what)) { Rf_error("not a compiled closure"); } @@ -296,72 +316,104 @@ SEXP pirCompile(SEXP what, const Context& assumptions, const std::string& name, Rf_error("Cannot optimize compiled expression, only closure"); } - PROTECT(what); - - bool dryRun = debug.includes(pir::DebugFlag::DryRun); - // compile to pir - pir::Module* m = new pir::Module; - pir::Log logger(debug); - logger.title("Compiling " + name); - pir::Compiler cmp(m, logger); - auto compile = [&](pir::ClosureVersion* c) { - logger.flushAll(); - cmp.optimizeModule(); - - if (dryRun) - return; - - rir::Function* done = nullptr; - { - // Single Backend instance, gets destroyed at the end of this block - // to finalize the LLVM module so that we can eagerly compile the - // body - pir::Backend backend(m, logger, name); - auto apply = [&](SEXP body, pir::ClosureVersion* c) { - auto fun = backend.getOrCompile(c); - Protect p(fun->container()); - DispatchTable::unpack(body)->insert(fun); - if (body == BODY(what)) - done = fun; + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_COMPILED_CLOSURES, "api.cpp: pirCompile", what, [&]() { + auto compilerServerHandle = + CompilerClient::pirCompile(what, assumptions, name, debug); + + if (!compilerServerHandle || PIR_CLIENT_DRY_RUN) { + // Actually pirCompile on the client + auto dryRun = debug.includes(pir::DebugFlag::DryRun); + // compile to pir + auto m = new pir::Module; + pir::Log logger(debug); + logger.title("Compiling " + name); + pir::Compiler cmp(m, logger); + auto compile = [&](pir::ClosureVersion* c) { + logger.flushAll(); + cmp.optimizeModule(); + + if (dryRun) + return; + + rir::Function* done = nullptr; + { + // Single Backend instance, gets destroyed at the end of this block to finalize the LLVM module so that we can eagerly compile the body + pir::Backend backend(m, logger, name, serialOpts); + auto apply = [&](SEXP body, pir::ClosureVersion* c) { + auto fun = backend.getOrCompile(c); + p(fun->container()); + DispatchTable::unpack(body)->insert(fun); + if (body == BODY(what)) + done = fun; + }; + m->eachPirClosureVersion([&](pir::ClosureVersion* c) { + if (c->owner()->hasOriginClosure()) { + auto cls = c->owner()->rirClosure(); + auto body = BODY(cls); + auto dt = DispatchTable::unpack(body); + if (dt->contains(c->context())) { + // Dispatch also to versions with pending compilation since we're not evaluating + auto other = dt->dispatch(c->context(), false); + assert(other != dt->baseline()); + assert(other->context() == c->context()); + if (other->body()->isCompiled()) + return; + } + // Don't lower functions that have not been called often, as they have incomplete type-feedback. + if (dt->size() == 1 && + dt->baseline()->invocationCount() < 2) + return; + apply(body, c); + } + }); + if (!done) + apply(BODY(what), c); + } + // Eagerly compile the main function + done->body()->nativeCode(); + if (closureVersionPirPrint) { + *closureVersionPirPrint = + printClosureVersionForCompilerServerComparison(c); + } + if (compilerServerHandle) { + // Compare compiled version with remote for discrepancies + compilerServerHandle->compare(c); + } }; - m->eachPirClosureVersion([&](pir::ClosureVersion* c) { - if (c->owner()->hasOriginClosure()) { - auto cls = c->owner()->rirClosure(); - auto body = BODY(cls); - auto dt = DispatchTable::unpack(body); - if (dt->contains(c->context())) { - // Dispatch also to versions with pending compilation - // since we're not evaluating - auto other = dt->dispatch(c->context(), false); - assert(other != dt->baseline()); - assert(other->context() == c->context()); - if (other->body()->isCompiled()) - return; - } - // Don't lower functions that have not been called often, as - // they have incomplete type-feedback. - if (dt->size() == 1 && - dt->baseline()->invocationCount() < 2) - return; - apply(body, c); + + cmp.compileClosure( + what, name, assumptions, true, compile, + [&]() { + if (debug.includes(pir::DebugFlag::ShowWarnings)) + std::cerr << "Compilation failed\n"; + }, + {}); + + delete m; + } else { + if (debug.flags.contains(pir::DebugFlag::PrintFinalPir)) { + auto finalPir = compilerServerHandle->getFinalPir(); + std::cerr << "Final PIR of '" << name << "':\n" + << finalPir << "\n"; + } + + // replace with the compiler server's version + auto newWhat = compilerServerHandle->getSexp(); + auto dt = DispatchTable::unpack(BODY(what)); + auto newDt = DispatchTable::unpack(BODY(newWhat)); + for (unsigned i = 0; i < newDt->size(); ++i) { + if (i == 0) { + dt->baseline(newDt->baseline()); + } else { + dt->insert(newDt->get(i)); } - }); - if (!done) - apply(BODY(what), c); + } } - // Eagerly compile the main function - done->body()->nativeCode(); - }; + delete compilerServerHandle; + }); - cmp.compileClosure(what, name, assumptions, true, compile, - [&]() { - if (debug.includes(pir::DebugFlag::ShowWarnings)) - std::cerr << "Compilation failed\n"; - }, - {}); + printPrettyGraphOfCompiledIfNecessary(what, name); - delete m; - UNPROTECT(1); return what; } @@ -603,12 +655,53 @@ REXPORT SEXP rirCreateSimpleIntContext() { return res; } +REXPORT SEXP rirKillCompilerServers() { + R_Visible = (Rboolean)false; + if (!CompilerClient::isRunning()) { + Rf_warning("Compiler client isn't running"); + return R_NilValue; + } + CompilerClient::killServers(); + return R_NilValue; +} + +REXPORT SEXP initializeUUIDPool() { + UUIDPool::initialize(); + R_Visible = (Rboolean)false; + return R_NilValue; +} + +REXPORT SEXP initializePrintPrettyGraphFromEnv() { + rir::initializePrintPrettyGraphFromEnv(); + R_Visible = (Rboolean)false; + return R_NilValue; +} + +REXPORT SEXP initPirTraceSerializationExcludeFlags() { + rir::initPirTraceSerializationExcludeFlags(); + R_Visible = (Rboolean)false; + return R_NilValue; +} + +REXPORT SEXP tryToRunCompilerServer() { + CompilerServer::tryRun(); + R_Visible = (Rboolean)false; + return R_NilValue; +} + REXPORT SEXP playground() { return R_NilValue; } bool startup() { + if (getenv("R_DISABLE_GC") && + strcmp(getenv("R_DISABLE_GC"), "") != 0 && + strcmp(getenv("R_DISABLE_GC"), "0") != 0 && + strcmp(getenv("R_DISABLE_GC"), "false") != 0) { + Rf_warning("R GC is disabled"); + R_GCEnabled = false; + } initializeRuntime(); return true; } diff --git a/rir/src/api.h b/rir/src/api.h index 2ef31ee4f..74dfa2503 100644 --- a/rir/src/api.h +++ b/rir/src/api.h @@ -7,10 +7,14 @@ #include -#define REXPORT extern "C" - extern int R_ENABLE_JIT; +namespace rir { + +struct SerialOptions; + +}; // namespace rir + REXPORT SEXP rirInvocationCount(SEXP what); REXPORT SEXP pirCompileWrapper(SEXP closure, SEXP name, SEXP debugFlags, SEXP debugStyle); @@ -19,7 +23,12 @@ REXPORT SEXP pirTests(); REXPORT SEXP pirCheck(SEXP f, SEXP check, SEXP env); REXPORT SEXP pirSetDebugFlags(SEXP debugFlags); SEXP pirCompile(SEXP closure, const rir::Context& assumptions, - const std::string& name, const rir::pir::DebugOptions& debug); + const std::string& name, const rir::pir::DebugOptions& debug, + std::string* closureVersionPirPrint = nullptr); +SEXP pirCompile(SEXP closure, const rir::Context& assumptions, + const std::string& name, const rir::pir::DebugOptions& debug, + std::string* closureVersionPirPrint, + const rir::SerialOptions& serialOpts); extern SEXP rirOptDefaultOpts(SEXP closure, const rir::Context&, SEXP name); extern SEXP rirOptDefaultOptsDryrun(SEXP closure, const rir::Context&, SEXP name); @@ -29,6 +38,15 @@ REXPORT SEXP rirDeserialize(SEXP file); REXPORT SEXP rirSetUserContext(SEXP f, SEXP udc); REXPORT SEXP rirCreateSimpleIntContext(); +REXPORT SEXP initializeUUIDPool(); +REXPORT SEXP initializePrintPrettyGraphFromEnv(); +REXPORT SEXP initPirTraceSerializationExcludeFlags(); +/// Send a message from the compiler client (this) to each connected compiler +/// server, which kills the server (exit 0) on receive. Then stops the client +/// for the remainder of the session +REXPORT SEXP rirKillCompilerServers(); +REXPORT SEXP tryToRunCompilerServer(); + // this method is just to have an easy way to play around with the code and get // feedback by calling .Call('playground') REXPORT SEXP playground(); diff --git a/rir/src/bc/BC.cpp b/rir/src/bc/BC.cpp index 1dea8a49a..a7bc88278 100644 --- a/rir/src/bc/BC.cpp +++ b/rir/src/bc/BC.cpp @@ -1,14 +1,12 @@ #include "BC.h" -#include "R/Funtab.h" #include "R/Printing.h" -#include "R/RList.h" -#include "R/Serialize.h" #include "R/r.h" #include "bc/CodeStream.h" +#include "runtime/log/printRirObject.h" +#include "serializeHash/serialize/serialize.h" #include "utils/Pool.h" #include -#include namespace rir { @@ -24,23 +22,6 @@ void BC::write(CodeStream& cs) const { cs.insert(immediate.cacheIdx); return; - case Opcode::record_call_: - // Call feedback targets are stored in the code extra pool. We don't - // have access to them here, so we can't write a call feedback with - // preseeded values. - assert(immediate.callFeedback.numTargets == 0 && - "cannot write call feedback targets"); - cs.insert(immediate.callFeedback); - return; - - case Opcode::record_test_: - cs.insert(immediate.testFeedback); - break; - - case Opcode::record_type_: - cs.insert(immediate.typeFeedback); - break; - case Opcode::push_: case Opcode::ldfun_: case Opcode::ldddvar_: @@ -96,6 +77,9 @@ void BC::write(CodeStream& cs) const { case Opcode::pull_: case Opcode::is_: case Opcode::put_: + case Opcode::record_call_: + case Opcode::record_test_: + case Opcode::record_type_: cs.insert(immediate.i); return; @@ -113,14 +97,15 @@ SEXP BC::immediateConst() const { return Pool::get(immediate.pool); } -// #define DEBUG_SERIAL #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" -void BC::deserialize(SEXP refTable, R_inpstream_t inp, Opcode* code, +void BC::deserialize(AbstractDeserializer& deserializer, Opcode* code, size_t codeSize, Code* container) { while (codeSize > 0) { - *code = (Opcode)InChar(inp); + if (deserializer.willRead(SerialFlags::CodeMisc)) { + *code = deserializer.readBytesOf(SerialFlags::CodeMisc); + } unsigned size = BC::fixedSize(*code); ImmediateArguments& i = *(ImmediateArguments*)(code + 1); switch (*code) { @@ -139,46 +124,57 @@ void BC::deserialize(SEXP refTable, R_inpstream_t inp, Opcode* code, case Opcode::stvar_: case Opcode::stvar_super_: case Opcode::missing_: - i.pool = Pool::insert(ReadItem(refTable, inp)); + DESERIALIZE(i.pool, readConst, SerialFlags::CodeMisc); break; case Opcode::ldvar_cached_: case Opcode::ldvar_for_update_cache_: case Opcode::stvar_cached_: - i.poolAndCache.poolIndex = Pool::insert(ReadItem(refTable, inp)); - i.poolAndCache.cacheIndex = InInteger(inp); + DESERIALIZE(i.poolAndCache.poolIndex, readConst, SerialFlags::CodeMisc); + DESERIALIZE(i.poolAndCache.cacheIndex, readBytesOf, SerialFlags::CodeMisc); break; case Opcode::guard_fun_: - i.guard_fun_args.name = Pool::insert(ReadItem(refTable, inp)); - i.guard_fun_args.expected = Pool::insert(ReadItem(refTable, inp)); - i.guard_fun_args.id = InInteger(inp); + DESERIALIZE(i.guard_fun_args.name, readConst, SerialFlags::CodeMisc); + DESERIALIZE(i.guard_fun_args.expected, readConst, SerialFlags::CodeMisc); + DESERIALIZE(i.guard_fun_args.id, readBytesOf, SerialFlags::CodeMisc); break; case Opcode::call_: case Opcode::named_call_: - case Opcode::call_dots_: { - i.callFixedArgs.nargs = InInteger(inp); - i.callFixedArgs.ast = Pool::insert(ReadItem(refTable, inp)); - InBytes(inp, &i.callFixedArgs.given, sizeof(Context)); - Opcode* c = code + 1 + sizeof(CallFixedArgs); - // Read implicit promise argument offsets - // Read named arguments - if (*code == Opcode::named_call_ || *code == Opcode::call_dots_) { - PoolIdx* names = (PoolIdx*)c; - for (size_t j = 0; j < i.callFixedArgs.nargs; j++) - names[j] = Pool::insert(ReadItem(refTable, inp)); + case Opcode::call_dots_: + if (deserializer.willRead(SerialFlags::CodeMisc)) { + i.callFixedArgs.nargs = + deserializer.readBytesOf(SerialFlags::CodeMisc); + i.callFixedArgs.ast = + deserializer.readConst(SerialFlags::CodeMisc); + i.callFixedArgs.given = + Context(deserializer.readBytesOf( + SerialFlags::CodeMisc)); + Opcode* c = code + 1 + sizeof(CallFixedArgs); + // Read implicit promise argument offsets + // Read named arguments + if (*code == Opcode::named_call_ || + *code == Opcode::call_dots_) { + auto names = (PoolIdx*)c; + for (size_t j = 0; j < i.callFixedArgs.nargs; j++) { + names[j] = + deserializer.readConst(SerialFlags::CodeMisc); + } + } } break; - } case Opcode::call_builtin_: - i.callBuiltinFixedArgs.nargs = InInteger(inp); - i.callBuiltinFixedArgs.ast = Pool::insert(ReadItem(refTable, inp)); - i.callBuiltinFixedArgs.builtin = - Pool::insert(ReadItem(refTable, inp)); + DESERIALIZE(i.callBuiltinFixedArgs.nargs, readBytesOf, SerialFlags::CodeMisc); + DESERIALIZE(i.callBuiltinFixedArgs.ast, readConst, SerialFlags::CodeMisc); + DESERIALIZE(i.callBuiltinFixedArgs.builtin, readConst, SerialFlags::CodeMisc); + break; + case Opcode::mk_promise_: + case Opcode::mk_eager_promise_: + DESERIALIZE(i.fun, readBytesOf, SerialFlags::CodeMisc); break; case Opcode::record_call_: case Opcode::record_type_: case Opcode::record_test_: - case Opcode::mk_promise_: - case Opcode::mk_eager_promise_: + DESERIALIZE(i.i, readBytesOf, SerialFlags::CodeFeedback); + break; case Opcode::br_: case Opcode::brtrue_: case Opcode::beginloop_: @@ -190,7 +186,10 @@ void BC::deserialize(SEXP refTable, R_inpstream_t inp, Opcode* code, case Opcode::put_: case Opcode::clear_binding_cache_: assert((size - 1) % 4 == 0); - InBytes(inp, code + 1, size - 1); + if (size > 1 && deserializer.willRead(SerialFlags::CodeMisc)) { + deserializer.readBytes((void*)(code + 1), size - 1, + SerialFlags::CodeMisc); + } break; case Opcode::invalid_: case Opcode::num_of: @@ -198,24 +197,19 @@ void BC::deserialize(SEXP refTable, R_inpstream_t inp, Opcode* code, break; } size = BC::size(code); -#ifdef DEBUG_SERIAL - if (*code == Opcode::deopt_) { - BC aBc = BC::decode(code, container); - std::cout << "deserialized: "; - aBc.print(std::cout); - } -#endif assert(codeSize >= size); code += size; codeSize -= size; } } -void BC::serialize(SEXP refTable, R_outpstream_t out, const Opcode* code, - size_t codeSize, const Code* container) { +void BC::serialize(AbstractSerializer& serializer, + std::vector& extraPoolFlags, + const Opcode* code, size_t codeSize, + const Code* container) { while (codeSize > 0) { - const BC bc = BC::decode((Opcode*)code, container); - OutChar(out, (int)*code); + const auto bc = BC::decode((Opcode*)code, container); + serializer.writeBytesOf(*code, SerialFlags::CodeMisc); unsigned size = BC::fixedSize(*code); ImmediateArguments i = bc.immediate; switch (*code) { @@ -234,40 +228,165 @@ void BC::serialize(SEXP refTable, R_outpstream_t out, const Opcode* code, case Opcode::stvar_: case Opcode::stvar_super_: case Opcode::missing_: - WriteItem(Pool::get(i.pool), refTable, out); + serializer.writeConst(i.pool, SerialFlags::CodeMisc); break; case Opcode::ldvar_cached_: case Opcode::ldvar_for_update_cache_: case Opcode::stvar_cached_: - WriteItem(Pool::get(i.poolAndCache.poolIndex), refTable, out); - OutInteger(out, i.poolAndCache.cacheIndex); + serializer.writeConst(i.poolAndCache.poolIndex, SerialFlags::CodeMisc); + serializer.writeBytesOf(i.poolAndCache.cacheIndex, SerialFlags::CodeMisc); break; case Opcode::guard_fun_: - WriteItem(Pool::get(i.guard_fun_args.name), refTable, out); - WriteItem(Pool::get(i.guard_fun_args.expected), refTable, out); - OutInteger(out, i.guard_fun_args.id); + serializer.writeConst(i.guard_fun_args.name, SerialFlags::CodeMisc); + serializer.writeConst(i.guard_fun_args.expected, SerialFlags::CodeMisc); + serializer.writeBytesOf(i.guard_fun_args.id, SerialFlags::CodeMisc); break; case Opcode::call_: case Opcode::call_dots_: case Opcode::named_call_: - OutInteger(out, i.callFixedArgs.nargs); - WriteItem(Pool::get(i.callFixedArgs.ast), refTable, out); - OutBytes(out, &i.callFixedArgs.given, sizeof(Context)); + serializer.writeBytesOf(i.callFixedArgs.nargs, SerialFlags::CodeMisc); + serializer.writeConst(i.callFixedArgs.ast, SerialFlags::CodeMisc); + serializer.writeBytesOf(i.callFixedArgs.given.toI(), SerialFlags::CodeMisc); // Write named arguments if (*code == Opcode::named_call_ || *code == Opcode::call_dots_) { - for (size_t j = 0; j < i.callFixedArgs.nargs; j++) - WriteItem(Pool::get(bc.callExtra().callArgumentNames[j]), - refTable, out); + for (size_t j = 0; j < i.callFixedArgs.nargs; j++) { + serializer.writeConst(bc.callExtra().callArgumentNames[j], SerialFlags::CodeMisc); + } } break; case Opcode::call_builtin_: - OutInteger(out, i.callBuiltinFixedArgs.nargs); - WriteItem(Pool::get(i.callBuiltinFixedArgs.ast), refTable, out); - WriteItem(Pool::get(i.callBuiltinFixedArgs.builtin), refTable, out); + serializer.writeBytesOf(i.callBuiltinFixedArgs.nargs, SerialFlags::CodeMisc); + serializer.writeConst(i.callBuiltinFixedArgs.ast, SerialFlags::CodeMisc); + serializer.writeConst(i.callBuiltinFixedArgs.builtin, SerialFlags::CodeMisc); + break; + case Opcode::mk_promise_: + case Opcode::mk_eager_promise_: + serializer.writeBytesOf(i.fun, SerialFlags::CodeMisc); + extraPoolFlags[i.fun] = SerialFlags::CodePromise; break; case Opcode::record_call_: + serializer.writeBytesOf(i.i, SerialFlags::CodeFeedback); + if (container->function()->body() == container) { + // The feedback itself is already serialized, but we also want to record which extra pool entries are part of it + auto feedback = + container->function()->typeFeedback()->callees(i.i); + // Don't hash because this is a recording instruction, + // but we also want to skip hashing recorded extra pool entries + for (size_t j = 0; j < feedback.numTargets; j++) { + extraPoolFlags[feedback.targets[j]] = + SerialFlags::CodeFeedback; + } + } + break; + case Opcode::record_type_: + case Opcode::record_test_: + serializer.writeBytesOf(i.i, SerialFlags::CodeFeedback); + break; + case Opcode::br_: + case Opcode::brtrue_: + case Opcode::beginloop_: + case Opcode::brfalse_: + case Opcode::popn_: + case Opcode::pick_: + case Opcode::pull_: + case Opcode::is_: + case Opcode::put_: + case Opcode::clear_binding_cache_: + assert((size - 1) % 4 == 0); + if (size > 1) { + serializer.writeBytes((void*)(code + 1), size - 1, SerialFlags::CodeMisc); + } + break; + case Opcode::invalid_: + case Opcode::num_of: + assert(false); + break; + } + size = bc.size(); + assert(codeSize >= size); + code += size; + codeSize -= size; + } +} + +void BC::hash(HasherOld& hasher, std::vector& extraPoolIgnored, + const Opcode* code, size_t codeSize, const Code* container) { + while (codeSize > 0) { + const BC bc = BC::decode((Opcode*)code, container); + hasher.hashBytesOf(*code); + unsigned size = BC::fixedSize(*code); + ImmediateArguments i = bc.immediate; + switch (*code) { +#define V(NESTED, name, name_) case Opcode::name_##_: + BC_NOARGS(V, _) +#undef V + assert(*code != Opcode::nop_); + break; + case Opcode::push_: + case Opcode::ldfun_: + case Opcode::ldddvar_: + case Opcode::ldvar_: + case Opcode::ldvar_noforce_: + case Opcode::ldvar_for_update_: + case Opcode::ldvar_super_: + case Opcode::stvar_: + case Opcode::stvar_super_: + case Opcode::missing_: + hasher.hashConstant(i.pool); + break; + case Opcode::ldvar_cached_: + case Opcode::ldvar_for_update_cache_: + case Opcode::stvar_cached_: + hasher.hashConstant(i.poolAndCache.poolIndex); + hasher.hashBytesOf(i.poolAndCache.cacheIndex); + break; + case Opcode::guard_fun_: + hasher.hashConstant(i.guard_fun_args.name); + hasher.hashConstant(i.guard_fun_args.expected); + hasher.hashBytesOf(i.guard_fun_args.id); + break; + case Opcode::call_: + case Opcode::call_dots_: + case Opcode::named_call_: + hasher.hashBytesOf(i.callFixedArgs.nargs); + hasher.hashConstant(i.callFixedArgs.ast); + hasher.hashBytesOf(i.callFixedArgs.given); + // Hash named arguments + if (*code == Opcode::named_call_ || *code == Opcode::call_dots_) { + for (size_t j = 0; j < i.callFixedArgs.nargs; j++) { + hasher.hashConstant(bc.callExtra().callArgumentNames[j]); + } + } + break; + case Opcode::call_builtin_: + hasher.hashBytesOf(i.callBuiltinFixedArgs.nargs); + hasher.hashConstant(i.callBuiltinFixedArgs.ast); + hasher.hashConstant(i.callBuiltinFixedArgs.builtin); + break; + case Opcode::record_call_: { + auto feedback = container->function()->typeFeedback(); + if (i.i >= feedback->numCallees()) { + // TODO: Bug where, when we only send the compiler server the + // client source and feedback, we get record_call instructions + // with corrupt indices + std::cerr << "BC.cpp hash record_call_ index out of range\n"; + break; + } + auto callees = feedback->callees(i.i); + if (container->function()->body() == container) { + // Don't hash because this is a recording instruction, + // but we also want to skip hashing recorded extra pool entries + for (size_t j = 0; j < callees.numTargets; j++) { + extraPoolIgnored[callees.targets[j]] = true; + } + } + break; + } case Opcode::record_type_: case Opcode::record_test_: + assert((size - 1) % 4 == 0); + // Don't hash because these are recording instructions + break; case Opcode::mk_promise_: case Opcode::mk_eager_promise_: case Opcode::br_: @@ -281,8 +400,9 @@ void BC::serialize(SEXP refTable, R_outpstream_t out, const Opcode* code, case Opcode::put_: case Opcode::clear_binding_cache_: assert((size - 1) % 4 == 0); - if (size != 0) - OutBytes(out, code + 1, size - 1); + if (size > 1) { + hasher.hashBytes(code + 1, (int)size - 1); + } break; case Opcode::invalid_: case Opcode::num_of: @@ -290,18 +410,277 @@ void BC::serialize(SEXP refTable, R_outpstream_t out, const Opcode* code, break; } size = bc.size(); -#ifdef DEBUG_SERIAL - if (bc.bc == Opcode::deopt_) { - std::cout << "serialized: "; - bc.print(std::cout); + assert(codeSize >= size); + code += size; + codeSize -= size; + } +} + +void BC::addConnected(std::vector& extraPoolChildren, + ConnectedCollectorOld& collector, const Opcode* code, + size_t codeSize, const Code* container) { + while (codeSize > 0) { + const BC bc = BC::decode((Opcode*)code, container); + unsigned size = BC::fixedSize(*code); + ImmediateArguments i = bc.immediate; + switch (*code) { +#define V(NESTED, name, name_) case Opcode::name_##_: + BC_NOARGS(V, _) +#undef V + assert(*code != Opcode::nop_); + break; + case Opcode::push_: + case Opcode::ldfun_: + case Opcode::ldddvar_: + case Opcode::ldvar_: + case Opcode::ldvar_noforce_: + case Opcode::ldvar_for_update_: + case Opcode::ldvar_super_: + case Opcode::stvar_: + case Opcode::stvar_super_: + case Opcode::missing_: + collector.addConstant(i.pool); + break; + case Opcode::ldvar_cached_: + case Opcode::ldvar_for_update_cache_: + case Opcode::stvar_cached_: + collector.addConstant(i.poolAndCache.poolIndex); + break; + case Opcode::guard_fun_: + collector.addConstant(i.guard_fun_args.name); + collector.addConstant(i.guard_fun_args.expected); + break; + case Opcode::call_: + case Opcode::call_dots_: + case Opcode::named_call_: + collector.addConstant(i.callFixedArgs.ast); + // Add named arguments + if (*code == Opcode::named_call_ || *code == Opcode::call_dots_) { + for (size_t j = 0; j < i.callFixedArgs.nargs; j++) { + collector.addConstant(bc.callExtra().callArgumentNames[j]); + } + } + break; + case Opcode::call_builtin_: + collector.addConstant(i.callBuiltinFixedArgs.ast); + collector.addConstant(i.callBuiltinFixedArgs.builtin); + break; + case Opcode::record_call_: + case Opcode::record_type_: + case Opcode::record_test_: + break; + case Opcode::mk_promise_: + case Opcode::mk_eager_promise_: + extraPoolChildren[i.fun] = true; + break; + case Opcode::br_: + case Opcode::brtrue_: + case Opcode::beginloop_: + case Opcode::brfalse_: + case Opcode::popn_: + case Opcode::pick_: + case Opcode::pull_: + case Opcode::is_: + case Opcode::put_: + case Opcode::clear_binding_cache_: + break; + case Opcode::invalid_: + case Opcode::num_of: + assert(false); } -#endif + size = bc.size(); assert(codeSize >= size); code += size; codeSize -= size; } } +void BC::addToPrettyGraph(const PrettyGraphInnerPrinter& p, + std::vector& addedExtraPoolEntries, + const Opcode* code, size_t codeSize, + const Code* container) { + auto addEntry = [&](SEXP sexp, const char* type, PrettyGraphContentPrinter description){ + bool isInPool = false; + for (unsigned i = 0; i < container->extraPoolSize; i++) { + if (sexp == container->getExtraPoolEntry(i)) { + addedExtraPoolEntries[i] = true; + isInPool = true; + } + } + if (TYPEOF(sexp) == EXTERNALSXP) { + p.addEdgeTo(sexp, false, type, description, + !isInPool); + } + }; + auto addConstant = [&](PoolIdx idx, const char* type, PrettyGraphContentPrinter description = [](std::ostream& s){}){ + addEntry(Pool::get(idx), type, description); + }; + auto addExtraPoolEntry = [&](PoolIdx idx, bool isChild, const char* type, PrettyGraphContentPrinter description = [](std::ostream& s){}){ + auto sexp = container->getExtraPoolEntry(idx); + addedExtraPoolEntries[idx] = true; + p.addEdgeTo(sexp, isChild, type, description); + }; + + while (codeSize > 0) { + const BC bc = BC::decode((Opcode*)code, container); + unsigned size = BC::fixedSize(*code); + ImmediateArguments i = bc.immediate; + switch (*code) { +#define V(NESTED, name, name_) case Opcode::name_##_: + BC_NOARGS(V, _) +#undef V + assert(*code != Opcode::nop_); + break; +#define CONSTANT_CASE(op, accessor, type) case Opcode::op##_: \ + addConstant(i.accessor, type); \ + break; + CONSTANT_CASE(push, pool, "push") + CONSTANT_CASE(ldfun, pool, "unexpected-name") // NOLINT(*-branch-clone) + CONSTANT_CASE(ldddvar, pool, "unexpected-name") + CONSTANT_CASE(ldvar, pool, "unexpected-name") + CONSTANT_CASE(ldvar_noforce, pool, "unexpected-name") + CONSTANT_CASE(ldvar_for_update, pool, "unexpected-name") + CONSTANT_CASE(ldvar_super, pool, "unexpected-name") + CONSTANT_CASE(stvar, pool, "unexpected-name") + CONSTANT_CASE(stvar_super, pool, "unexpected-name") + CONSTANT_CASE(missing, pool, "unexpected-name") + CONSTANT_CASE(ldvar_cached, poolAndCache.poolIndex, "unexpected-name") // NOLINT(*-branch-clone) + CONSTANT_CASE(ldvar_for_update_cache, poolAndCache.poolIndex, "unexpected-name") + CONSTANT_CASE(stvar_cached, poolAndCache.poolIndex, "unexpected-name") + case Opcode::guard_fun_: + addConstant(i.guard_fun_args.name, "unexpected-name", [&](std::ostream& s){ + s << "guard_fun"; + }); + addConstant(i.guard_fun_args.expected, "guard"); + break; + case Opcode::call_: + case Opcode::call_dots_: + case Opcode::named_call_: { + auto callType = + *code == Opcode::call_ ? "call" : + *code == Opcode::call_dots_ ? "call_dots" : + "named_call"; + addConstant(i.callFixedArgs.ast, "unexpected-ast", [&](std::ostream& s){ + s << callType << " ast"; + }); + // Add named arguments + if (*code == Opcode::named_call_ || *code == Opcode::call_dots_) { + for (size_t j = 0; j < i.callFixedArgs.nargs; j++) { + addConstant(bc.callExtra().callArgumentNames[j], "unexpected-name", [&](std::ostream& s){ + s << callType << " argument"; + }); + } + } + break; + } + case Opcode::call_builtin_: + addConstant(i.callBuiltinFixedArgs.ast, "unexpected-ast"); + addConstant(i.callBuiltinFixedArgs.builtin, "unexpected-builtin"); + break; + case Opcode::record_call_: + if (container->function()->body() == container) { + auto feedback = + container->function()->typeFeedback()->callees(i.i); + for (auto j = 0; j < feedback.numTargets; j++) { + addExtraPoolEntry( + feedback.targets[j], false, "target", + [&](std::ostream& s) { s << "record_call " << j; }); + } + } + break; + case Opcode::record_type_: + case Opcode::record_test_: + break; + case Opcode::mk_promise_: + case Opcode::mk_eager_promise_: + addExtraPoolEntry(i.fun, true, "promise"); + break; + case Opcode::br_: + case Opcode::brtrue_: + case Opcode::beginloop_: + case Opcode::brfalse_: + case Opcode::popn_: + case Opcode::pick_: + case Opcode::pull_: + case Opcode::is_: + case Opcode::put_: + case Opcode::clear_binding_cache_: + break; + case Opcode::invalid_: + case Opcode::num_of: + // TODO: mark extra pool entry and add edge for any other bytecodes + // which reference extra pool entries + assert(false); + break; + } + size = bc.size(); + assert(codeSize >= size); + code += size; + codeSize -= size; + } +} + +/// Compare bytecodes and print differences. +void BC::debugCompare(const Opcode* code1, const Opcode* code2, + size_t codeSize1, size_t codeSize2, + const Code* container1, const Code* container2, + const char* prefix, std::stringstream& differences) { + auto loggedDifferences = false; + auto initialCodeSize1 = codeSize1; + while (codeSize1 > 0 && codeSize2 > 0) { + auto pc1 = (Opcode*)code1; + auto pc2 = (Opcode*)code2; + auto opcode1 = *pc1; + auto opcode2 = *pc2; + const BC bc1 = BC::decode(pc1, container1); + const BC bc2 = BC::decode(pc2, container2); + auto size1 = BC::fixedSize(opcode1); + auto size2 = BC::fixedSize(opcode2); + if (opcode1 != opcode2 || size1 != size2 || + (memcmp(pc1, pc2, size1) != 0 && + // For non-trivial SEXPs like environments, calls will push + // different values + opcode1 != Opcode::push_)) { + // Even if the bytecode data is different, it could just be different pool + // entries for equivalent SEXPs. So we check by printing the bytecode (not + // perfect, there's a slim chance of true negative, but good enough) + std::string associated1; + std::string associated2; + if (opcode1 == opcode2) { + std::stringstream associated1Stream; + bc1.printAssociatedData(associated1Stream, true); + std::stringstream associated2Stream; + bc2.printAssociatedData(associated2Stream, true); + associated1 = associated1Stream.str(); + associated2 = associated2Stream.str(); + } + if (opcode1 != opcode2 || associated1 != associated2) { + if (!loggedDifferences) { + differences << prefix << " bytecode differs, first at " + << initialCodeSize1 - codeSize1 << "\n" + << prefix << " bytecode:"; + loggedDifferences = true; + } + differences << " "; + if (opcode1 == opcode2) { + differences << name(opcode1) << "(" << associated1 << ")|(" << associated2 << ")"; + } else { + differences << name(opcode1) << "|" << name(opcode2); + } + } + } + size1 = bc1.size(); + size2 = bc2.size(); + code1 += size1; + code2 += size2; + codeSize1 -= size1; + codeSize2 -= size2; + } + if (loggedDifferences) { + differences << "\n"; + } +} + #pragma GCC diagnostic pop void BC::printImmediateArgs(std::ostream& out) const { @@ -331,10 +710,20 @@ void BC::printOpcode(std::ostream& out) const { out << name(bc) << " "; } void BC::print(std::ostream& out) const { out << " "; - if (bc != Opcode::record_call_ && bc != Opcode::record_type_ && - bc != Opcode::record_test_) - printOpcode(out); + printOpcode(out); + printAssociatedData(out); + out << "\n"; +} + +void BC::printAssociatedData(std::ostream& out, bool printDetailed) const { + auto printSexp = [&](SEXP s) { + if (printDetailed) { + printRirObject(s, out); + } else { + out << Print::dumpSexp(s); + } + }; switch (bc) { case Opcode::invalid_: case Opcode::num_of: @@ -364,11 +753,12 @@ void BC::print(std::ostream& out) const { auto args = immediate.callBuiltinFixedArgs; BC::NumArgs nargs = args.nargs; auto target = Pool::get(args.builtin); - out << nargs << " : " << Print::dumpSexp(target); + out << nargs << " : "; + printSexp(target); break; } case Opcode::push_: - out << Print::dumpSexp(immediateConst()); + printSexp(immediateConst()); break; case Opcode::ldfun_: case Opcode::ldvar_: @@ -402,54 +792,11 @@ void BC::print(std::ostream& out) const { case Opcode::is_: out << (BC::RirTypecheck)immediate.i; break; - case Opcode::record_call_: { - ObservedCallees prof = immediate.callFeedback; - out << "[ "; - if (prof.taken == ObservedCallees::CounterOverflow) - out << "*, <"; - else - out << prof.taken << ", <"; - if (prof.numTargets == ObservedCallees::MaxTargets) - out << "*>, "; - else - out << prof.numTargets << ">, "; - - out << (prof.invalid ? "invalid" : "valid"); - out << (prof.numTargets ? ", " : " "); - - for (int i = 0; i < prof.numTargets; ++i) - out << callFeedbackExtra().targets[i] << "(" - << Rf_type2char(TYPEOF(callFeedbackExtra().targets[i])) << ") "; - out << "]"; - break; - } - - case Opcode::record_test_: { - out << "[ "; - switch (immediate.testFeedback.seen) { - case ObservedTest::None: - out << "_"; - break; - case ObservedTest::OnlyTrue: - out << "T"; - break; - case ObservedTest::OnlyFalse: - out << "F"; - break; - case ObservedTest::Both: - out << "?"; - break; - } - out << " ]"; - break; - } - - case Opcode::record_type_: { - out << "[ "; - immediate.typeFeedback.print(out); - out << " ]"; + case Opcode::record_test_: + case Opcode::record_type_: + case Opcode::record_call_: + out << "#" << immediate.i; break; - } #define V(NESTED, name, name_) case Opcode::name_##_: BC_NOARGS(V, _) @@ -469,7 +816,6 @@ void BC::print(std::ostream& out) const { out << immediate.cacheIdx.start << " " << immediate.cacheIdx.size; break; } - out << "\n"; } std::ostream& operator<<(std::ostream& out, BC::RirTypecheck t) { diff --git a/rir/src/bc/BC.h b/rir/src/bc/BC.h index d24169f2e..1bb6e23f7 100644 --- a/rir/src/bc/BC.h +++ b/rir/src/bc/BC.h @@ -23,11 +23,23 @@ class CodeStream; BC_NOARGS(V, _) #undef V -BC BC::recordCall() { return BC(Opcode::record_call_); } +BC BC::recordCall(uint32_t idx) { + ImmediateArguments i; + i.i = idx; + return BC(Opcode::record_call_, i); +} -BC BC::recordType() { return BC(Opcode::record_type_); } +BC BC::recordType(uint32_t idx) { + ImmediateArguments i; + i.i = idx; + return BC(Opcode::record_type_, i); +} -BC BC::recordTest() { return BC(Opcode::record_test_); } +BC BC::recordTest(uint32_t idx) { + ImmediateArguments i; + i.i = idx; + return BC(Opcode::record_test_, i); +} BC BC::asSwitchIdx() { return BC(Opcode::as_switch_idx_); } diff --git a/rir/src/bc/BC_inc.h b/rir/src/bc/BC_inc.h index 20653494d..e975229a7 100644 --- a/rir/src/bc/BC_inc.h +++ b/rir/src/bc/BC_inc.h @@ -7,6 +7,11 @@ #include "compiler/pir/type.h" #include "runtime/Context.h" #include "runtime/TypeFeedback.h" +#include "runtime/log/printPrettyGraph.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/hash/hashRootOld.h" +#include "serializeHash/serializeUni.h" +#include "utils/ByteBuffer.h" #include #include @@ -59,6 +64,16 @@ enum class Opcode : uint8_t { num_of }; +struct ExtraPoolEntryRefInSrc { + enum Type : unsigned { + Promise, + ArbitrarySexp + }; + + unsigned idx; + Type type; +}; + // ============================================================ // ==== Creation and decoding of Bytecodes // @@ -152,9 +167,6 @@ class BC { uint32_t i; RirTypecheck typecheck; NumLocals loc; - ObservedCallees callFeedback; - ObservedValues typeFeedback; - ObservedTest testFeedback; PoolAndCachePositionRange poolAndCache; CachePositionRange cacheIdx; ImmediateArguments() { @@ -217,13 +229,32 @@ class BC { // Used to serialize bc to CodeStream void write(CodeStream& cs) const; - static void deserialize(SEXP refTable, R_inpstream_t inp, Opcode* code, + static void deserialize(AbstractDeserializer& deserializer, Opcode* code, size_t codeSize, Code* container); - static void serialize(SEXP refTable, R_outpstream_t out, const Opcode* code, - size_t codeSize, const Code* container); + static void serialize(AbstractSerializer& serializer, + std::vector& extraPoolFlags, + const Opcode* code, size_t codeSize, + const Code* container); + static void hash(HasherOld& hasher, std::vector& extraPoolIgnored, + const Opcode* code, size_t codeSize, + const Code* container); + static void addConnected(std::vector& extraPoolChildren, + ConnectedCollectorOld& collector, const Opcode* code, + size_t codeSize, const Code* container); + static void addToPrettyGraph(const PrettyGraphInnerPrinter& p, + std::vector& addedExtraPoolEntries, + const Opcode* code, size_t codeSize, + const Code* container); + /// Compare bytecodes and print differences. + static void debugCompare(const Opcode* code1, const Opcode* code2, + size_t codeSize1, size_t codeSize2, + const Code* container1, const Code* container2, + const char* prefix, + std::stringstream& differences); // Print it to the stream passed as argument void print(std::ostream& out) const; + void printAssociatedData(std::ostream& out, bool printDetailed = false) const; void printImmediateArgs(std::ostream& out) const; void printNames(std::ostream& out, const std::vector&) const; void printProfile(std::ostream& out) const; @@ -267,6 +298,11 @@ class BC { bool isJmp() const { return isCondJmp() || isUncondJmp(); } + bool isRecord() const { + return bc == Opcode::record_call_ || bc == Opcode::record_test_ || + bc == Opcode::record_type_; + } + bool isExit() const { return bc == Opcode::ret_ || bc == Opcode::return_; } // This code performs the same as `BC::decode(pc).size()`, but for @@ -309,10 +345,10 @@ class BC { #define V(NESTED, name, name_) inline static BC name(); BC_NOARGS(V, _) #undef V - inline static BC recordCall(); + inline static BC recordCall(uint32_t idx); inline static BC recordBinop(); - inline static BC recordType(); - inline static BC recordTest(); + inline static BC recordType(uint32_t idx); + inline static BC recordTest(uint32_t idx); inline static BC asSwitchIdx(); inline static BC popn(unsigned n); inline static BC push(SEXP constant); @@ -406,14 +442,6 @@ class BC { extraInformation.get()); } - CallFeedbackExtraInformation& callFeedbackExtra() const { - assert(bc == Opcode::record_call_ && "not a record call instruction"); - assert(extraInformation.get() && - "missing extra information. created through decodeShallow?"); - return *static_cast( - extraInformation.get()); - } - private: void allocExtraInformation() { assert(extraInformation == nullptr); @@ -428,10 +456,6 @@ class BC { extraInformation.reset(new CallInstructionExtraInformation); break; } - case Opcode::record_call_: { - extraInformation.reset(new CallFeedbackExtraInformation); - break; - } default: { } } @@ -455,13 +479,6 @@ class BC { break; } - case Opcode::record_call_: { - // Read call target feedback from the extra pool - for (size_t i = 0; i < immediate.callFeedback.numTargets; ++i) - callFeedbackExtra().targets.push_back( - immediate.callFeedback.getTarget(code, i)); - break; - } default: { } } @@ -580,18 +597,10 @@ class BC { case Opcode::pull_: case Opcode::is_: case Opcode::put_: - memcpy(&immediate.i, pc, sizeof(immediate.i)); - break; case Opcode::record_call_: - memcpy(&immediate.callFeedback, pc, sizeof(ObservedCallees)); - break; case Opcode::record_test_: - memcpy(reinterpret_cast(&immediate.testFeedback), pc, - sizeof(ObservedValues)); - break; case Opcode::record_type_: - memcpy(reinterpret_cast(&immediate.typeFeedback), pc, - sizeof(ObservedValues)); + memcpy(&immediate.i, pc, sizeof(immediate.i)); break; #define V(NESTED, name, name_) case Opcode::name_##_: BC_NOARGS(V, _) diff --git a/rir/src/bc/CodeVerifier.cpp b/rir/src/bc/CodeVerifier.cpp index 102068d82..93b2b3aa4 100644 --- a/rir/src/bc/CodeVerifier.cpp +++ b/rir/src/bc/CodeVerifier.cpp @@ -174,7 +174,7 @@ void CodeVerifier::verifyFunctionLayout(SEXP sexp) { if (f->defaultArg(i)) objs.push_back(f->defaultArg(i)); - if (f->size > XLENGTH(sexp)) + if (f->size > RAW_LENGTH(sexp)) Rf_error("RIR Verifier: Reported size must be smaller than the size of " "the vector"); diff --git a/rir/src/bc/Compiler.cpp b/rir/src/bc/Compiler.cpp index 2ad133d76..8efe1447e 100644 --- a/rir/src/bc/Compiler.cpp +++ b/rir/src/bc/Compiler.cpp @@ -3,6 +3,7 @@ #include "R/RList.h" #include "R/Symbols.h" #include "R/r.h" +#include "Rinternals.h" #include "bc/BC.h" #include "bc/CodeStream.h" #include "bc/CodeVerifier.h" @@ -10,6 +11,7 @@ #include "interpreter/interp.h" #include "interpreter/interp_incl.h" #include "interpreter/safe_force.h" +#include "runtime/TypeFeedback.h" #include "simple_instruction_list.h" #include "utils/Pool.h" @@ -136,6 +138,7 @@ class CompilerContext { FunctionWriter& fun; Preserve& preserve; + TypeFeedback::Builder typeFeedbackBuilder; CompilerContext(FunctionWriter& fun, Preserve& preserve) : fun(fun), preserve(preserve) {} @@ -201,6 +204,12 @@ class CompilerContext { << BC::callBuiltin(4, ast, getBuiltinFun("warning")) << BC::pop(); } + BC recordType() { return BC::recordType(typeFeedbackBuilder.addType()); } + + BC recordCall() { return BC::recordCall(typeFeedbackBuilder.addCallee()); } + + BC recordTest() { return BC::recordTest(typeFeedbackBuilder.addTest()); } + private: unsigned int pushedPromiseContexts = 0; }; @@ -257,7 +266,7 @@ void compileWhile(CompilerContext& ctx, std::function compileCond, // loop peel is a copy of the condition and body, with no backwards jumps if (Compiler::loopPeelingEnabled && peelLoop) { compileCond(); - cs << BC::recordTest() << BC::brfalse(breakBranch); + cs << ctx.recordTest() << BC::brfalse(breakBranch); compileBody(); } @@ -348,7 +357,7 @@ bool compileSimpleFor(CompilerContext& ctx, SEXP fullAst, SEXP sym, SEXP seq, // branch cs << BC::pop(); } else { - cs << BC::recordTest() << BC::brtrue(skipRegularForBranch); + cs << ctx.recordTest() << BC::brtrue(skipRegularForBranch); // // Note that we call the builtin `for` and pass the body as a // promise to lower the bytecode size @@ -378,7 +387,7 @@ bool compileSimpleFor(CompilerContext& ctx, SEXP fullAst, SEXP sym, SEXP seq, if (voidContext) cs << BC::pop(); else if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); cs << BC::br(endBranch); cs << skipRegularForBranch; @@ -386,16 +395,16 @@ bool compileSimpleFor(CompilerContext& ctx, SEXP fullAst, SEXP sym, SEXP seq, // } else { // m' <- colonCastLhs(m') - cs << BC::swap() << BC::colonCastLhs() << BC::recordType() + cs << BC::swap() << BC::colonCastLhs() << ctx.recordType() << BC::ensureNamed() << BC::swap(); // n' <- colonCastRhs(m', n') - cs << BC::colonCastRhs() << BC::ensureNamed() << BC::recordType(); + cs << BC::colonCastRhs() << BC::ensureNamed() << ctx.recordType(); // step <- if (m' <= n') 1L else -1L cs << BC::dup2() << BC::le(); cs.addSrc(R_NilValue); - cs << BC::recordTest() << BC::brfalse(stepElseBranch) << BC::push(1) + cs << ctx.recordTest() << BC::brfalse(stepElseBranch) << BC::push(1) << BC::br(stepEndBranch) << stepElseBranch << BC::push(-1) << stepEndBranch; @@ -486,7 +495,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, Protect p(dt); // Mark this as an inner function to prevent the optimizer from // assuming a stable environment - DispatchTable::check(dt)->baseline()->flags.set( + DispatchTable::check(dt)->baseline()->setFlag( Function::InnerFunction); assert(TYPEOF(dt) == EXTERNALSXP); cs << BC::push(args[0]) << BC::push(dt) << BC::push(args[2]) @@ -539,7 +548,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, if (voidContext) cs << BC::pop(); else if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); return true; } @@ -794,7 +803,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, } if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); if (maybeChanges(target, *idx) || (dims > 1 && maybeChanges(target, *(idx + 1))) || @@ -943,7 +952,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs << BC::ldfun(farrow_sym); if (Compiler::profile) - cs << BC::recordCall(); + cs << ctx.recordCall(); // prepare x, yk, z as promises LoadArgsResult load_arg_res; @@ -1017,7 +1026,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, // The return value, RHS, is TOS cs << BC::invisible(); if (Compiler::profile) { - cs << BC::recordType(); + cs << ctx.recordType(); } } @@ -1157,7 +1166,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, BC::Label contBranch = cs.mkLabel(); cs << BC::dup() << BC::is(BC::RirTypecheck::isNonObject) - << BC::recordTest() << BC::brfalse(objBranch) + << ctx.recordTest() << BC::brfalse(objBranch) << BC::br(nonObjBranch); cs << objBranch; @@ -1198,7 +1207,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs.addSrc(ast); if (!voidContext) { if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); cs << BC::visible(); } else { cs << BC::pop(); @@ -1307,7 +1316,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs.addSrc(R_NilValue); if (record) - cs << BC::recordTest(); + cs << ctx.recordTest(); // If outside bound, branch, otherwise index into the vector cs << BC::brtrue(breakBranch) << BC::pull(2) << BC::pull(1) @@ -1493,14 +1502,14 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, // !isVector(x) cs << BC::dup() << BC::is(BC::RirTypecheck::isVector) - << BC::recordTest() << BC::brtrue(vecArityBr); + << ctx.recordTest() << BC::brtrue(vecArityBr); cs << BC::br(vecErrorBr); // ... || LENGTH(x) != 1 cs << vecArityBr << BC::dup() << BC::length_() << BC::push(1) << BC::eq(); cs.addSrc(R_NilValue); // to make code verifier happy - cs << BC::recordTest() << BC::brtrue(vecEContBr); + cs << ctx.recordTest() << BC::brtrue(vecEContBr); cs << vecErrorBr; ctx.emitError("EXPR must be a length 1 vector", ast); @@ -1509,7 +1518,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs << vecEContBr; cs << BC::dup() << BC::is(BC::RirTypecheck::isFactor) - << BC::recordTest() << BC::brfalse(facWContBr); + << ctx.recordTest() << BC::brfalse(facWContBr); ctx.emitWarning("EXPR is a \"factor\", treated as integer.\n Consider " "using 'switch(as.character( * ), ...)' instead.", @@ -1522,14 +1531,14 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs << BC::br(nilBr); } cs << BC::dup() << BC::is(BC::RirTypecheck::isSTRSXP) - << BC::recordTest() << BC::brtrue(strBr); + << ctx.recordTest() << BC::brtrue(strBr); cs << BC::asSwitchIdx(); // currently stack is [arg[0]] (converted to integer) for (size_t i = 0; i < labels.size(); ++i) { cs << BC::dup() << BC::push(Rf_ScalarInteger(i + 1)) << BC::eq(); cs.addSrc(R_NilValue); // call argument for builtin - cs << BC::asbool() << BC::recordTest() << BC::brtrue(labels[i]); + cs << BC::asbool() << ctx.recordTest() << BC::brtrue(labels[i]); } cs << BC::br(nilBr) << strBr; if (dupDflt) { @@ -1543,14 +1552,14 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, // BC::asbool to compare the cases. cs << BC::dup() << BC::callBuiltin(1, R_NilValue, getBuiltinFun("is.na")) - << BC::asbool() << BC::recordTest() << BC::brfalse(strNAContBr) + << BC::asbool() << ctx.recordTest() << BC::brfalse(strNAContBr) << BC::pop() << BC::push(Rf_mkString("NA")) << strNAContBr; for (size_t i = 0; i < expressions.size(); ++i) { for (auto& n : groups[i]) { cs << BC::dup() << BC::push(n) << BC::eq(); cs.addSrc(R_NilValue); // call argument for builtin - cs << BC::asbool() << BC::recordTest() + cs << BC::asbool() << ctx.recordTest() << BC::brtrue(groupLabels[i]); } } @@ -1858,7 +1867,7 @@ void compileCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args, theEnd = cs.mkLabel(); cs << BC::push(builtin) << BC::dup() << BC::ldvarNoForce(fun) << BC::identicalNoforce() - << BC::recordTest() << BC::brtrue(eager); + << ctx.recordTest() << BC::brtrue(eager); cs << BC::pop(); } @@ -1872,7 +1881,7 @@ void compileCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args, } if (Compiler::profile) - cs << BC::recordCall(); + cs << ctx.recordCall(); auto compileCall = [&](LoadArgsResult& info) { if (info.hasDots) { @@ -1892,7 +1901,7 @@ void compileCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args, compileLoadOneArg(ctx, args, ArgType::RAW_VALUE, info); compileLoadOneArg(ctx, CDR(args), ArgType::RAW_VALUE, info); if (Compiler::profile) - cs << BC::recordCall(); + cs << ctx.recordCall(); // Load the rest of the args compileLoadArgs(ctx, ast, fun, args, info, voidContext, 2, 0); } else { @@ -1915,7 +1924,7 @@ void compileCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args, if (voidContext) cs << BC::pop(); else if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); } // Lookup @@ -1933,7 +1942,7 @@ void compileGetvar(CompilerContext& ctx, SEXP name) { cs << BC::ldvar(name); } if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); } } @@ -2052,7 +2061,10 @@ SEXP Compiler::finalize() { compileExpr(ctx, exp); ctx.cs() << BC::ret(); Code* body = ctx.pop(); - function.finalize(body, signature, Context()); + TypeFeedback* feedback = ctx.typeFeedbackBuilder.build(); + PROTECT(feedback->container()); + function.finalize(body, signature, Context(), feedback); + UNPROTECT(1); #ifdef ENABLE_SLOWASSERT CodeVerifier::verifyFunctionLayout(function.function()->container()); diff --git a/rir/src/bc/Compiler.h b/rir/src/bc/Compiler.h index 6032b856e..3c5bc72c3 100644 --- a/rir/src/bc/Compiler.h +++ b/rir/src/bc/Compiler.h @@ -4,7 +4,10 @@ #include "R/Preserve.h" #include "R/Protect.h" #include "R/r.h" +#include "compilerClientServer/CompilerClient.h" +#include "compilerClientServer/CompilerServer.h" #include "runtime/DispatchTable.h" +#include "runtime/TypeFeedback.h" #include "utils/FunctionWriter.h" #include "utils/Pool.h" @@ -93,6 +96,19 @@ class Compiler { // Set the closure fields. SET_BODY(inClosure, dt->container()); } + + /// Takes a closure with a RIR body and returns a copy with same formals and + /// environment, but decompiled (AST) body + static SEXP decompileClosure(SEXP closure) { + assert(TYPEOF(closure) == CLOSXP && "not a closure"); + auto dt = DispatchTable::check(BODY(closure)); + assert(dt && "closure's body isn't a RIR dispatch table"); + auto result = Rf_allocSExp(CLOSXP); + SET_FORMALS(result, FORMALS(closure)); + SET_BODY(result, rirDecompile(BODY(closure))); + SET_CLOENV(result, CLOENV(closure)); + return result; + } }; } // namespace rir diff --git a/rir/src/bc/insns.h b/rir/src/bc/insns.h index dcdf864c0..a9bcc91ee 100644 --- a/rir/src/bc/insns.h +++ b/rir/src/bc/insns.h @@ -443,7 +443,7 @@ DEF_INSTR(ret_, 0, 1, 0) * They keep a struct from RuntimeFeedback.h inline, that's why they are quite * heavy in size. */ -DEF_INSTR(record_call_, 4, 1, 1) +DEF_INSTR(record_call_, 1, 1, 1) DEF_INSTR(record_type_, 1, 1, 1) DEF_INSTR(record_test_, 1, 1, 1) diff --git a/rir/src/common.h b/rir/src/common.h index 51cd01b98..884b13c90 100644 --- a/rir/src/common.h +++ b/rir/src/common.h @@ -15,6 +15,9 @@ extern void printBacktrace(); {} #endif +#define REXPORT extern "C" +#define ALWAYS_INLINE __attribute__((always_inline)) inline + // from boost #include template diff --git a/rir/src/compiler/analysis/reference_count.h b/rir/src/compiler/analysis/reference_count.h index 111ed9847..af1c465aa 100644 --- a/rir/src/compiler/analysis/reference_count.h +++ b/rir/src/compiler/analysis/reference_count.h @@ -5,6 +5,7 @@ #include "dead.h" #include "generic_static_analysis.h" #include "utils/Map.h" +#include namespace rir { namespace pir { diff --git a/rir/src/compiler/analysis/verifier.cpp b/rir/src/compiler/analysis/verifier.cpp index 4cedab71d..4002054fe 100644 --- a/rir/src/compiler/analysis/verifier.cpp +++ b/rir/src/compiler/analysis/verifier.cpp @@ -284,7 +284,7 @@ class TheVerifier { } if (auto assume = Assume::Cast(i)) { if (IsType::Cast(assume->arg(0).val())) { - if (!assume->reason.pc()) { + if (!assume->reason.origin.hasSlot()) { std::cerr << "Error: instruction '"; i->print(std::cerr); std::cerr << "' typecheck without origin information\n"; diff --git a/rir/src/compiler/backend.cpp b/rir/src/compiler/backend.cpp index e85d012bd..3b11d216a 100644 --- a/rir/src/compiler/backend.cpp +++ b/rir/src/compiler/backend.cpp @@ -16,6 +16,7 @@ #include "compiler/util/visitor.h" #include "interpreter/instance.h" #include "runtime/DispatchTable.h" +#include "runtime/TypeFeedback.h" #include "simple_instruction_list.h" #include "utils/FunctionWriter.h" #include "utils/measuring.h" @@ -404,7 +405,9 @@ rir::Function* Backend::doCompile(ClosureVersion* cls, ClosureLog& log) { } log.finalPIR(); - function.finalize(body, signature, cls->context()); + // the type feedback is only used at the baseline + function.finalize(body, signature, cls->context(), + rir::TypeFeedback::empty()); for (auto& c : done) c.second->function(function.function()); diff --git a/rir/src/compiler/backend.h b/rir/src/compiler/backend.h index e1df44dbe..1d6dc6a7e 100644 --- a/rir/src/compiler/backend.h +++ b/rir/src/compiler/backend.h @@ -16,8 +16,9 @@ namespace pir { class Backend { public: - Backend(Module* m, Log& logger, const std::string& name) - : module(m), jit(name), logger(logger) {} + Backend(Module* m, Log& logger, const std::string& name, + const SerialOptions& serialOpts) + : module(m), jit(name, serialOpts), logger(logger) {} ~Backend() { jit.finalize(); } Backend(const Backend&) = delete; Backend& operator=(const Backend&) = delete; diff --git a/rir/src/compiler/compiler.cpp b/rir/src/compiler/compiler.cpp index b4c367e28..5abbf18e1 100644 --- a/rir/src/compiler/compiler.cpp +++ b/rir/src/compiler/compiler.cpp @@ -3,10 +3,10 @@ #include "pir/continuation.h" #include "pir/pir_impl.h" #include "rir2pir/rir2pir.h" -#include "utils/Map.h" +#include "runtime/ProxyEnv.h" +#include "runtime/TypeFeedback.h" #include "utils/measuring.h" -#include "compiler/analysis/query.h" #include "compiler/analysis/verifier.h" #include "compiler/opt/pass_definitions.h" #include "compiler/opt/pass_scheduler.h" @@ -41,10 +41,10 @@ void Compiler::compileClosure(SEXP closure, const std::string& name, fun->clearDisabledAssumptions(assumptions); assumptions = tbl->combineContextWith(assumptions); - auto frame = RList(FRAME(CLOENV(closure))); std::string closureName = name; - if (name.compare("") == 0) { + if (name.empty() && !ProxyEnv::check(CLOENV(closure))) { + auto frame = RList(FRAME(CLOENV(closure))); // Serach for name in environment for (auto e = frame.begin(); e != frame.end(); ++e) { if (*e == closure) @@ -55,7 +55,8 @@ void Compiler::compileClosure(SEXP closure, const std::string& name, tbl->userDefinedContext()); Context context(assumptions); compileClosure(pirClosure, tbl->dispatch(assumptions), context, root, - success, fail, outerFeedback); + success, fail, outerFeedback, + tbl->baseline()->typeFeedback()); } void Compiler::compileFunction(rir::DispatchTable* src, const std::string& name, @@ -71,7 +72,7 @@ void Compiler::compileFunction(rir::DispatchTable* src, const std::string& name, auto closure = module->getOrDeclareRirFunction( name, srcFunction, formals, srcRef, src->userDefinedContext()); compileClosure(closure, src->dispatch(assumptions), context, false, success, - fail, outerFeedback); + fail, outerFeedback, src->baseline()->typeFeedback()); } void Compiler::compileContinuation(SEXP closure, rir::Function* curFun, @@ -89,7 +90,8 @@ void Compiler::compileContinuation(SEXP closure, rir::Function* curFun, Builder builder(version, pirClosure->closureEnv()); auto& log = logger.open(version); - Rir2Pir rir2pir(*this, version, log, pirClosure->name(), {}); + auto typeFeedback = tbl->baseline()->typeFeedback(); + Rir2Pir rir2pir(*this, version, log, pirClosure->name(), {}, typeFeedback); if (rir2pir.tryCompileContinuation(builder, ctx->pc(), ctx->stack())) { log.flush(); @@ -105,7 +107,8 @@ void Compiler::compileContinuation(SEXP closure, rir::Function* curFun, void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, const Context& ctx, bool root, MaybeCls success, Maybe fail, - std::list outerFeedback) { + std::list outerFeedback, + rir::TypeFeedback* typeFeedback) { if (!ctx.includes(minimalContext)) { for (const auto a : minimalContext) { @@ -119,10 +122,11 @@ void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, } // Currently dots args are not supported in PIR. Unless if we statically - // matched all arguments correctly and are therefore guaranteed to receive a + // matched all arguments correctly and are therefore guaranteed to + // receive a // `...` list as DOTSXP in the correct location, we can support them. - // TODO: extend call instruction to do the necessary argument shuffling to - // support it in all cases + // TODO: extend call instruction to do the necessary argument shuffling + // to support it in all cases if (!ctx.includes(Assumption::StaticallyArgmatched) && closure->formals().hasDots()) { logger.warn("no support for ..."); @@ -130,7 +134,7 @@ void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, } if (closure->rirFunction()->body()->codeSize > Parameter::MAX_INPUT_SIZE) { - closure->rirFunction()->flags.set(Function::NotOptimizable); + closure->rirFunction()->setFlag(Function::NotOptimizable); logger.warn("skipping huge function"); return fail(); } @@ -141,7 +145,8 @@ void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, auto version = closure->declareVersion(ctx, root, optFunction); Builder builder(version, closure->closureEnv()); auto& log = logger.open(version); - Rir2Pir rir2pir(*this, version, log, closure->name(), outerFeedback); + Rir2Pir rir2pir(*this, version, log, closure->name(), outerFeedback, + typeFeedback); auto& context = version->context(); bool failedToCompileDefaultArgs = false; @@ -175,8 +180,8 @@ void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, for (unsigned i = 0; i < closure->nargs() - context.numMissing(); ++i) { if (closure->formals().defaultArgs()[i] != R_MissingArg) { - // If this arg has a default, then test if the argument is - // missing and if so, load the default arg. + // If this arg has a default, then test if the argument + // is missing and if so, load the default arg. auto a = builder(new LdArg(i)); auto testMissing = builder(new Identical( a, MissingArg::instance(), PirType::any())); @@ -270,7 +275,8 @@ static void findUnreachable(Module* m, Log& log, const std::string& where) { if (!call->tryDispatch()) { std::stringstream msg; msg << "After pass " << where - << " found a broken static call. Available " + << " found a broken static call. " + "Available " "versions:\n"; call->cls()->eachVersion( [&](ClosureVersion* v) { diff --git a/rir/src/compiler/compiler.h b/rir/src/compiler/compiler.h index 628d4abc2..30f700aad 100644 --- a/rir/src/compiler/compiler.h +++ b/rir/src/compiler/compiler.h @@ -4,6 +4,7 @@ #include "R/Preserve.h" #include "compiler/log/log.h" #include "pir/pir.h" +#include "runtime/TypeFeedback.h" #include "utils/FormalArgs.h" #include @@ -57,7 +58,8 @@ class Compiler { void compileClosure(Closure* closure, rir::Function* optFunction, const Context& ctx, bool root, MaybeCls success, - Maybe fail, std::list outerFeedback); + Maybe fail, std::list outerFeedback, + rir::TypeFeedback* typeFeedback); Preserve preserve_; diff --git a/rir/src/compiler/log/debug.h b/rir/src/compiler/log/debug.h index ae5c8c0a1..79188d57f 100644 --- a/rir/src/compiler/log/debug.h +++ b/rir/src/compiler/log/debug.h @@ -4,6 +4,7 @@ #include "utils/EnumSet.h" #include +#include namespace rir { namespace pir { @@ -57,12 +58,14 @@ enum class DebugStyle { struct DebugOptions { typedef EnumSet DebugFlags; DebugFlags flags; - const std::regex passFilter; - const std::regex functionFilter; + std::regex passFilter; + std::string passFilterString; + std::regex functionFilter; + std::string functionFilterString; DebugStyle style; DebugOptions operator|(const DebugFlags& f) const { - return {flags | f, passFilter, functionFilter, style}; + return {flags | f, passFilter, passFilterString, functionFilter, functionFilterString, style}; } bool includes(const DebugFlags& otherFlags) const { return flags.includes(otherFlags); @@ -74,20 +77,62 @@ struct DebugOptions { return flags.intersects(otherFlags); } - explicit DebugOptions(unsigned long long flags) - : flags(flags), passFilter(".*"), functionFilter(".*"), - style(DebugStyle::Standard) {} - DebugOptions(const DebugFlags& flags, const std::regex& filter, - const std::regex& functionFilter, DebugStyle style) - : flags(flags), passFilter(filter), functionFilter(functionFilter), - style(style) {} - DebugOptions() {} + explicit DebugOptions(int flags) + : DebugOptions(DebugFlags(flags)) {} + explicit DebugOptions(DebugFlags flags) + : DebugOptions(flags, ".*", ".*", + DebugStyle::Standard) {} + DebugOptions(const DebugFlags& flags, const std::string& passFilter, + const std::string& functionFilter, DebugStyle style) + : flags(flags), passFilter(std::regex(passFilter)), + passFilterString(passFilter), functionFilter(functionFilter), + functionFilterString(functionFilter), style(style) {} + DebugOptions(const DebugFlags& flags, std::regex passFilter, + std::string passFilterString, + std::regex functionFilter, + std::string functionFilterString, + DebugStyle style) + : flags(flags), passFilter(std::move(passFilter)), + passFilterString(std::move(passFilterString)), + functionFilter(std::move(functionFilter)), + functionFilterString(std::move(functionFilterString)), style(style) {} + DebugOptions() : DebugOptions(0) {} bool multipleFiles() const { return includes(DebugFlag::PrintPassesIntoFolders) || style != DebugStyle::Standard; } + friend std::ostream& operator<<(std::ostream& out, const DebugOptions& o) { + out << "DebugOptions("; + bool first = true; +#define V(n) \ + if (o.includes(DebugFlag::n)) { \ + if (!first) out << ", "; \ + out << #n; \ + first = false; \ + } + LIST_OF_PIR_DEBUGGING_FLAGS(V) +#undef V + if (o.passFilterString != ".*") { + if (!first) out << ", "; + out << "passFilter=" << o.passFilterString; + first = false; + } + if (o.functionFilterString != ".*") { + if (!first) out << ", "; + out << "functionFilter=" << o.functionFilterString; + first = false; + } + if (o.style != DebugStyle::Standard) { + if (!first) out << ", "; + out << "style=" << (int)o.style; + // first = false; + } + out << ")"; + return out; + } + static DebugOptions DefaultDebugOptions; }; diff --git a/rir/src/compiler/log/loggers.h b/rir/src/compiler/log/loggers.h index 1d70fa22f..9579397a2 100644 --- a/rir/src/compiler/log/loggers.h +++ b/rir/src/compiler/log/loggers.h @@ -26,7 +26,7 @@ class AbstractLog { const DebugOptions options; const ClosureVersion* version; - AbstractLog(DebugOptions options, const ClosureVersion* version, + AbstractLog(const DebugOptions& options, const ClosureVersion* version, std::shared_ptr out) : _out(out), options(options), version(version) {} diff --git a/rir/src/compiler/native/allocator.cpp b/rir/src/compiler/native/allocator.cpp index d63e89987..154cbbdf5 100644 --- a/rir/src/compiler/native/allocator.cpp +++ b/rir/src/compiler/native/allocator.cpp @@ -22,8 +22,8 @@ void NativeAllocator::compute() { // them accessible to the runtime profiler. // TODO: this needs to be replaced by proper mapping of slots. if (RuntimeProfiler::enabled() && a != b && - (a->typeFeedback().feedbackOrigin.pc() || - b->typeFeedback().feedbackOrigin.pc())) + (a->typeFeedback().feedbackOrigin.hasSlot() || + b->typeFeedback().feedbackOrigin.hasSlot())) return true; return livenessIntervals.interfere(a, b); }; @@ -102,10 +102,7 @@ void NativeAllocator::compute() { } }; - size_t pos = 0; for (auto i : *bb) { - ++pos; - if (!needsASlot(i)) continue; diff --git a/rir/src/compiler/native/builtins.cpp b/rir/src/compiler/native/builtins.cpp index 934709157..e5dce3a14 100644 --- a/rir/src/compiler/native/builtins.cpp +++ b/rir/src/compiler/native/builtins.cpp @@ -767,13 +767,13 @@ int asSwitchIdxImpl(SEXP val) { int checkTrueFalseImpl(SEXP val) { int cond = NA_LOGICAL; - if (XLENGTH(val) > 1) + if (RAW_LENGTH(val) > 1) Rf_warningcall( // TODO: pass srcid R_NilValue, "the condition has length > 1 and only the first " "element will be used"); - if (XLENGTH(val) > 0) { + if (RAW_LENGTH(val) > 0) { switch (TYPEOF(val)) { case LGLSXP: cond = LOGICAL(val)[0]; @@ -788,7 +788,7 @@ int checkTrueFalseImpl(SEXP val) { if (cond == NA_LOGICAL) { const char* msg = - XLENGTH(val) ? (Rf_isLogical(val) + RAW_LENGTH(val) ? (Rf_isLogical(val) ? ("missing value where TRUE/FALSE needed") : ("argument is not interpretable as logical")) : ("argument is of length zero"); @@ -821,8 +821,9 @@ static SEXP deoptSentinelContainer = []() { PROTECT(c->container()); SEXP store = Rf_allocVector(EXTERNALSXP, sizeof(Function)); R_PreserveObject(store); - deoptSentinel = new (INTEGER(store)) - Function(0, c->container(), {}, deoptSentinelSig, Context()); + deoptSentinel = + new (INTEGER(store)) Function(0, c->container(), {}, deoptSentinelSig, + Context(), rir::TypeFeedback::empty()); deoptSentinel->registerDeopt(); UNPROTECT(1); return store; @@ -954,44 +955,37 @@ void deoptImpl(rir::Code* c, SEXP cls, DeoptMetadata* m, R_bcstack_t* args, assert(false); } -void recordTypefeedbackImpl(Opcode* pos, rir::Code* code, SEXP value) { - switch (*pos) { - case Opcode::record_test_: { - ObservedTest* feedback = (ObservedTest*)(pos + 1); - feedback->record(value); - break; - } - case Opcode::record_type_: { - assert(*pos == Opcode::record_type_); - ObservedValues* feedback = (ObservedValues*)(pos + 1); - feedback->record(value); - if (TYPEOF(value) == PROMSXP) { - if (PRVALUE(value) == R_UnboundValue && - feedback->stateBeforeLastForce < ObservedValues::promise) - feedback->stateBeforeLastForce = ObservedValues::promise; - else if (feedback->stateBeforeLastForce < - ObservedValues::evaluatedPromise) - feedback->stateBeforeLastForce = - ObservedValues::evaluatedPromise; - } else { - if (feedback->stateBeforeLastForce < ObservedValues::value) - feedback->stateBeforeLastForce = ObservedValues::value; +void recordTypeFeedbackImpl(rir::TypeFeedback* feedback, uint32_t idx, + SEXP value) { + auto& slot = feedback->types(idx); + slot.record(value); + + if (TYPEOF(value) == PROMSXP) { + if (PRVALUE(value) == R_UnboundValue && + slot.stateBeforeLastForce < ObservedValues::promise) { + slot.stateBeforeLastForce = ObservedValues::promise; + } else if (slot.stateBeforeLastForce < + ObservedValues::evaluatedPromise) { + slot.stateBeforeLastForce = ObservedValues::evaluatedPromise; } - break; - } - case Opcode::record_call_: { - ObservedCallees* feedback = (ObservedCallees*)(pos + 1); - feedback->record(code, value); - break; - } - default: - assert(false); + } else { + if (slot.stateBeforeLastForce < ObservedValues::value) + slot.stateBeforeLastForce = ObservedValues::value; } } +void recordCallFeedbackImpl(rir::TypeFeedback* feedback, uint32_t idx, + SEXP value) { + feedback->callees(idx).record(feedback->owner(), value); +} + void assertFailImpl(const char* msg) { std::cout << "Assertion in jitted code failed: '" << msg << "'\n"; +#ifdef __ARM_ARCH + __builtin_debugtrap(); +#else asm("int3"); +#endif } void printValueImpl(SEXP v) { Rf_PrintValue(v); } @@ -1312,6 +1306,10 @@ static SEXP nativeCallTrampolineImpl(ArglistOrder::CallId callId, rir::Code* c, SLOWASSERT(env == symbol::delayedEnv || TYPEOF(env) == ENVSXP || env == R_NilValue || LazyEnvironment::check(env)); + if (!INTERPRETER_IS_ACTIVE) { + std::cerr << "TODO: Interpreting code during serialization or comparison\n"; + } + auto fun = Function::unpack(Pool::get(target)); CallContext call(callId, c, callee, nargs, astP, @@ -1450,8 +1448,8 @@ static SEXP nativeCallTrampolineImpl(ArglistOrder::CallId callId, rir::Code* c, RCNTXT cntxt; - // This code needs to be protected, because its slot in the dispatch table - // could get overwritten while we are executing it. + // This code needs to be protected, because its slot in the dispatch + // table could get overwritten while we are executing it. PROTECT(fun->container()); initClosureContext(ast, &cntxt, symbol::delayedEnv, env, lazyArgs.asSexp(), @@ -2423,10 +2421,17 @@ void NativeBuiltins::initializeBuiltins() { (void*)&lengthImpl, llvm::FunctionType::get(t::Int, {t::SEXP}, false), {}}; - get_(Id::recordTypefeedback) = { - "recordTypefeedback", - (void*)&recordTypefeedbackImpl, - llvm::FunctionType::get(t::t_void, {t::i64, t::i64, t::SEXP}, false), + get_(Id::recordTypeFeedback) = { + "recordTypeFeedback", + (void*)&recordTypeFeedbackImpl, + llvm::FunctionType::get(t::t_void, {t::voidPtr, t::i32, t::SEXP}, + false), + {}}; + get_(Id::recordCallFeedback) = { + "recordCallFeedback", + (void*)&recordCallFeedbackImpl, + llvm::FunctionType::get(t::t_void, {t::voidPtr, t::i32, t::SEXP}, + false), {}}; get_(Id::deopt) = {"deopt", (void*)&deoptImpl, diff --git a/rir/src/compiler/native/builtins.h b/rir/src/compiler/native/builtins.h index dfaf0dadc..1becc9ed5 100644 --- a/rir/src/compiler/native/builtins.h +++ b/rir/src/compiler/native/builtins.h @@ -85,7 +85,8 @@ struct NativeBuiltins { checkTrueFalse, asLogicalBlt, length, - recordTypefeedback, + recordTypeFeedback, + recordCallFeedback, deopt, assertFail, printValue, diff --git a/rir/src/compiler/native/lower_function_llvm.cpp b/rir/src/compiler/native/lower_function_llvm.cpp index d97ee815f..4702fe470 100644 --- a/rir/src/compiler/native/lower_function_llvm.cpp +++ b/rir/src/compiler/native/lower_function_llvm.cpp @@ -25,10 +25,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -64,12 +62,20 @@ llvm::Value* LowerFunctionLLVM::PhiBuilder::operator()(size_t numInputs) { return phi_; } -llvm::Value* LowerFunctionLLVM::globalConst(llvm::Constant* init, - llvm::Type* ty) { +llvm::GlobalVariable* LowerFunctionLLVM::globalConst(llvm::Module& mod, + llvm::Constant* init, + llvm::Type* ty) { if (!ty) ty = init->getType(); - return new llvm::GlobalVariable(getModule(), ty, true, - llvm::GlobalValue::PrivateLinkage, init); + // ???: Should this be inserted with getOrInsertGlobal? + return new llvm::GlobalVariable(mod, ty, true, + llvm::GlobalValue::PrivateLinkage, + init); +} + +llvm::GlobalVariable* LowerFunctionLLVM::globalConst(llvm::Constant* init, + llvm::Type* ty) { + return globalConst(getModule(), init, ty); } llvm::FunctionCallee @@ -77,31 +83,152 @@ LowerFunctionLLVM::getBuiltin(const rir::pir::NativeBuiltin& b) { return getModule().getOrInsertFunction(b.name, b.llvmSignature); } -llvm::Value* LowerFunctionLLVM::convertToPointer(const void* what, +llvm::Value* LowerFunctionLLVM::convertToPointer(llvm::Module& mod, + const void* what, llvm::Type* ty, - bool constant) { + bool constant, + llvm::MDNode* reprMeta) { assert(what); - char name[21]; - sprintf(name, "ept_%lx", (uintptr_t)what); - return getModule().getOrInsertGlobal(name, ty, [&]() { - return new llvm::GlobalVariable( - getModule(), ty, constant, + char llvmName[21]; + sprintf(llvmName, "ept_%lx", (uintptr_t)what); + return mod.getOrInsertGlobal(llvmName, ty, [&]() { + auto var = new llvm::GlobalVariable( + mod, ty, constant, llvm::GlobalValue::LinkageTypes::AvailableExternallyLinkage, - nullptr, name, nullptr, + nullptr, llvmName, nullptr, llvm::GlobalValue::ThreadLocalMode::NotThreadLocal, 0, true); + if (reprMeta) { + var->setMetadata(SerialRepr::POINTER_METADATA_NAME, reprMeta); + } + return var; }); } +llvm::Value* LowerFunctionLLVM::convertToPointer(const void* what, + llvm::Type* ty, + const SerialRepr& repr, + bool constant) { + return convertToPointer(getModule(), what, ty, constant, + Parameter::SERIALIZE_LLVM + ? repr.metadata(getModule().getContext(), serialOpts) + : nullptr); +} + llvm::FunctionCallee -LowerFunctionLLVM::convertToFunction(const void* what, llvm::FunctionType* ty) { +LowerFunctionLLVM::convertToFunction(llvm::Module& mod, const void* what, + llvm::FunctionType* ty, int builtinId) { assert(what); - char name[21]; - sprintf(name, "efn_%lx", (uintptr_t)what); - return getModule().getOrInsertFunction(name, ty); + char llvmName[21]; + sprintf(llvmName, "efn_%lx", (uintptr_t)what); + auto llvmFn = mod.getOrInsertFunction(llvmName, ty); + if (Parameter::SERIALIZE_LLVM) { + mod.getOrInsertNamedMetadata(SerialRepr::FUNCTION_METADATA_NAME) + ->addOperand(SerialRepr::functionMetadata( + llvmFn.getCallee()->getContext(), llvmName, builtinId)); + } + return llvmFn; +} + +llvm::FunctionCallee +LowerFunctionLLVM::convertToFunction(const void* what, llvm::FunctionType* ty, + int builtinId) { + return convertToFunction(getModule(), what, ty, builtinId); +} + +llvm::Value* LowerFunctionLLVM::llvmSrcIdx(llvm::Module& mod, Immediate i, + const SerialOptions& serialOpts) { + char llvmName[13]; + sprintf(llvmName, "src_%08x", i); + return mod.getOrInsertGlobal(llvmName, t::i32, [&]() { + auto value = + new llvm::GlobalVariable(mod, t::i32, true, + llvm::GlobalValue::AvailableExternallyLinkage, + nullptr, llvmName, nullptr, + llvm::GlobalValue::ThreadLocalMode::NotThreadLocal, + 0, true); + if (Parameter::SERIALIZE_LLVM) { + value->setMetadata(SerialRepr::SRC_IDX_METADATA_NAME, + SerialRepr::srcIdxMetadata(mod.getContext(), i, + serialOpts)); + } + return value; + }); +} + +llvm::Value* LowerFunctionLLVM::llvmSrcIdx(Immediate i) { + if (Parameter::SERIALIZE_LLVM) { + // Assuming this gets optimized out. Otherwise we can use regular + // ConstantInt like before, but we need to find a way to effectively add + // metadata to each src-idx ConstantInt. + return builder.CreateLoad(llvmSrcIdx(getModule(), i, serialOpts)); + } else { + return c(i); + } +} + +llvm::Value* LowerFunctionLLVM::llvmPoolIdx(llvm::Module& mod, BC::PoolIdx i, + const SerialOptions& serialOpts) { + char llvmName[12]; + sprintf(llvmName, "cp_%08x", i); + return mod.getOrInsertGlobal(llvmName, t::i32, [&]() { + auto value = + new llvm::GlobalVariable(mod, t::i32, true, + llvm::GlobalValue::AvailableExternallyLinkage, + nullptr, llvmName, nullptr, + llvm::GlobalValue::ThreadLocalMode::NotThreadLocal, + 0, true); + if (Parameter::SERIALIZE_LLVM) { + value->setMetadata(SerialRepr::POOL_IDX_METADATA_NAME, + SerialRepr::poolIdxMetadata(mod.getContext(), i, + serialOpts)); + } + return value; + }); +} + +llvm::Value* LowerFunctionLLVM::llvmPoolIdx(BC::PoolIdx i) { + if (Parameter::SERIALIZE_LLVM) { + // Assuming this gets optimized out. Otherwise we can use regular + // ConstantInt like before, but we need to find a way to effectively add + // metadata to each pool-idx ConstantInt. + return builder.CreateLoad(llvmPoolIdx(getModule(), i, serialOpts)); + } else { + return c(i); + } +} + +llvm::Value* LowerFunctionLLVM::llvmNames(llvm::Module& mod, const std::vector& names) { + std::stringstream llvmNameStr; + llvmNameStr << "names"; + if (names.empty()) { + // Special case so that the empty vector name still starts with "names_" + llvmNameStr << "_"; + } else for (const auto& e : names) { + llvmNameStr << "_" << std::hex << e; + } + auto llvmName = llvmNameStr.str(); + auto ty = llvm::ArrayType::get(t::Int, names.size()); + return mod.getOrInsertGlobal(llvmName, ty, [&]() { + auto vectorStore = + new llvm::GlobalVariable(mod, ty, true, + llvm::GlobalValue::AvailableExternallyLinkage, + nullptr, llvmName, nullptr, + llvm::GlobalValue::ThreadLocalMode::NotThreadLocal, + 0, true); + if (Parameter::SERIALIZE_LLVM) { + vectorStore->setMetadata(SerialRepr::NAMES_METADATA_NAME, + SerialRepr::namesMetadata(mod.getContext(), names)); + } + return vectorStore; + }); +} + +llvm::Value* LowerFunctionLLVM::llvmNames(const std::vector& names) { + return builder.CreateBitCast(llvmNames(getModule(), names), t::IntPtr); } void LowerFunctionLLVM::setVisible(int i) { - builder.CreateStore(c(i), convertToPointer(&R_Visible, t::Int)); + builder.CreateStore(c(i), convertToPointer(&R_Visible, t::Int, SerialRepr::R_Visible{})); } llvm::Value* LowerFunctionLLVM::force(Instruction* i, llvm::Value* arg) { @@ -162,7 +289,7 @@ void LowerFunctionLLVM::insn_assert(llvm::Value* v, const char* msg, if (p) call(NativeBuiltins::get(NativeBuiltins::Id::printValue), {p}); call(NativeBuiltins::get(NativeBuiltins::Id::assertFail), - {convertToPointer((void*)msg, t::i8, true)}); + {convertToPointer((void*)msg, t::i8, SerialRepr::String{msg}, true)}); builder.CreateUnreachable(); builder.SetInsertPoint(ok); @@ -236,12 +363,9 @@ llvm::Value* LowerFunctionLLVM::constant(SEXP co, const Rep& needed) { eternalConst.count(co)) return convertToPointer(co, true); - auto i = Pool::insert(co); - llvm::Value* pos = builder.CreateLoad(constantpool); - pos = builder.CreateBitCast(dataPtr(pos, false), - PointerType::get(t::SEXP, 0)); - pos = builder.CreateGEP(pos, c(i)); - return builder.CreateLoad(pos); + // Could also Pool::insert or UUIDPool::intern + R_PreserveObject(co); + return convertToPointer(co); } llvm::Value* LowerFunctionLLVM::nodestackPtr() { @@ -296,7 +420,8 @@ void LowerFunctionLLVM::decStack(int i) { builder.CreateStore(up, nodestackPtrAddr); } -llvm::Value* LowerFunctionLLVM::callRBuiltin(SEXP builtin, +llvm::Value* LowerFunctionLLVM::callRBuiltin(int builtinId, + SEXP builtin, const std::vector& args, int srcIdx, CCODE builtinFun, llvm::Value* env) { @@ -305,7 +430,8 @@ llvm::Value* LowerFunctionLLVM::callRBuiltin(SEXP builtin, return call(NativeBuiltins::get(NativeBuiltins::Id::callBuiltin), { paramCode(), - c(srcIdx), + // Call ASTs in cp pool, not src pool + llvmPoolIdx(srcIdx), constant(builtin, t::SEXP), env, c(args.size()), @@ -313,7 +439,7 @@ llvm::Value* LowerFunctionLLVM::callRBuiltin(SEXP builtin, }); } - auto f = convertToFunction((void*)builtinFun, t::builtinFunction); + auto f = convertToFunction((void*)builtinFun, t::builtinFunction, builtinId); std::stack loadedArgs; auto n = numTemps; @@ -410,21 +536,16 @@ llvm::Value* LowerFunctionLLVM::load(Value* val, PirType type, Rep needed) { } else if (val == OpaqueTrue::instance()) { static int one = 1; // Something that is always true, but llvm does not know about - res = builder.CreateLoad(convertToPointer(&one, t::Int, true)); + res = builder.CreateLoad(convertToPointer(&one, t::Int, SerialRepr::OpaqueTrue{}, true)); } else if (auto ld = Const::Cast(val)) { res = constant(ld->c(), needed); } else if (val->tag == Tag::DeoptReason) { auto dr = (DeoptReasonWrapper*)val; - auto srcAddr = (Constant*)builder.CreateIntToPtr( - llvm::ConstantInt::get( - PirJitLLVM::getContext(), - llvm::APInt(64, - reinterpret_cast(dr->reason.srcCode()), - false)), - t::voidPtr); + auto srcAddr = llvm::cast( + convertToPointer(dr->reason.origin.function(), true)); auto drs = llvm::ConstantStruct::get( t::DeoptReason, {c(dr->reason.reason, 32), - c(dr->reason.origin.offset(), 32), srcAddr}); + c(dr->reason.origin.index().asInteger(), 32), srcAddr}); res = globalConst(drs); } else { val->printRef(std::cerr); @@ -669,7 +790,7 @@ void LowerFunctionLLVM::compilePushContext(Instruction* i) { { builder.SetInsertPoint(didLongjmp); llvm::Value* returned = builder.CreateLoad( - builder.CreateIntToPtr(c((void*)&R_ReturnedValue), t::SEXP_ptr)); + convertToPointer((const void*)&R_ReturnedValue, t::SEXP, SerialRepr::R_ReturnedValue{})); auto restart = builder.CreateICmpEQ(returned, constant(R_RestartToken, t::SEXP)); @@ -1577,7 +1698,7 @@ void LowerFunctionLLVM::compileRelop( if (i->hasEnv()) { auto e = loadSxp(i->env()); res = call(NativeBuiltins::get(NativeBuiltins::Id::binopEnv), - {a, b, e, c(i->srcIdx), c((uint8_t)i->tag, 8)}); + {a, b, e, llvmSrcIdx(i->srcIdx), c((uint8_t)i->tag, 8)}); } else { res = call(NativeBuiltins::get(NativeBuiltins::Id::binop), {a, b, c((uint8_t)i->tag, 8)}); @@ -1645,7 +1766,7 @@ void LowerFunctionLLVM::compileBinop( if (i->hasEnv()) { auto e = loadSxp(i->env()); res = call(NativeBuiltins::get(NativeBuiltins::Id::binopEnv), - {a, b, e, c(i->srcIdx), c((uint8_t)i->tag, 8)}); + {a, b, e, llvmSrcIdx(i->srcIdx), c((uint8_t)i->tag, 8)}); } else { res = call(NativeBuiltins::get(NativeBuiltins::Id::binop), {a, b, c((uint8_t)i->tag, 8)}); @@ -1730,7 +1851,7 @@ void LowerFunctionLLVM::compileUnop( if (i->hasEnv()) { auto e = loadSxp(i->env()); res = call(NativeBuiltins::get(NativeBuiltins::Id::unopEnv), - {a, e, c(i->srcIdx), c((uint8_t)i->tag, 8)}); + {a, e, llvmSrcIdx(i->srcIdx), c((uint8_t)i->tag, 8)}); } else { res = call(NativeBuiltins::get(NativeBuiltins::Id::unop), {a, c((uint8_t)i->tag, 8)}); @@ -1851,8 +1972,7 @@ bool LowerFunctionLLVM::compileDotcall( if (!seenDots) return false; Context asmpt = calli->inferAvailableAssumptions(); - auto namesConst = c(newNames); - auto namesStore = globalConst(namesConst); + auto namesStore = llvmNames(newNames); auto callId = ArglistOrder::NOT_REORDERED; if (calli->isReordered()) @@ -1866,12 +1986,13 @@ bool LowerFunctionLLVM::compileDotcall( { c(callId), paramCode(), - c(i->srcIdx), + // Call ASTs in cp pool, not src pool + llvmPoolIdx(i->srcIdx), callee(), i->hasEnv() ? loadSxp(i->env()) : constant(R_BaseEnv, t::SEXP), c(calli->nCallArgs()), - builder.CreateBitCast(namesStore, t::IntPtr), + namesStore, c(asmpt.toI()), }); }, @@ -2101,7 +2222,7 @@ void LowerFunctionLLVM::compile() { } } - nodestackPtrAddr = convertToPointer(&R_BCNodeStackTop, t::stackCellPtr); + nodestackPtrAddr = convertToPointer(&R_BCNodeStackTop, t::stackCellPtr, SerialRepr::R_BCNodeStackTop{}); basepointer = nodestackPtr(); size_t additionalStackSlots = 0; @@ -2168,9 +2289,6 @@ void LowerFunctionLLVM::compile() { } }; - constantpool = builder.CreateIntToPtr(c(globalContext()), t::SEXP_ptr); - constantpool = builder.CreateGEP(constantpool, c(1)); - Visitor::run(code->entry, [&](BB* bb) { for (auto i : *bb) { if (!liveness.count(i) || !allocator.needsAVariable(i)) @@ -2354,7 +2472,7 @@ void LowerFunctionLLVM::compile() { case Tag::DropContext: { auto globalContextPtrAddr = - convertToPointer(&R_GlobalContext, t::RCNTXT_ptr); + convertToPointer(&R_GlobalContext, t::RCNTXT_ptr, SerialRepr::R_GlobalContext{}); auto globalContextPtr = builder.CreateLoad(globalContextPtrAddr); auto callflagAddr = @@ -2471,8 +2589,8 @@ void LowerFunctionLLVM::compile() { auto callTheBuiltin = [&]() -> llvm::Value* { // Some "safe" builtins still look up functions in the base // env - return callRBuiltin(b->builtinSexp, args, i->srcIdx, - b->builtin, + return callRBuiltin(b->builtinId, b->builtinSexp, args, + i->srcIdx, b->builtin, constant(R_BaseEnv, t::SEXP)); }; @@ -3323,7 +3441,8 @@ void LowerFunctionLLVM::compile() { std::vector args; b->eachCallArg([&](Value* v) { args.push_back(v); }); setVal(i, callRBuiltin( - b->builtinSexp, args, i->srcIdx, b->builtin, + b->builtinId, b->builtinSexp, args, i->srcIdx, + b->builtin, b->hasEnv() ? loadSxp(b->env()) : constant(R_BaseEnv, t::SEXP))); break; @@ -3349,7 +3468,9 @@ void LowerFunctionLLVM::compile() { setVal(i, withCallFrame(args, [&]() -> llvm::Value* { return call( NativeBuiltins::get(NativeBuiltins::Id::call), - {c(callId), paramCode(), c(b->srcIdx), + {c(callId), paramCode(), + // Call ASTs in cp pool, not src pool + llvmPoolIdx(b->srcIdx), loadSxp(b->cls()), loadSxp(b->env()), c(b->nCallArgs()), c(asmpt.toI())}); })); @@ -3370,8 +3491,7 @@ void LowerFunctionLLVM::compile() { std::vector names; for (size_t i = 0; i < b->names.size(); ++i) names.push_back(Pool::insert((b->names[i]))); - auto namesConst = c(names); - auto namesStore = globalConst(namesConst); + auto namesStore = llvmNames(names); auto callId = ArglistOrder::NOT_REORDERED; if (b->isReordered()) @@ -3384,11 +3504,12 @@ void LowerFunctionLLVM::compile() { { c(callId), paramCode(), - c(b->srcIdx), + // Call ASTs in cp pool, not src pool + llvmPoolIdx(b->srcIdx), loadSxp(b->cls()), loadSxp(b->env()), c(b->nCallArgs()), - builder.CreateBitCast(namesStore, t::IntPtr), + namesStore, c(asmpt.toI()), }); })); @@ -3413,7 +3534,9 @@ void LowerFunctionLLVM::compile() { i, withCallFrame(args, [&]() -> llvm::Value* { return call( NativeBuiltins::get(NativeBuiltins::Id::call), - {c(callId), paramCode(), c(calli->srcIdx), + {c(callId), paramCode(), + // Call ASTs in cp pool, not src pool + llvmPoolIdx(calli->srcIdx), loadSxp(calli->runtimeClosure()), loadSxp(calli->env()), c(calli->nCallArgs()), c(asmpt.toI())}); @@ -3452,12 +3575,13 @@ void LowerFunctionLLVM::compile() { c(callId), paramCode(), constant(callee, t::SEXP), - c(idx), - c(calli->srcIdx), + llvmPoolIdx(idx), + // Call ASTs in cp pool, not src pool + llvmPoolIdx(calli->srcIdx), loadSxp(calli->env()), c(args.size()), c(asmpt.toI()), - c(missAsmptIdx), + llvmPoolIdx(missAsmptIdx), }); }); setVal(i, res); @@ -3472,9 +3596,9 @@ void LowerFunctionLLVM::compile() { { c(callId), paramCode(), - c(calli->srcIdx), - builder.CreateIntToPtr( - c(calli->cls()->rirClosure()), t::SEXP), + // Call ASTs in cp pool, not src pool + llvmPoolIdx(calli->srcIdx), + convertToPointer(calli->cls()->rirClosure()), loadSxp(calli->env()), c(calli->nCallArgs()), c(asmpt.toI()), @@ -3585,7 +3709,10 @@ void LowerFunctionLLVM::compile() { withCallFrame(args, [&]() { return call(NativeBuiltins::get(NativeBuiltins::Id::deopt), {paramCode(), paramClosure(), - convertToPointer(m, t::i8, true), paramArgs(), + convertToPointer(m, t::i8, + SerialRepr::DeoptMetadata{m}, + true), + paramArgs(), c(deopt->escapedEnv, 1), load(deopt->deoptReason()), loadSxp(deopt->deoptTrigger())}); @@ -3605,15 +3732,14 @@ void LowerFunctionLLVM::compile() { n = CONS_NR(n, R_NilValue); names.push_back(Pool::insert(n)); } - auto namesConst = c(names); - auto namesStore = globalConst(namesConst); + auto namesStore = llvmNames(names); if (mkenv->stub) { auto env = call(NativeBuiltins::get( NativeBuiltins::Id::createStubEnvironment), {parent, c((int)mkenv->nLocals()), - builder.CreateBitCast(namesStore, t::IntPtr), + namesStore, c(mkenv->context)}); protectTemp(env); size_t pos = 0; @@ -3713,7 +3839,7 @@ void LowerFunctionLLVM::compile() { if (i->hasEnv()) { res = call( NativeBuiltins::get(NativeBuiltins::Id::notEnv), - {argumentNative, loadSxp(i->env()), c(i->srcIdx)}); + {argumentNative, loadSxp(i->env()), llvmSrcIdx(i->srcIdx)}); } else { res = call(NativeBuiltins::get(NativeBuiltins::Id::notOp), @@ -4277,7 +4403,7 @@ void LowerFunctionLLVM::compile() { auto e = loadSxp(i->env()); res = call(NativeBuiltins::get(NativeBuiltins::Id::binopEnv), - {loadSxp(a), loadSxp(b), e, c(i->srcIdx), + {loadSxp(a), loadSxp(b), e, llvmSrcIdx(i->srcIdx), c((uint8_t)i->tag, 8)}); } else if (Rep::Of(a) == Rep::i32 && Rep::Of(b) == Rep::i32) { res = call(NativeBuiltins::get(NativeBuiltins::Id::colon), @@ -4599,6 +4725,8 @@ void LowerFunctionLLVM::compile() { {loadSxp(arg)}); } break; + default: + assert(false); } } else { assert(i->type.isA(RType::integer) || @@ -5014,7 +5142,7 @@ void LowerFunctionLLVM::compile() { auto idx = loadSxp(extract->idx()); auto res0 = call(NativeBuiltins::get(NativeBuiltins::Id::extract11), - {vector, idx, env, c(extract->srcIdx)}); + {vector, idx, env, llvmSrcIdx(extract->srcIdx)}); res.addInput(convert(res0, i->type)); if (fastcase) { @@ -5102,7 +5230,7 @@ void LowerFunctionLLVM::compile() { auto res0 = call(NativeBuiltins::get(NativeBuiltins::Id::extract12), {vector, idx1, idx2, loadSxp(extract->env()), - c(extract->srcIdx)}); + llvmSrcIdx(extract->srcIdx)}); res.addInput(convert(res0, i->type)); if (fastcase) { @@ -5170,14 +5298,14 @@ void LowerFunctionLLVM::compile() { auto vector = loadSxp(extract->vec()); res0 = call(getter, {vector, load(extract->idx()), - loadSxp(extract->env()), c(extract->srcIdx)}); + loadSxp(extract->env()), llvmSrcIdx(extract->srcIdx)}); } else { auto vector = loadSxp(extract->vec()); auto idx = loadSxp(extract->idx()); res0 = call(NativeBuiltins::get(NativeBuiltins::Id::extract21), {vector, idx, loadSxp(extract->env()), - c(extract->srcIdx)}); + llvmSrcIdx(extract->srcIdx)}); } res.addInput(convert(res0, i->type)); @@ -5205,7 +5333,7 @@ void LowerFunctionLLVM::compile() { auto res = call(NativeBuiltins::get(NativeBuiltins::Id::extract13), - {vector, idx1, idx2, idx3, env, c(extract->srcIdx)}); + {vector, idx1, idx2, idx3, env, llvmSrcIdx(extract->srcIdx)}); setVal(i, res); break; @@ -5287,7 +5415,7 @@ void LowerFunctionLLVM::compile() { res0 = call(getter, {vector, load(extract->idx1()), load(extract->idx2()), loadSxp(extract->env()), - c(extract->srcIdx)}); + llvmSrcIdx(extract->srcIdx)}); } else { auto vector = loadSxp(extract->vec()); @@ -5296,7 +5424,7 @@ void LowerFunctionLLVM::compile() { res0 = call(NativeBuiltins::get(NativeBuiltins::Id::extract22), {vector, idx1, idx2, loadSxp(extract->env()), - c(extract->srcIdx)}); + llvmSrcIdx(extract->srcIdx)}); } res.addInput(convert(res0, i->type)); @@ -5322,7 +5450,7 @@ void LowerFunctionLLVM::compile() { auto res = call(NativeBuiltins::get(NativeBuiltins::Id::subassign13), {vector, idx1, idx2, idx3, val, - loadSxp(subAssign->env()), c(subAssign->srcIdx)}); + loadSxp(subAssign->env()), llvmSrcIdx(subAssign->srcIdx)}); setVal(i, res); break; } @@ -5339,7 +5467,7 @@ void LowerFunctionLLVM::compile() { auto res = call(NativeBuiltins::get(NativeBuiltins::Id::subassign12), {vector, idx1, idx2, val, loadSxp(subAssign->env()), - c(subAssign->srcIdx)}); + llvmSrcIdx(subAssign->srcIdx)}); setVal(i, res); break; } @@ -5445,13 +5573,13 @@ void LowerFunctionLLVM::compile() { setter, {loadSxp(subAssign->vec()), load(subAssign->idx1()), load(subAssign->idx2()), load(subAssign->val()), - loadSxp(subAssign->env()), c(subAssign->srcIdx)}); + loadSxp(subAssign->env()), llvmSrcIdx(subAssign->srcIdx)}); } else { assign = call( NativeBuiltins::get(NativeBuiltins::Id::subassign22), {loadSxp(subAssign->vec()), idx1, idx2, loadSxp(subAssign->val()), loadSxp(subAssign->env()), - c(subAssign->srcIdx)}); + llvmSrcIdx(subAssign->srcIdx)}); } res.addInput(assign); @@ -5539,7 +5667,7 @@ void LowerFunctionLLVM::compile() { call(NativeBuiltins::get(NativeBuiltins::Id::subassign11), {loadSxp(subAssign->vec()), loadSxp(subAssign->idx()), loadSxp(subAssign->val()), loadSxp(subAssign->env()), - c(subAssign->srcIdx)}); + llvmSrcIdx(subAssign->srcIdx)}); res.addInput(convert(res0, i->type)); if (fastcase) { @@ -5652,13 +5780,13 @@ void LowerFunctionLLVM::compile() { call(setter, {loadSxp(subAssign->vec()), load(subAssign->idx()), load(subAssign->val()), loadSxp(subAssign->env()), - c(subAssign->srcIdx)}); + llvmSrcIdx(subAssign->srcIdx)}); } else { res0 = call( NativeBuiltins::get(NativeBuiltins::Id::subassign21), {loadSxp(subAssign->vec()), loadSxp(subAssign->idx()), loadSxp(subAssign->val()), loadSxp(subAssign->env()), - c(subAssign->srcIdx)}); + llvmSrcIdx(subAssign->srcIdx)}); } res.addInput(convert(res0, i->type)); @@ -5917,7 +6045,7 @@ void LowerFunctionLLVM::compile() { if (Rep::Of(a) == Rep::SEXP || Rep::Of(b) == Rep::SEXP) { setVal(i, call(NativeBuiltins::get( NativeBuiltins::Id::colonInputEffects), - {loadSxp(a), loadSxp(b), c(i->srcIdx)})); + {loadSxp(a), loadSxp(b), llvmSrcIdx(i->srcIdx)})); break; } @@ -6125,21 +6253,23 @@ void LowerFunctionLLVM::compile() { if (cls->isContinuation() && Rep::Of(i) == Rep::SEXP && variables_.count(i) && !cls->isContinuation()->continuationContext->asDeoptContext()) { - if (i->hasTypeFeedback() && - i->typeFeedback().feedbackOrigin.pc()) { - call(NativeBuiltins::get( - NativeBuiltins::Id::recordTypefeedback), - {c((void*)i->typeFeedback().feedbackOrigin.pc()), - c((void*)i->typeFeedback().feedbackOrigin.srcCode()), - load(i)}); + if (i->hasTypeFeedback()) { + auto& origin = i->typeFeedback().feedbackOrigin; + if (origin.hasSlot()) { + call( + NativeBuiltins::get( + NativeBuiltins::Id::recordTypeFeedback), + {convertToPointer(origin.function()->typeFeedback(), true), + c(origin.index().idx, 32), load(i)}); + } } if (i->hasCallFeedback()) { - assert(i->callFeedback().feedbackOrigin.pc()); + auto& origin = i->callFeedback().feedbackOrigin; + assert(origin.hasSlot()); call(NativeBuiltins::get( - NativeBuiltins::Id::recordTypefeedback), - {c((void*)i->callFeedback().feedbackOrigin.pc()), - c((void*)i->callFeedback().feedbackOrigin.srcCode()), - load(i)}); + NativeBuiltins::Id::recordCallFeedback), + {convertToPointer(origin.function()->typeFeedback(), true), + c(origin.index().idx, 32), load(i)}); } } @@ -6162,7 +6292,7 @@ void LowerFunctionLLVM::compile() { } call(NativeBuiltins::get(NativeBuiltins::Id::checkType), {loadSxp(i), c((unsigned long)i->type.serialize()), - convertToPointer(msg, t::i8, true)}); + convertToPointer(msg, t::i8, SerialRepr::String{msg}, true)}); } } #ifdef ENABLE_SLOWASSERT @@ -6245,12 +6375,13 @@ void LowerFunctionLLVM::compile() { auto i = var.first; if (Rep::Of(i) != Rep::SEXP) continue; - if (!i->typeFeedback().feedbackOrigin.pc()) + if (!i->typeFeedback().feedbackOrigin.hasSlot()) continue; if (!var.second.initialized) continue; if (var.second.stackSlot < PirTypeFeedback::MAX_SLOT_IDX) { - codes.insert(i->typeFeedback().feedbackOrigin.srcCode()); + codes.insert( + i->typeFeedback().feedbackOrigin.function()->body()); variableMapping.emplace(var.second.stackSlot, i->typeFeedback()); #ifdef DEBUG_REGISTER_MAP diff --git a/rir/src/compiler/native/lower_function_llvm.h b/rir/src/compiler/native/lower_function_llvm.h index c54e31889..46dc012f2 100644 --- a/rir/src/compiler/native/lower_function_llvm.h +++ b/rir/src/compiler/native/lower_function_llvm.h @@ -9,6 +9,8 @@ #include "compiler/native/types_llvm.h" #include "compiler/pir/pir.h" #include "runtime/Code.h" +#include "serializeHash/serialize/native/SerialRepr.h" +#include "serializeHash/serialize/serialize.h" #include #include "llvm/IR/DIBuilder.h" @@ -42,7 +44,6 @@ class LowerFunctionLLVM { size_t numTemps; size_t maxTemps; llvm::Value* basepointer = nullptr; - llvm::Value* constantpool = nullptr; llvm::BasicBlock* entryBlock = nullptr; int inPushContext = 0; std::unordered_set escapesInlineContext; @@ -71,6 +72,8 @@ class LowerFunctionLLVM { PirJitLLVM::DebugInfo* DI; llvm::DIBuilder* DIB; + SerialOptions serialOpts; + Protect p_; public: @@ -85,7 +88,7 @@ class LowerFunctionLLVM { const std::unordered_set& needsLdVarForUpdate, PirJitLLVM::Declare declare, const PirJitLLVM::GetModule& getModule, const PirJitLLVM::GetFunction& getFunction, PirJitLLVM::DebugInfo* DI, - llvm::DIBuilder* DIB) + llvm::DIBuilder* DIB, const SerialOptions& serialOpts) : target(target), cls(cls), code(code), promMap(promMap), refcount(refcount), needsLdVarForUpdate(needsLdVarForUpdate), builder(PirJitLLVM::getContext()), MDB(PirJitLLVM::getContext()), @@ -94,8 +97,8 @@ class LowerFunctionLLVM { branchAlwaysFalse(MDB.createBranchWeights(1, 100000000)), branchMostlyTrue(MDB.createBranchWeights(1000, 1)), branchMostlyFalse(MDB.createBranchWeights(1, 1000)), - getModule(getModule), getFunction(getFunction), DI(DI), DIB(DIB) { - + getModule(getModule), getFunction(getFunction), DI(DI), DIB(DIB), + serialOpts(serialOpts) { fun = declare(code, name, t::nativeFunction); auto p = promMap.find(code); @@ -111,13 +114,46 @@ class LowerFunctionLLVM { llvm::FunctionCallee getBuiltin(const rir::pir::NativeBuiltin& b); + static llvm::FunctionCallee convertToFunction(llvm::Module& mod, + const void* what, + llvm::FunctionType* ty, + /// Currently only for builtins, if + /// we need to convert more functions + /// we'll need to change to fn-id, + /// tagged union or something else + int builtinId); llvm::FunctionCallee convertToFunction(const void* what, - llvm::FunctionType* ty); + llvm::FunctionType* ty, + /// Currently only for builtins, if + /// we need to convert more functions + /// we'll need to change to fn-id, + /// tagged union or something else + int builtinId); + static llvm::Value* convertToPointer(llvm::Module& mod, const void* what, + llvm::Type* ty, bool constant, + llvm::MDNode* reprMeta); llvm::Value* convertToPointer(const void* what, llvm::Type* ty, + const SerialRepr& repr, bool constant = false); llvm::Value* convertToPointer(SEXP what, bool constant = false) { - return convertToPointer(what, t::SEXPREC, constant); + return convertToPointer(what, t::SEXPREC, SerialRepr::SEXP{what}, constant); + } + llvm::Value* convertToPointer(rir::Function* fun, bool constant = false) { + return convertToPointer(fun, t::RirRuntimeObject, SerialRepr::Function{fun}, constant); } + llvm::Value* convertToPointer(rir::TypeFeedback* typeFeedback, bool constant) { + return convertToPointer(typeFeedback, t::i8, SerialRepr::TypeFeedback{typeFeedback}, constant); + } + + static llvm::Value* llvmSrcIdx(llvm::Module& mod, Immediate i, + const SerialOptions& serialOpts); + llvm::Value* llvmSrcIdx(Immediate i); + static llvm::Value* llvmPoolIdx(llvm::Module& mod, BC::PoolIdx i, + const SerialOptions& serialOpts); + llvm::Value* llvmPoolIdx(BC::PoolIdx i); + static llvm::Value* llvmNames(llvm::Module& mod, + const std::vector& names); + llvm::Value* llvmNames(const std::vector& names); struct Variable { bool deadMove(const Variable& other) const; @@ -280,11 +316,6 @@ class LowerFunctionLLVM { std::vector loadedArgs; - static llvm::Constant* c(void* i) { - return llvm::ConstantInt::get(PirJitLLVM::getContext(), - llvm::APInt(64, (intptr_t)i)); - } - static llvm::Constant* c(unsigned long i, int bs = 64) { return llvm::ConstantInt::get(PirJitLLVM::getContext(), llvm::APInt(bs, i)); @@ -310,15 +341,11 @@ class LowerFunctionLLVM { llvm::APFloat(d)); } - static llvm::Constant* c(const std::vector& array) { - std::vector init; - for (const auto& e : array) - init.push_back(c(e)); - auto ty = llvm::ArrayType::get(t::Int, array.size()); - return llvm::ConstantArray::get(ty, init); - } - - llvm::Value* globalConst(llvm::Constant* init, llvm::Type* ty = nullptr); + static llvm::GlobalVariable* globalConst(llvm::Module& mod, + llvm::Constant* init, + llvm::Type* ty = nullptr); + llvm::GlobalVariable* globalConst(llvm::Constant* init, + llvm::Type* ty = nullptr); llvm::AllocaInst* topAlloca(llvm::Type* t, size_t len = 1); llvm::Value* argument(int i); @@ -393,7 +420,8 @@ class LowerFunctionLLVM { llvm::CallInst* call(const NativeBuiltin& builtin, const std::vector& args); - llvm::Value* callRBuiltin(SEXP builtin, const std::vector& args, + llvm::Value* callRBuiltin(int builtinId, SEXP builtin, + const std::vector& args, int srcIdx, CCODE, llvm::Value* env); llvm::Value* box(llvm::Value* v, PirType t, bool protect = true); diff --git a/rir/src/compiler/native/pir_jit_llvm.cpp b/rir/src/compiler/native/pir_jit_llvm.cpp index f4c688292..f35841b1b 100644 --- a/rir/src/compiler/native/pir_jit_llvm.cpp +++ b/rir/src/compiler/native/pir_jit_llvm.cpp @@ -4,8 +4,11 @@ #include "compiler/native/lower_function_llvm.h" #include "compiler/native/pass_schedule_llvm.h" #include "compiler/native/types_llvm.h" +#include "serializeHash/serialize/native/SerialModule.h" +#include "serializeHash/hash/hashRoot.h" #include "utils/filesystem.h" +#include "compiler/parameter.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" @@ -24,6 +27,7 @@ namespace rir { namespace pir { std::unique_ptr PirJitLLVM::JIT; +std::unordered_map PirJitLLVM::internedModules; size_t PirJitLLVM::nModules = 1; bool PirJitLLVM::initialized = false; @@ -297,7 +301,8 @@ void PirJitLLVM::DebugInfo::clearLocation(llvm::IRBuilder<>& builder) { builder.SetCurrentDebugLocation(llvm::DebugLoc()); } -PirJitLLVM::PirJitLLVM(const std::string& name) : name(name) { +PirJitLLVM::PirJitLLVM(const std::string& name, const SerialOptions& serialOpts) + : name(name), serialOpts(serialOpts) { if (!initialized) initializeLLVM(); } @@ -312,16 +317,24 @@ PirJitLLVM::~PirJitLLVM() { void PirJitLLVM::finalize() { assert(!finalized); if (M) { + auto serialModule = + Parameter::SERIALIZE_LLVM ? + internModule(SerialModule::unpack(SerialModule::create(*M, serialOpts))).first : + nullptr; + if (serialModule) { + PROTECT(serialModule->container()); + } // Should this happen before finalize or after? if (LLVMDebugInfo()) { DIB->finalize(); } - // TODO: maybe later have TSM from the start and use locking - // to allow concurrent compilation? - auto TSM = llvm::orc::ThreadSafeModule(std::move(M), TSC); - ExitOnErr(JIT->addIRModule(std::move(TSM))); - for (auto& fix : jitFixup) - fix.second.first->lazyCodeHandle(fix.second.second.str()); + addToJit(std::move(M)); + for (auto& fix : jitFixup) { + fix.second.first->lazyCode(fix.second.second, serialModule); + } + if (serialModule) { + UNPROTECT(1); + } nModules++; } finalized = true; @@ -353,11 +366,8 @@ void PirJitLLVM::compile( DI->initializeTypes(DIB.get()); - // Darwin only supports dwarf2. M->addModuleFlag(llvm::Module::Warning, "Dwarf Version", - JIT->getTargetTriple().isOSDarwin() - ? 2 - : llvm::dwarf::DWARF_VERSION); + llvm::dwarf::DWARF_VERSION); // Add the current debug info version into the module. M->addModuleFlag(llvm::Module::Warning, "Debug Info Version", @@ -403,7 +413,7 @@ void PirJitLLVM::compile( return r->second; return nullptr; }, - DI.get(), DIB.get()); + DI.get(), DIB.get(), serialOpts); llvm::DISubprogram* SP = nullptr; if (LLVMDebugInfo()) { @@ -441,7 +451,7 @@ void PirJitLLVM::compile( target->pirTypeFeedback(funCompiler.pirTypeFeedback); if (funCompiler.hasArgReordering()) target->arglistOrder(ArglistOrder::New(funCompiler.getArgReordering())); - jitFixup.emplace(code, std::make_pair(target, funCompiler.fun->getName())); + jitFixup.emplace(code, std::make_pair(target, funCompiler.fun->getName().str())); log.LLVMBitcode([&](std::ostream& out, bool tty) { bool debug = true; @@ -457,7 +467,28 @@ void PirJitLLVM::compile( }); } -llvm::LLVMContext& PirJitLLVM::getContext() { return *TSC.getContext(); } +llvm::LLVMContext& PirJitLLVM::getContext() { + if (!initialized) { + initializeLLVM(); + } + return *TSC.getContext(); +} + +SerialModule* +PirJitLLVM::deserializeModule(AbstractDeserializer& deserializer, + rir::Code* outer, + const SerialOptions& overrideSerialOpts) { + auto serialModule = SerialModule::unpack(deserializer.read(SerialFlags::CodeNative)); + auto serialModuleAndIsNew = internModule(serialModule); + PROTECT(serialModule->container()); + serialModule = serialModuleAndIsNew.first; + if (serialModuleAndIsNew.second) { + addToJit(serialModule->decode(outer, overrideSerialOpts)); + } + UNPROTECT(1); + return serialModule; + +} void PirJitLLVM::initializeLLVM() { if (initialized) @@ -552,10 +583,19 @@ void PirJitLLVM::initializeLLVM() { [MainName = JIT->mangleAndIntern("main")]( const SymbolStringPtr& Name) { return Name != MainName; }))); - // TODO this is a bit of a hack but it works: the address is stored in the - // name. symbols starting with "ept_" are external pointers, the ones - // starting with "efn_" are external function pointers. these must exist in - // the host process. + // The address or pool index is stored in the name: + // - symbols starting with "ept_" are external pointers + // - symbols starting with "efn_" are external function pointers + // - symbols starting with "src_" are source pool entries + // - symbols starting with "cp_" are constant pool entries + // - symbols starting with "names_" are vectors of names (constant pool + // entries). "names_" is the symbol for the empty vector, others won't + // have a trailing "_" + // + // These all must exist in the host process. + // + // On macOS/clang/ARM (which one? idk) the symbols sometimes start with '_' + // before everything else, so we trim that. class ExtSymbolGenerator : public llvm::orc::DefinitionGenerator { public: Error tryToGenerate(LookupState& LS, LookupKind K, JITDylib& JD, @@ -565,11 +605,19 @@ void PirJitLLVM::initializeLLVM() { for (auto s : LookupSet) { auto& Name = s.first; auto n = (*Name).str(); + if (n[0] == '_') { + n = n.substr(1); + } + auto ept = n.substr(0, 4) == "ept_"; auto efn = n.substr(0, 4) == "efn_"; + auto src = n.substr(0, 4) == "src_"; + auto cp = n.substr(0, 3) == "cp_"; + auto names = n.substr(0, 6) == "names_"; if (ept || efn) { - auto addrStr = n.substr(4); + // 16 = sizeof(uintptr_t) + auto addrStr = n.substr(4, 16); auto addr = std::strtoul(addrStr.c_str(), nullptr, 16); NewSymbols[Name] = JITEvaluatedSymbol( static_cast( @@ -577,6 +625,51 @@ void PirJitLLVM::initializeLLVM() { JITSymbolFlags::Exported | (efn ? JITSymbolFlags::Callable : JITSymbolFlags::None)); + } else if (src || cp) { + auto idxStr = n.substr(src ? 4 : 3, 8); + auto idx = std::strtoul(idxStr.c_str(), nullptr, 16); + + auto container = Rf_allocVector(INTSXP, 1); + // TODO: Don't leak memory, attach to object so that this + // gets freed when the last Code object using it does (also + // in SerialRepr) + R_PreserveObject(container); + INTEGER(container)[0] = (int)idx; + + NewSymbols[Name] = JITEvaluatedSymbol( + static_cast( + reinterpret_cast(INTEGER(container))), + JITSymbolFlags::Exported); + } else if (names) { + if (n == "names_") { + // Special case, we have an empty vector. + // It won't be read, so we can pass a dangling address + // (idk if there's an idiomatic way to do this in LLVM + // or if it causes some kind of UB) + NewSymbols[Name] = + JITEvaluatedSymbol(static_cast( + (uintptr_t)0xdeadbeef), + JITSymbolFlags::Exported); + } else { + auto numNames = (R_xlen_t)std::count(n.begin(), n.end(), '_'); + auto container = Rf_allocVector(INTSXP, numNames); + // TODO: Don't leak memory, attach to object so that this + // gets freed when the last Code object using it does (also + // in SerialRepr) + R_PreserveObject(container); + size_t idx = 6; + for (R_xlen_t i = 0; i < numNames; ++i) { + auto nextIdx = n.find('_', idx); + auto idxStr = n.substr(idx, nextIdx - idx); + INTEGER(container)[i] = (int)std::strtoul(idxStr.c_str(), nullptr, 16); + idx = nextIdx + 1; + } + + NewSymbols[Name] = JITEvaluatedSymbol( + static_cast( + reinterpret_cast(INTEGER(container))), + JITSymbolFlags::Exported); + } } else { std::cout << "unknown symbol " << n << "\n"; } @@ -604,5 +697,34 @@ void PirJitLLVM::initializeLLVM() { initialized = true; } +void PirJitLLVM::addToJit(std::unique_ptr&& M) { + // TODO: maybe later have TSM from the start and use locking + // to allow concurrent compilation? + auto TSM = llvm::orc::ThreadSafeModule(std::move(M), TSC); + ExitOnErr(JIT->addIRModule(std::move(TSM))); +} + +void PirJitLLVM::uninternModuleBeforeGc(SEXP moduleSexp) { + assert(SerialModule::check(moduleSexp)); + auto moduleId = hashRoot(moduleSexp); + assert(!internedModules.count(moduleId) || + internedModules.at(moduleId)->container() == moduleSexp); + internedModules.erase(moduleId); +} + +std::pair PirJitLLVM::internModule(SerialModule* module) { + assert(module); + PROTECT(module->container()); + auto moduleId = hashRoot(module->container()); + if (internedModules.count(moduleId)) { + UNPROTECT(1); + return std::make_pair(internedModules.at(moduleId), false); + } + module->makeFinalizer(uninternModuleBeforeGc, false); + internedModules.emplace(moduleId, module); + UNPROTECT(1); + return std::make_pair(module, true); +} + } // namespace pir } // namespace rir diff --git a/rir/src/compiler/native/pir_jit_llvm.h b/rir/src/compiler/native/pir_jit_llvm.h index 3338430d8..e185b26ef 100644 --- a/rir/src/compiler/native/pir_jit_llvm.h +++ b/rir/src/compiler/native/pir_jit_llvm.h @@ -10,6 +10,7 @@ #include "compiler/pir/pir.h" #include "compiler/pir/promise.h" #include "compiler/util/visitor.h" +#include "serializeHash/serialize/serialize.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/IR/DIBuilder.h" @@ -26,6 +27,8 @@ namespace rir { struct Code; +struct SerialOptions; +class SerialModule; namespace pir { @@ -43,7 +46,8 @@ using PromMap = std::unordered_map>; class PirJitLLVM { public: static std::unique_ptr JIT; - explicit PirJitLLVM(const std::string& name); + static std::unordered_map internedModules; + PirJitLLVM(const std::string& name, const SerialOptions& serialOpts); PirJitLLVM(const PirJitLLVM&) = delete; PirJitLLVM(PirJitLLVM&&) = delete; ~PirJitLLVM(); @@ -62,6 +66,21 @@ class PirJitLLVM { static llvm::LLVMContext& getContext(); + public: + /// Deserialize and the module. Then if interned, return the interned + /// version, otherwise intern AND add to LLJIT. + /// + /// `outer` is the code object which will contain the module, needed because + /// we add stuff to its extra pool so that it remains alive while being used + /// by the code. It can be nullptr if we only create the objects for a short + /// period of time (when printing). + /// + /// `overrideSerialOpts` are the options used to deserialize SEXPs in the + /// module. Specifically, we pass special options on the compiler client to + /// materialize `ProxyEnv`s. + static SerialModule* deserializeModule( + AbstractDeserializer& deserializer, rir::Code* outer, + const SerialOptions& overrideSerialOpts); private: std::string name; @@ -71,6 +90,10 @@ class PirJitLLVM { // Directory of all functions and builtins std::unordered_map funs; + // Options we use when serializing SEXPs within the bitcode so that it can + // be transferred across processes. + SerialOptions serialOpts; + // We prepend `rshN_` to all user functions, as a mechanism to // differentiate them from builtins. `N` denotes that the definition // belongs to module N. Builtins will be declared in the module with @@ -82,13 +105,17 @@ class PirJitLLVM { return ss.str().substr(0, rir::Code::MAX_CODE_HANDLE_LENGTH - 6); } - std::unordered_map> jitFixup; + std::unordered_map> jitFixup; bool finalized = false; static size_t nModules; static void initializeLLVM(); static bool initialized; + static void addToJit(std::unique_ptr&& module); + static void uninternModuleBeforeGc(SEXP moduleSexp); + static std::pair internModule(SerialModule* module); + // Support for debugging pir in gdb public: static std::string makeDbgFileName(const std::string& base) { diff --git a/rir/src/compiler/native/types_llvm.cpp b/rir/src/compiler/native/types_llvm.cpp index 7bdb4fc39..ff45ad997 100644 --- a/rir/src/compiler/native/types_llvm.cpp +++ b/rir/src/compiler/native/types_llvm.cpp @@ -66,6 +66,10 @@ void initializeTypes(LLVMContext& context) { t::RirRuntimeObject = StructType::create(context, "RirRuntimeObject"); t::RirRuntimeObject->setBody(fields); + // Function is a subclass of RirRuntimeObject. It has additional fields but LLVM + // doesn't care + t::Function_ptr = PointerType::get(t::RirRuntimeObject, 0); + t::stackCell = StructType::create(context, "R_bcstack_t"); // struct { int tag; int flags; union { ival, dval, sxpval} } fields = {t::Int, t::Int, t::SEXP}; @@ -110,7 +114,7 @@ void initializeTypes(LLVMContext& context) { t::RCNTXT->setBody(fields); t::DeoptReason = StructType::create(context, "DeoptReason"); - fields = {t::i32, t::i32, t::voidPtr}; + fields = {t::i32, t::i32, t::Function_ptr}; t::DeoptReason->setBody(fields, true); t::DeoptReasonPtr = llvm::PointerType::get(t::DeoptReason, 0); @@ -166,6 +170,7 @@ StructType* SEXPREC; StructType* VECTOR_SEXPREC; StructType* LazyEnvironment; +PointerType* Function_ptr; StructType* RirRuntimeObject; StructType* setjmp_buf; diff --git a/rir/src/compiler/native/types_llvm.h b/rir/src/compiler/native/types_llvm.h index d73ad37a0..aae12adac 100644 --- a/rir/src/compiler/native/types_llvm.h +++ b/rir/src/compiler/native/types_llvm.h @@ -33,6 +33,7 @@ extern llvm::StructType* VECTOR_SEXPREC; extern llvm::PointerType* VECTOR_SEXPREC_ptr; extern llvm::StructType* RirRuntimeObject; +extern llvm::PointerType* Function_ptr; extern llvm::StructType* LazyEnvironment; extern llvm::StructType* DeoptReason; diff --git a/rir/src/compiler/opt/eager_calls.cpp b/rir/src/compiler/opt/eager_calls.cpp index a0504a217..47606005d 100644 --- a/rir/src/compiler/opt/eager_calls.cpp +++ b/rir/src/compiler/opt/eager_calls.cpp @@ -2,12 +2,11 @@ #include "../analysis/query.h" #include "../pir/pir_impl.h" #include "../util/safe_builtins_list.h" -#include "../util/visitor.h" -#include "R/Funtab.h" #include "R/Symbols.h" #include "R/r.h" #include "compiler/analysis/cfg.h" #include "compiler/compiler.h" +#include "runtime/ProxyEnv.h" #include "pass_definitions.h" #include @@ -22,11 +21,11 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, struct Speculation { SEXP builtin; Checkpoint* cp; - FeedbackOrigin origin; + FeedbackPosition origin; Speculation() {} - Speculation(SEXP builtin, Checkpoint* cp, const FeedbackOrigin& origin) + Speculation(SEXP builtin, Checkpoint* cp, const FeedbackPosition& origin) : builtin(builtin), cp(cp), origin(origin) { - assert(origin.pc()); + assert(origin.hasSlot()); } }; @@ -140,7 +139,8 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, break; VECTOR_RW_INSTRUCTIONS(V); #undef V - default: {} + default: { + } } if (auto call = Call::Cast(*ip)) { @@ -182,7 +182,12 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, if (!ldfun->guessedBinding()) { auto env = Env::Cast(cls->owner()->closureEnv()); - if (env != Env::notClosed() && env->rho) { + if (env != Env::notClosed() && env->rho && + // TODO: Speculate in proxies, either by + // providing a list of functions to stub or + // (probably better) sending a request to + // the client + !ProxyEnv::check(env->rho)) { auto name = ldfun->varName; auto builtin = Rf_findVar(name, env->rho); if (TYPEOF(builtin) == PROMSXP) @@ -210,7 +215,7 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, } if (!inBase && ldfun->typeFeedback() - .feedbackOrigin.pc()) + .feedbackOrigin.hasSlot()) needsGuard[ldfun] = { builtin, cp, ldfun->typeFeedback() @@ -262,7 +267,8 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, } break; VECTOR_RW_INSTRUCTIONS(V) #undef V - default: {} + default: { + } } // Look for static calls, where we statically know that all (or @@ -409,9 +415,9 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, } next = ip + 1; - // This might fire back, since we don't know if we really have no - // objects... We should have some profiling. It's still sound, since - // static_call_ will check the assumptions + // This might fire back, since we don't know if we really have + // no objects... We should have some profiling. It's still + // sound, since static_call_ will check the assumptions for (size_t i = 0; i < call->nCallArgs(); ++i) if (!newAssumptions.isNotObj(i) && newAssumptions.isEager(i)) diff --git a/rir/src/compiler/opt/inline.cpp b/rir/src/compiler/opt/inline.cpp index 4d2f79b81..0b583086e 100644 --- a/rir/src/compiler/opt/inline.cpp +++ b/rir/src/compiler/opt/inline.cpp @@ -11,6 +11,7 @@ #include "compiler/parameter.h" #include "compiler/util/bb_transform.h" #include "compiler/util/visitor.h" +#include "runtime/ProxyEnv.h" #include "pass_definitions.h" #include "utils/Pool.h" @@ -29,11 +30,11 @@ bool Inline::apply(Compiler& cmp, ClosureVersion* cls, Code* code, return false; auto dontInline = [](Closure* cls) { - if (cls->rirFunction()->flags.contains(rir::Function::DisableInline)) + if (cls->rirFunction()->flags().contains(rir::Function::DisableInline)) return true; - if (cls->rirFunction()->flags.contains(rir::Function::ForceInline)) + if (cls->rirFunction()->flags().contains(rir::Function::ForceInline)) return false; - return cls->rirFunction()->flags.contains(rir::Function::NotInlineable); + return cls->rirFunction()->flags().contains(rir::Function::NotInlineable); }; Visitor::run(code->entry, [&](BB* bb) { @@ -194,7 +195,8 @@ bool Inline::apply(Compiler& cmp, ClosureVersion* cls, Code* code, } } auto env = Env::Cast(inlineeCls->closureEnv()); - if (env && env->rho && R_IsNamespaceEnv(env->rho)) { + // If rho is a ProxyEnv, it's guaranteed not to be a namespace + if (env && env->rho && !ProxyEnv::check(env->rho) && R_IsNamespaceEnv(env->rho)) { auto expr = BODY_EXPR(inlineeCls->rirClosure()); // Closure wrappers for internals if (CAR(expr) == rir::symbol::Internal) @@ -220,24 +222,24 @@ bool Inline::apply(Compiler& cmp, ClosureVersion* cls, Code* code, inlinee->owner()->rirFunction()->body())) { continue; } else if (weight > Parameter::INLINER_MAX_INLINEE_SIZE) { - if (!inlineeCls->rirFunction()->flags.contains( + if (!inlineeCls->rirFunction()->flags().contains( rir::Function::ForceInline) && inlinee->numNonDeoptInstrs() > Parameter::INLINER_MAX_INLINEE_SIZE * 4) - inlineeCls->rirFunction()->flags.set( + inlineeCls->rirFunction()->setFlag( rir::Function::NotInlineable); continue; } else { updateAllowInline(inlinee); inlinee->eachPromise([&](Promise* p) { updateAllowInline(p); }); if (allowInline == SafeToInline::No) { - inlineeCls->rirFunction()->flags.set( + inlineeCls->rirFunction()->setFlag( rir::Function::NotInlineable); continue; } } - if (!inlineeCls->rirFunction()->flags.contains( + if (!inlineeCls->rirFunction()->flags().contains( rir::Function::ForceInline)) fuel--; @@ -376,7 +378,7 @@ bool Inline::apply(Compiler& cmp, ClosureVersion* cls, Code* code, for (auto bb : toDel) delete bb; bb->overrideNext(split); - inlineeCls->rirFunction()->flags.set( + inlineeCls->rirFunction()->setFlag( rir::Function::NotInlineable); } else { anyChange = true; diff --git a/rir/src/compiler/opt/inline_force_prom.cpp b/rir/src/compiler/opt/inline_force_prom.cpp index 31d1c11b4..9420497c3 100644 --- a/rir/src/compiler/opt/inline_force_prom.cpp +++ b/rir/src/compiler/opt/inline_force_prom.cpp @@ -41,7 +41,7 @@ bool InlineForcePromises::apply(Compiler&, ClosureVersion* cls, Code* code, if (clsCallee) { auto functionVersion = clsCallee->rirFunction(); - if (functionVersion->flags.contains( + if (functionVersion->flags().contains( rir::Function::Flag::DepromiseArgs)) { call->eachCallArg([&](InstrArg& v) { diff --git a/rir/src/compiler/opt/scope_resolution.cpp b/rir/src/compiler/opt/scope_resolution.cpp index 9ce2c44ef..6608e30ab 100644 --- a/rir/src/compiler/opt/scope_resolution.cpp +++ b/rir/src/compiler/opt/scope_resolution.cpp @@ -3,11 +3,11 @@ #include "../pir/pir_impl.h" #include "../util/phi_placement.h" #include "../util/safe_builtins_list.h" -#include "../util/visitor.h" #include "R/r.h" #include "compiler/analysis/context_stack.h" #include "compiler/compiler.h" #include "compiler/util/bb_transform.h" +#include "runtime/ProxyEnv.h" #include "pass_definitions.h" #include "utils/Set.h" @@ -679,7 +679,7 @@ bool ScopeResolution::apply(Compiler& cmp, ClosureVersion* cls, Code* code, SafeBuiltinsList::assumeStableInBaseEnv( name)) { auto value = SYMVALUE(name); - assert(Rf_findVar(name, env->rho) == value); + assert(ProxyEnv::check(env->rho) || Rf_findVar(name, env->rho) == value); if (TYPEOF(value) == PROMSXP) value = PRVALUE(value); if (value != R_UnboundValue) diff --git a/rir/src/compiler/opt/type_test.h b/rir/src/compiler/opt/type_test.h index cb6501e43..c0bb7636e 100644 --- a/rir/src/compiler/opt/type_test.h +++ b/rir/src/compiler/opt/type_test.h @@ -12,7 +12,7 @@ class TypeTest { PirType result; Instruction* test; bool expectation; - FeedbackOrigin feedbackOrigin; + FeedbackPosition feedbackOrigin; }; static void Create(Value* i, const TypeFeedback& feedback, const PirType& suggested, const PirType& required, @@ -35,10 +35,10 @@ class TypeTest { return failed(); } - if (!feedback.feedbackOrigin.pc()) + if (!feedback.feedbackOrigin.hasSlot()) return failed(); - assert(feedback.feedbackOrigin.pc()); + assert(feedback.feedbackOrigin.hasSlot()); // First try to refine the type if (!expected.maybeObj() && // TODO: Is this right? (expected.noAttribsOrObject().isA(RType::integer) || diff --git a/rir/src/compiler/opt/typefeedback_cleanup.cpp b/rir/src/compiler/opt/typefeedback_cleanup.cpp index 1f8489994..48c981047 100644 --- a/rir/src/compiler/opt/typefeedback_cleanup.cpp +++ b/rir/src/compiler/opt/typefeedback_cleanup.cpp @@ -25,7 +25,8 @@ bool TypefeedbackCleanup::apply(Compiler& cmp, ClosureVersion* cls, Code* code, std::unordered_set affected; if (deoptCtx) { - if (deoptCtx->reason().srcCode() != cls->rirSrc()) { + if (deoptCtx->reason().origin.function() != + cls->owner()->rirFunction()) { Visitor::run(version->entry, [&](Instruction* i) { if (!i->hasTypeFeedback()) return; @@ -36,8 +37,8 @@ bool TypefeedbackCleanup::apply(Compiler& cmp, ClosureVersion* cls, Code* code, if (!i->hasTypeFeedback()) return; - if (i->typeFeedback().feedbackOrigin.pc() == - deoptCtx->reason().pc()) { + if (i->typeFeedback().feedbackOrigin == + deoptCtx->reason().origin) { if (deoptCtx->reason().reason == DeoptReason::Typecheck) { i->updateTypeFeedback().type = deoptCtx->typeCheckTrigger(); diff --git a/rir/src/compiler/osr.cpp b/rir/src/compiler/osr.cpp index c6ac62c88..b41f16395 100644 --- a/rir/src/compiler/osr.cpp +++ b/rir/src/compiler/osr.cpp @@ -4,6 +4,7 @@ #include "compiler/compiler.h" #include "pir/deopt_context.h" #include "pir/pir_impl.h" +#include "runtime/DispatchTable.h" namespace rir { namespace pir { @@ -19,7 +20,12 @@ Function* OSR::compile(SEXP closure, rir::Code* c, logger.title("Compiling continuation"); pir::Compiler cmp(module, logger); - pir::Backend backend(module, logger, "continuation"); + pir::Backend backend(module, logger, "continuation", + // Right now, serial options aren't important here, + // because OSR and serialization are completely + // separate. So we could probably pass anything. What + // would we pass if they were used? idk + SerialOptions::DeepCopy); cmp.compileContinuation( closure, c->function(), &ctx, @@ -31,6 +37,9 @@ Function* OSR::compile(SEXP closure, rir::Code* c, delete module; + auto dt = DispatchTable::unpack(BODY(closure)); + fun->dispatchTable(dt); + return fun; } diff --git a/rir/src/compiler/parameter.h b/rir/src/compiler/parameter.h index 3a474abdf..16192b090 100644 --- a/rir/src/compiler/parameter.h +++ b/rir/src/compiler/parameter.h @@ -2,6 +2,7 @@ #define PIR_PARAMETER_H #include +#include namespace rir { namespace pir { @@ -16,6 +17,7 @@ struct Parameter { static const unsigned PIR_WARMUP; static const unsigned PIR_OPT_TIME; static const unsigned PIR_REOPT_TIME; + static const unsigned PIR_OPT_BC_SIZE; static const unsigned DEOPT_ABANDON; static size_t PROMISE_INLINER_MAX_SIZE; @@ -27,6 +29,7 @@ struct Parameter { static size_t RECOMPILE_THRESHOLD; + /// Controls whether we save RIR data in native R serialization (e.g. on quit()) static bool RIR_PRESERVE; static unsigned RIR_SERIALIZE_CHAOS; @@ -37,7 +40,38 @@ struct Parameter { static bool ENABLE_PIR2RIR; + /// Enabled by default, but PIR_OSR=0 will disable static bool ENABLE_OSR; + /// Enable OSR even during serialization, where it's known to break, and on + /// the compiler client (not dry-run), where it's also known to break and + /// the client shouldn't be serializing code anyways. + /// + /// Disabled by default, but PIR_OSR=1 will enable + static bool FORCE_ENABLE_OSR; + + /// Log every time a closure is compiled (no OSR) and how long it takes. + static bool PIR_MEASURE_COMPILED_CLOSURES; + + /// Serialize LLVM bitcode. Enabled regardless of env var iff the compiler + /// server is running, otherwise enabled if PIR_PIR_DEBUG_SERIALIZE_LLVM is set + static bool SERIALIZE_LLVM; + + static bool PIR_GRAPH_PRINT_RIR_OBJECTS; + static const char* PIR_GRAPH_PRINT_RIR_OBJECTS_PATH; + static unsigned PIR_GRAPH_PRINT_RIR_OBJECTS_FREQUENCY; + + static bool PIR_TRACE_SERIALIZATION; + static unsigned PIR_TRACE_SERIALIZATION_MAX_RAW_PRINT_LENGTH; + static size_t PIR_TRACE_SERIALIZATION_MIN_SIZE; + static std::vector* PIR_TRACE_SERIALIZATION_EXCLUDE; + static bool PIR_MEASURE_SERIALIZATION; + static bool PIR_LOG_INTERNING; + static bool PIR_WARN_INTERNING; + static bool PIR_MEASURE_INTERNING; + static bool PIR_TRACE_COMPILER_PEER; + static bool PIR_LOG_COMPILER_PEER; + static bool PIR_WARN_COMPILER_PEER; + static bool PIR_MEASURE_CLIENT_SERVER; }; } // namespace pir diff --git a/rir/src/compiler/pir/builder.cpp b/rir/src/compiler/pir/builder.cpp index e72412ae2..51e2e1c30 100644 --- a/rir/src/compiler/pir/builder.cpp +++ b/rir/src/compiler/pir/builder.cpp @@ -54,7 +54,8 @@ void Builder::add(Instruction* i) { assert(false && "Invalid instruction"); case Tag::PirCopy: assert(false && "This instruction is only allowed during lowering"); - default: {} + default: { + } } bb->append(i); } @@ -125,8 +126,9 @@ Builder::Builder(Continuation* cnt, Value* closureEnv) } auto mkenv = new MkEnv(closureEnv, names, args.data(), miss); - auto rirCode = cnt->owner()->rirFunction()->body(); - mkenv->updateTypeFeedback().feedbackOrigin.srcCode(rirCode); + // FIXME: what does this mean, we need both rirFun and we need idx + mkenv->updateTypeFeedback().feedbackOrigin.function( + cnt->owner()->rirFunction()); add(mkenv); this->env = mkenv; } else { @@ -150,7 +152,7 @@ Builder::Builder(ClosureVersion* version, Value* closureEnv) std::vector args(closure->nargs()); size_t nargs = version->effectiveNArgs(); - auto depromiseArgs = version->owner()->rirFunction()->flags.contains( + auto depromiseArgs = version->owner()->rirFunction()->flags().contains( rir::Function::Flag::DepromiseArgs); for (long i = nargs - 1; i >= 0; --i) { @@ -170,9 +172,10 @@ Builder::Builder(ClosureVersion* version, Value* closureEnv) auto mkenv = new MkEnv(closureEnv, closure->formals().names(), args.data()); auto rirFun = version->owner()->rirFunction(); - if (rirFun->flags.contains(rir::Function::NeedsFullEnv)) + if (rirFun->flags().contains(rir::Function::NeedsFullEnv)) mkenv->neverStub = true; - mkenv->updateTypeFeedback().feedbackOrigin.srcCode(rirFun->body()); + // FIXME: what does this mean, we need both rirFun and we need idx + mkenv->updateTypeFeedback().feedbackOrigin.function(rirFun); add(mkenv); this->env = mkenv; } diff --git a/rir/src/compiler/pir/closure.cpp b/rir/src/compiler/pir/closure.cpp index cb9e3df6f..39f88fb78 100644 --- a/rir/src/compiler/pir/closure.cpp +++ b/rir/src/compiler/pir/closure.cpp @@ -3,6 +3,7 @@ #include "continuation.h" #include "env.h" #include "runtime/DispatchTable.h" +#include "runtime/ProxyEnv.h" namespace rir { namespace pir { diff --git a/rir/src/compiler/pir/instruction.cpp b/rir/src/compiler/pir/instruction.cpp index c0644b955..0acb340a5 100644 --- a/rir/src/compiler/pir/instruction.cpp +++ b/rir/src/compiler/pir/instruction.cpp @@ -211,7 +211,7 @@ void Instruction::print(std::ostream& out, bool tty) const { typeFeedback().value->printRef(out); else if (!typeFeedback().type.isVoid()) out << typeFeedback().type; - if (!typeFeedback().feedbackOrigin.pc()) + if (!typeFeedback().feedbackOrigin.hasSlot()) out << "@?"; out << ">"; } diff --git a/rir/src/compiler/pir/instruction.h b/rir/src/compiler/pir/instruction.h index f186cca50..2c0e42634 100644 --- a/rir/src/compiler/pir/instruction.h +++ b/rir/src/compiler/pir/instruction.h @@ -146,10 +146,10 @@ enum class VisibilityFlag : uint8_t { struct TypeFeedback { PirType type = PirType::optimistic(); Value* value = nullptr; - FeedbackOrigin feedbackOrigin; + FeedbackPosition feedbackOrigin; }; struct CallFeedback { - FeedbackOrigin feedbackOrigin; + FeedbackPosition feedbackOrigin; size_t taken = 0; SEXP monomorphic = nullptr; SEXPTYPE type = NILSXP; @@ -984,12 +984,12 @@ class VLIE(FrameState, Effects() | Effect::ReadsEnv) { class FLIE(LdFun, 2, Effects::Any()) { private: SEXP hint_ = nullptr; - FeedbackOrigin hintOrigin_; + FeedbackPosition hintOrigin_; public: SEXP hint() { return hint_; } - const FeedbackOrigin& hintOrigin() { return hintOrigin_; } - void hint(SEXP hint, const FeedbackOrigin& hintOrigin) { + const FeedbackPosition& hintOrigin() { return hintOrigin_; } + void hint(SEXP hint, const FeedbackPosition& hintOrigin) { hint_ = hint; hintOrigin_ = hintOrigin; } diff --git a/rir/src/compiler/pir/module.cpp b/rir/src/compiler/pir/module.cpp index 50dfdc029..c66824f9a 100644 --- a/rir/src/compiler/pir/module.cpp +++ b/rir/src/compiler/pir/module.cpp @@ -1,6 +1,10 @@ #include "module.h" +#include "compiler/parameter.h" +#include "compilerClientServer/CompilerClient.h" +#include "compilerClientServer/CompilerServer.h" #include "pir_impl.h" +#include "runtime/ProxyEnv.h" #include "runtime/TypeFeedback.h" #include "utils/Pool.h" #include "values.h" @@ -31,12 +35,20 @@ Closure* Module::getOrDeclareRirClosure(const std::string& name, SEXP closure, // the real environment if this is not an inner function. When it is an // inner function, then the env is expected to change over time. auto id = Idx(f, getEnv(CLOENV(closure))); - auto env = f->flags.contains(Function::InnerFunction) + auto env = f->flags().contains(Function::InnerFunction) ? Env::notClosed() : getEnv(CLOENV(closure)); if (!closures.count(id)) closures[id] = new Closure(name, closure, f, env, userContext); - assert(closures.at(id)->rirClosure() == closure); + // If the compiler server is running sometimes this false. + // Or client, but only if we're not calling hashRoot on children. + // Thus it probably means closures.at(id) is an equivalent duplicate. + // TODO: Investigate + assert(closures.at(id)->rirClosure() == closure || + CompilerServer::isRunning() || + CompilerClient::isRunning() || + Parameter::RIR_SERIALIZE_CHAOS > 0 || + Parameter::SERIALIZE_LLVM); return closures.at(id); } @@ -59,8 +71,8 @@ Env* Module::getEnv(SEXP rho) { if (environments.count(rho)) return environments.at(rho); - assert(TYPEOF(rho) == ENVSXP); - Env* parent = getEnv(ENCLOS(rho)); + assert(TYPEOF(rho) == ENVSXP || ProxyEnv::check(rho)); + Env* parent = getEnv(ProxyEnv::check(rho) ? ProxyEnv::unpack(rho)->parent() : ENCLOS(rho)); Env* env = new Env(rho, parent); environments[rho] = env; return env; diff --git a/rir/src/compiler/rir2pir/rir2pir.cpp b/rir/src/compiler/rir2pir/rir2pir.cpp index e157fedc8..b341f75d0 100644 --- a/rir/src/compiler/rir2pir/rir2pir.cpp +++ b/rir/src/compiler/rir2pir/rir2pir.cpp @@ -17,6 +17,7 @@ #include "compiler/util/visitor.h" #include "insert_cast.h" #include "runtime/ArglistOrder.h" +#include "runtime/TypeFeedback.h" #include "simple_instruction_list.h" #include "utils/FormalArgs.h" @@ -144,9 +145,10 @@ namespace pir { Rir2Pir::Rir2Pir(Compiler& cmp, ClosureVersion* cls, ClosureLog& log, const std::string& name, - const std::list& outerFeedback) + const std::list& outerFeedback, + rir::TypeFeedback* typeFeedback) : compiler(cmp), cls(cls), log(log), name(name), - outerFeedback(outerFeedback) { + outerFeedback(outerFeedback), typeFeedback(typeFeedback) { if (cls->optFunction && cls->optFunction->body()->pirTypeFeedback()) this->outerFeedback.push_back( cls->optFunction->body()->pirTypeFeedback()); @@ -370,7 +372,9 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } case Opcode::record_test_: { - auto feedback = bc.immediate.testFeedback; + uint32_t idx = bc.immediate.i; + auto& feedback = typeFeedback->test(idx); + if (feedback.seen == ObservedTest::OnlyTrue || feedback.seen == ObservedTest::OnlyFalse) { if (auto i = Instruction::Cast(at(0))) { @@ -380,7 +384,8 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, if (!i->typeFeedback().value) { auto& t = i->updateTypeFeedback(); t.value = v; - t.feedbackOrigin = FeedbackOrigin(srcCode, pos); + t.feedbackOrigin = FeedbackPosition(srcCode->function(), + FeedbackIndex::test(idx)); } else if (i->typeFeedback().value != v) { i->updateTypeFeedback().value = nullptr; } @@ -395,7 +400,9 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } case Opcode::record_type_: { - auto feedback = bc.immediate.typeFeedback; + uint32_t idx = bc.immediate.i; + auto& feedback = typeFeedback->types(idx); + if (auto i = Instruction::Cast(at(0))) { // Search for the most specific feedback for this location for (auto fb : outerFeedback) { @@ -404,8 +411,9 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, fb->forEachSlot( [&](size_t i, const PirTypeFeedback::MDEntry& mdEntry) { found = true; - auto origin = fb->getOriginOfSlot(i); - if (origin == pos && mdEntry.readyForReopt) { + auto origin = fb->rirIdx(i); + if (origin == FeedbackIndex::type(idx) && + mdEntry.readyForReopt) { feedback = mdEntry.feedback; } }); @@ -414,7 +422,8 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } // TODO: deal with multiple locations auto& t = i->updateTypeFeedback(); - t.feedbackOrigin = FeedbackOrigin(srcCode, pos); + t.feedbackOrigin = + FeedbackPosition(srcCode->function(), FeedbackIndex::type(idx)); if (feedback.numTypes) { t.type.merge(feedback); if (auto force = Force::Cast(i)) { @@ -431,9 +440,10 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } case Opcode::record_call_: { + uint32_t idx = bc.immediate.i; Value* target = top(); - auto feedback = bc.immediate.callFeedback; + auto& feedback = typeFeedback->callees(bc.immediate.i); // If this call was never executed we might as well compile an // unconditional deopt. @@ -443,8 +453,9 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, auto sp = insert.registerFrameState(srcCode, pos, stack, inPromise()); - DeoptReason reason = DeoptReason(FeedbackOrigin(srcCode, pos), - DeoptReason::DeadCall); + DeoptReason reason = DeoptReason( + FeedbackPosition(srcCode->function(), FeedbackIndex::call(idx)), + DeoptReason::DeadCall); auto d = insert(new Deopt(sp)); d->setDeoptReason(compiler.module->deoptReasonValue(reason), @@ -453,17 +464,17 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } else if (auto i = Instruction::Cast(target)) { // See if the call feedback suggests a monomorphic target // TODO: Deopts in promises are not supported by the promise - // inliner. So currently it does not pay off to put any deopts in - // there. + // inliner. So currently it does not pay off to put any deopts + // in there. // auto& f = i->updateCallFeedback(); - const auto& feedback = bc.immediate.callFeedback; f.taken = feedback.taken; - f.feedbackOrigin = FeedbackOrigin(srcCode, pos); + f.feedbackOrigin = + FeedbackPosition(srcCode->function(), FeedbackIndex::call(idx)); if (feedback.numTargets == 1) { assert(!feedback.invalid && "feedback can't be invalid if numTargets is 1"); - f.monomorphic = feedback.getTarget(srcCode, 0); + f.monomorphic = feedback.getTarget(srcCode->function(), 0); f.type = TYPEOF(f.monomorphic); f.stableEnv = true; } else if (feedback.numTargets > 1) { @@ -472,7 +483,7 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, bool stableBody = !feedback.invalid; bool stableEnv = !feedback.invalid; for (size_t i = 0; i < feedback.numTargets; ++i) { - SEXP b = feedback.getTarget(srcCode, i); + SEXP b = feedback.getTarget(srcCode->function(), i); if (!first) { first = b; } else { @@ -492,11 +503,12 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, if (auto c = cls->isContinuation()) { if (auto d = c->continuationContext->asDeoptContext()) { if (d->reason().reason == DeoptReason::CallTarget) { - if (d->reason().pc() == pos) { + if (d->reason().origin.idx() == idx) { auto deoptCallTarget = d->callTargetTrigger(); for (size_t i = 0; i < feedback.numTargets; ++i) { - SEXP b = feedback.getTarget(srcCode, i); + SEXP b = feedback.getTarget( + srcCode->function(), i); if (b != deoptCallTarget) deoptedCallTargets.insert(b); } @@ -617,7 +629,7 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, bool stableEnv = ti.stableEnv; if (monomorphicClosure) if (auto dt = DispatchTable::check(BODY(ti.monomorphic))) - if (dt->baseline()->flags.includes( + if (dt->baseline()->flags().includes( Function::Flag::InnerFunction)) stableEnv = false; @@ -1347,12 +1359,14 @@ bool Rir2Pir::tryCompile(rir::Code* srcCode, Builder& insert, Opcode* start, } bool Rir2Pir::tryCompilePromise(rir::Code* prom, Builder& insert) { - return PromiseRir2Pir(compiler, cls, log, name, outerFeedback, false) + return PromiseRir2Pir(compiler, cls, log, name, outerFeedback, typeFeedback, + false) .tryCompile(prom, insert); } Value* Rir2Pir::tryInlinePromise(rir::Code* srcCode, Builder& insert) { - return PromiseRir2Pir(compiler, cls, log, name, outerFeedback, true) + return PromiseRir2Pir(compiler, cls, log, name, outerFeedback, typeFeedback, + true) .tryTranslate(srcCode, insert); } @@ -1363,6 +1377,8 @@ Value* Rir2Pir::tryTranslate(rir::Code* srcCode, Builder& insert) { Value* Rir2Pir::tryTranslate(rir::Code* srcCode, Builder& insert, Opcode* start, const std::vector& initialStack) { assert(!finalized); + assert(start >= srcCode->code()); + assert(start <= srcCode->endCode()); auto firstBB = insert.getCurrentBB(); insert.createNextBB(); @@ -1434,10 +1450,12 @@ Value* Rir2Pir::tryTranslate(rir::Code* srcCode, Builder& insert, Opcode* start, BC bc = BC::advance(&finger, srcCode); // cppcheck-suppress variableScope const auto nextPos = finger; + assert(nextPos <= end); assert(pos != end); if (bc.isJmp()) { auto trg = bc.jmpTarget(pos); + assert(trg <= end); if (bc.isUncondJmp()) { finger = trg; continue; @@ -1576,6 +1594,7 @@ Value* Rir2Pir::tryTranslate(rir::Code* srcCode, Builder& insert, Opcode* start, BC ldcode = BC::advance(&pc, srcCode); BC ldsrc = BC::advance(&pc, srcCode); pc = BC::next(pc); // close + assert(pc <= end); SEXP formals = ldfmls.immediateConst(); SEXP code = ldcode.immediateConst(); diff --git a/rir/src/compiler/rir2pir/rir2pir.h b/rir/src/compiler/rir2pir/rir2pir.h index afe022161..1f463b383 100644 --- a/rir/src/compiler/rir2pir/rir2pir.h +++ b/rir/src/compiler/rir2pir/rir2pir.h @@ -3,6 +3,7 @@ #include "compiler/compiler.h" #include "compiler/pir/builder.h" +#include "runtime/TypeFeedback.h" #include #include @@ -17,7 +18,8 @@ class Rir2Pir { public: Rir2Pir(Compiler& cmp, ClosureVersion* cls, ClosureLog& log, const std::string& name, - const std::list& outerFeedback); + const std::list& outerFeedback, + rir::TypeFeedback* typeFeedback); bool tryCompile(Builder& insert) __attribute__((warn_unused_result)); bool tryCompileContinuation(Builder& insert, Opcode* start, @@ -58,6 +60,7 @@ class Rir2Pir { ClosureLog& log; std::string name; std::list outerFeedback; + rir::TypeFeedback* typeFeedback; std::unordered_map localFuns; std::unordered_set deoptedCallTargets; @@ -88,8 +91,9 @@ class PromiseRir2Pir : public Rir2Pir { PromiseRir2Pir(Compiler& cmp, ClosureVersion* cls, ClosureLog& log, const std::string& name, const std::list& outerFeedback, - bool inlining) - : Rir2Pir(cmp, cls, log, name, outerFeedback), inlining_(inlining) {} + rir::TypeFeedback* feedback, bool inlining) + : Rir2Pir(cmp, cls, log, name, outerFeedback, feedback), + inlining_(inlining) {} private: bool inlining_; diff --git a/rir/src/compiler/test/PirTests.cpp b/rir/src/compiler/test/PirTests.cpp index d42569c56..12f0af36a 100644 --- a/rir/src/compiler/test/PirTests.cpp +++ b/rir/src/compiler/test/PirTests.cpp @@ -50,13 +50,12 @@ SEXP compileToRir(const std::string& context, const std::string& expr, typedef std::unordered_map ClosuresByName; ClosuresByName compileRir2Pir(SEXP env, pir::Module* m) { - pir::Log logger({pir::DebugOptions::DebugFlags() | - // pir::DebugFlag::PrintIntoStdout | - // pir::DebugFlag::PrintEarlyPir | - // pir::DebugFlag::PrintOptimizationPasses | - pir::DebugFlag::PrintFinalPir, - std::regex(".*"), std::regex(".*"), - pir::DebugStyle::Standard}); + pir::Log logger(pir::DebugOptions( + pir::DebugOptions::DebugFlags() | + // pir::DebugFlag::PrintIntoStdout | + // pir::DebugFlag::PrintEarlyPir | + // pir::DebugFlag::PrintOptimizationPasses | + pir::DebugFlag::PrintFinalPir)); pir::Compiler cmp(m, logger); // Compile every function in the environment @@ -353,7 +352,8 @@ bool testPir2Rir(const std::string& name, const std::string& fun, rCall = createRWrapperCall(wrapper); } - pirCompile(rirFun, {}, "from_testPir2Rir", rir::pir::DebugOptions()); + rirFun = pirCompile(rirFun, {}, "from_testPir2Rir", rir::pir::DebugOptions()); + (void)rirFun; auto after = p(Rf_eval(rCall, execEnv)); if (verbose) { diff --git a/rir/src/compiler/util/bb_transform.cpp b/rir/src/compiler/util/bb_transform.cpp index 7682c5c2d..4315e66cb 100644 --- a/rir/src/compiler/util/bb_transform.cpp +++ b/rir/src/compiler/util/bb_transform.cpp @@ -261,7 +261,7 @@ BB* BBTransform::lowerAssume(Module* m, Code* code, BB* srcBlock, } void BBTransform::insertAssume(Instruction* condition, bool assumePositive, - Checkpoint* cp, const FeedbackOrigin& origin, + Checkpoint* cp, const FeedbackPosition& origin, DeoptReason::Reason reason, BB* bb, BB::Instrs::iterator& position) { position = bb->insert(position, condition); @@ -272,7 +272,7 @@ void BBTransform::insertAssume(Instruction* condition, bool assumePositive, } void BBTransform::insertAssume(Instruction* condition, bool assumePositive, - Checkpoint* cp, const FeedbackOrigin& origin, + Checkpoint* cp, const FeedbackPosition& origin, DeoptReason::Reason reason) { auto contBB = cp->bb()->trueBranch(); auto contBegin = contBB->begin(); diff --git a/rir/src/compiler/util/bb_transform.h b/rir/src/compiler/util/bb_transform.h index a2c5dbc45..bdbaece4c 100644 --- a/rir/src/compiler/util/bb_transform.h +++ b/rir/src/compiler/util/bb_transform.h @@ -33,11 +33,11 @@ class BBTransform { size_t nDropContexts, bool condition, BB* deoptBlock_, const std::string& debugMesage); static void insertAssume(Instruction* condition, bool assumePositive, - Checkpoint* cp, const FeedbackOrigin& origin, + Checkpoint* cp, const FeedbackPosition& origin, DeoptReason::Reason reason, BB* bb, BB::Instrs::iterator& position); static void insertAssume(Instruction* condition, bool assumePositive, - Checkpoint* cp, const FeedbackOrigin& origin, + Checkpoint* cp, const FeedbackPosition& origin, DeoptReason::Reason reason); static Value* insertCalleeGuard(Compiler& compiler, const CallFeedback& fb, const DeoptReason& dr, Value* callee, diff --git a/rir/src/compilerClientServer/CompilerClient.cpp b/rir/src/compilerClientServer/CompilerClient.cpp new file mode 100644 index 000000000..a3762b261 --- /dev/null +++ b/rir/src/compilerClientServer/CompilerClient.cpp @@ -0,0 +1,672 @@ +// +// Created by Jakob Hain on 5/25/23. +// + +#include "CompilerClient.h" +#include "api.h" +#include "compiler_server_client_shared_utils.h" +#include "serializeHash/hash/UUID.h" +#include "serializeHash/hash/UUIDPool.h" +#include "serializeHash/serialize/serialize.h" +#include "utils/ByteBuffer.h" +#include "utils/Terminal.h" +#include "utils/measuring.h" +#ifdef MULTI_THREADED_COMPILER_CLIENT +#include "utils/ctpl.h" +#endif +#include "R/Printing.h" +#include "bc/Compiler.h" +#include +#include + +namespace rir { + +#ifdef MULTI_THREADED_COMPILER_CLIENT +using namespace ctpl; + +// Thread pool to handle compiler-server requests (AKA will only wait for this +// many requests simultaneously). Right now it's #servers, because each server +// is single-threded, but if we have multi-threaded servers in the future we can +// increase. +static int NUM_THREADS; +thread_pool* threads; +static std::chrono::milliseconds PIR_CLIENT_TIMEOUT; +#endif + +#define CHECK_MSG_SIZE(size, size2) if (size != size2) \ + std::cerr << "Different sizes: " << #size << "=" << size << ", " \ + << #size2 << "=" << size2 << std::endl + +#define LOG(stmt) if (pir::Parameter::PIR_TRACE_COMPILER_PEER || pir::Parameter::PIR_LOG_COMPILER_PEER) stmt +#define LOG_WARN(stmt) if (pir::Parameter::PIR_TRACE_COMPILER_PEER || pir::Parameter::PIR_LOG_COMPILER_PEER || pir::Parameter::PIR_WARN_COMPILER_PEER) stmt +#define LOG_DETAILED(stmt) if (pir::Parameter::PIR_TRACE_COMPILER_PEER) stmt +#define START_LOGGING_REQUEST() LOG_DETAILED(do { \ + logDetailedDepth++; \ + logDetailedIndent = std::string(logDetailedDepth * 2, ' '); \ + } while (0)) +#define END_LOGGING_REQUEST() LOG_DETAILED(do { \ + logDetailedDepth--; \ + logDetailedIndent = std::string(logDetailedDepth * 2, ' '); \ + } while (0)) +#define START_LOGGING_RESPONSE() START_LOGGING_REQUEST() +#define END_LOGGING_RESPONSE() END_LOGGING_REQUEST() +#define START_LOGGING_SERVER_REQUEST() START_LOGGING_REQUEST() +#define END_LOGGING_SERVER_REQUEST() END_LOGGING_REQUEST() +#define START_LOGGING_CLIENT_RESPONSE() START_LOGGING_REQUEST() +#define END_LOGGING_CLIENT_RESPONSE() END_LOGGING_REQUEST() +static int logDetailedDepth = 0; +static std::string logDetailedIndent; +// Arrows are different directions than CompilerServer.cpp, since we send +// requests and receive responses, receive server requests and send client +// responses +#define LOG_REQUEST(message) LOG_DETAILED(std::cerr << logDetailedIndent << ">> " << message << std::endl) +#define LOG_RESPONSE(message) LOG_DETAILED(std::cerr << logDetailedIndent << "<< " << message << std::endl) +#define LOG_SERVER_REQUEST(message) LOG_DETAILED(std::cerr << logDetailedIndent << "<<< " << message << std::endl) +#define LOG_CLIENT_RESPONSE(message) LOG_DETAILED(std::cerr << logDetailedIndent << ">>> " << message << std::endl) + +static const char* SENDING_REQUEST_TIMER_NAME = "CompilerClient.cpp: sending request"; +static const char* RECEIVING_RESPONSE_TIMER_NAME = "CompilerClient.cpp: receiving response"; +static const char* RETRIEVE_TIMER_NAME = "CompilerClient.cpp: retriving SEXP"; + +static bool PIR_CLIENT_INTERN = + getenv("PIR_CLIENT_INTERN") != nullptr && + strcmp(getenv("PIR_CLIENT_INTERN"), "") != 0 && + strcmp(getenv("PIR_CLIENT_INTERN"), "0") != 0; + +static bool PIR_CLIENT_SKIP_DISCREPANCY_CHECK = + getenv("PIR_CLIENT_SKIP_DISCREPANCY_CHECK") != nullptr && + strcmp(getenv("PIR_CLIENT_SKIP_DISCREPANCY_CHECK"), "") != 0 && + strcmp(getenv("PIR_CLIENT_SKIP_DISCREPANCY_CHECK"), "0") != 0; + +bool CompilerClient::_isRunning = false; +static zmq::context_t* context; +static std::vector* serverAddrs; +static std::vector* sockets; +static std::vector* socketsConnected; + +void CompilerClient::tryInit() { + // get the server address from the environment + const char* serverAddrStr = getenv("PIR_CLIENT_ADDR"); + if (serverAddrStr) { + std::cerr << "PIR_CLIENT_ADDR=" << serverAddrStr + << ", CompilerClient initializing..." << std::endl; + } else { +#ifdef FORCE_LOG_COMPILER_SERVER + std::cerr << "PIR_CLIENT_ADDR not set, CompilerClient won't initialize" << std::endl; +#endif + return; + } + + assert(!isRunning()); + _isRunning = true; + + // TODO: Figure out what objects we fail to retain and where, so we don't + // need this. Currently, enabling the GC causes a crash later on, and I've + // already tried preserving all deserialized objects, extra pool entries in + // compiled closures, and the compiled closures themselves + R_GCEnabled = false; + + serverAddrs = new std::vector(); + std::istringstream serverAddrReader(serverAddrStr); + while (!serverAddrReader.fail()) { + std::string serverAddr; + std::getline(serverAddrReader, serverAddr, ','); + if (serverAddr.empty()) + continue; + serverAddrs->push_back(serverAddr); + } +#ifdef MULTI_THREADED_COMPILER_CLIENT + PIR_CLIENT_TIMEOUT = std::chrono::milliseconds( + getenv("PIR_CLIENT_TIMEOUT") == nullptr + ? 10000 + : strtol(getenv("PIR_CLIENT_TIMEOUT"), nullptr, 10) + ); + NUM_THREADS = (int)serverAddrs->size(); + // initialize the thread pool + threads = new thread_pool(NUM_THREADS); + // initialize the zmq context + context = new zmq::context_t( + // We have our own thread pool, but zeromq also uses background threads. + // Presumably the socket polls on the background while it blocks the + // main thread for a response. Each socket runs on its own thread, and + // ideally each socket will take one of these io_threads for any of its + // background tasks, so that sockets won't have to wait for each other. + NUM_THREADS, + NUM_THREADS + ); +#else + assert(serverAddrs->size() == 1 && + "can't have multiple servers without multi-threaded client"); + context = new zmq::context_t(1, 1); +#endif + + // initialize the zmq sockets and connect to the servers + sockets = new std::vector(); + socketsConnected = new std::vector(); + for (const auto& serverAddr : *serverAddrs) { + auto socket = new zmq::socket_t(*context, zmq::socket_type::req); + socket->connect(serverAddr); + sockets->push_back(socket); + socketsConnected->push_back(true); + } +} + +static zmq::message_t +handleRetrieveServerRequest(int index, zmq::socket_t* socket, + const ByteBuffer& serverRequestBuffer) { + assert(PIR_CLIENT_INTERN && "interning disabled for this session"); + LOG(std::cerr << "Socket " << index << " received retrieve request" + << std::endl); + + // Deserialize the retrieve server-side request + // Data format = + // Response::NeedsRetrieve + // + UUID hash + START_LOGGING_SERVER_REQUEST(); + auto requestMagic = (Response)serverRequestBuffer.getLong(); + assert(requestMagic == Response::NeedsRetrieve); + LOG_SERVER_REQUEST("Response::NeedsRetrieve"); + UUID hash; + serverRequestBuffer.getBytes((uint8_t*)&hash, sizeof(UUID)); + LOG_SERVER_REQUEST("hash = " << hash); + END_LOGGING_SERVER_REQUEST(); + + LOG(std::cerr << "Retrieve " << hash << " -> "); + + // Get SEXP + SEXP what = UUIDPool::get(hash); + + // Serialize the client-side response + ByteBuffer clientResponse; + if (what) { + LOG(std::cerr << what << " " << Print::dumpSexp(what) << std::endl); + // Data format = + // Request::Retrieved + // + serialize(what, CompilerClientRetrieve) + START_LOGGING_CLIENT_RESPONSE(); + LOG_CLIENT_RESPONSE("Request::Retrieved"); + clientResponse.putLong((uint64_t)Request::Retrieved); + LOG_CLIENT_RESPONSE("serialize(" << Print::dumpSexp(what) << ", CompilerClientRetrieve)"); + serialize(what, clientResponse, SerialOptions::CompilerClientRetrieve); + END_LOGGING_CLIENT_RESPONSE(); + } else { + std::cerr << "(not found)" << std::endl; + // Data format = + // Request::RetrieveFailed + START_LOGGING_CLIENT_RESPONSE(); + LOG_CLIENT_RESPONSE("Request::RetrieveFailed"); + clientResponse.putLong((uint64_t)Request::RetrieveFailed); + END_LOGGING_CLIENT_RESPONSE(); + } + + // Send the client response + LOG(std::cerr << "Socket " << index << " sending retrieve response" + << std::endl); + CHECK_MSG_NOT_TOO_LARGE(clientResponse.size()); + auto clientResponseSize = *socket->send( + zmq::message_t(clientResponse.data(), + clientResponse.size()), + zmq::send_flags::none); + auto clientResponseSize2 = clientResponse.size(); + CHECK_MSG_SIZE(clientResponseSize, clientResponseSize2); + + // Return the server's next response + zmq::message_t serverResponse; + socket->recv(serverResponse, zmq::recv_flags::none); + return serverResponse; +} + +template +CompilerClient::Handle* CompilerClient::request( + const std::function&& makeRequest, + const std::function&& makeResponse) { + if (!isRunning()) { + return nullptr; + } + auto getResponse = [=](int index) { + auto socket = (*sockets)[index]; + auto socketConnected = (*socketsConnected)[index]; + if (!socket->handle()) { + LOG_WARN(std::cerr << "CompilerClient: socket closed" << std::endl); + *socket = zmq::socket_t(*context, zmq::socket_type::req); + socketConnected = false; + } + if (!socketConnected) { + const auto& serverAddr = (*serverAddrs)[index]; + LOG_WARN(std::cerr << "CompilerClient: reconnecting to " << serverAddr + << std::endl); + socket->connect(serverAddr); + (*socketsConnected)[index] = true; + } + + // Serialize the request + // Request data format = + // from makeRequest() + ByteBuffer request; + LOG_DETAILED(std::cerr << "Socket " << index << " building request" + << std::endl); + makeRequest(request); + + if (request.size() >= PIR_CLIENT_COMPILE_SIZE_TO_HASH_ONLY) { + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_REQUEST_TIMER_NAME, true); + UUID requestHash = UUID::hash(request.data(), request.size()); + LOG_DETAILED(std::cerr << "Socket " << index + << " building hashOnly request" << std::endl); + // Serialize the hash-only request + // Request data format = + // Request::Memoize + // + hash + START_LOGGING_REQUEST(); + ByteBuffer hashOnlyRequest; + LOG_REQUEST("Request::Memoize"); + hashOnlyRequest.putLong((uint64_t)Request::Memoize); + LOG_REQUEST("hash = " << requestHash); + hashOnlyRequest.putBytes((uint8_t*)&requestHash, sizeof(requestHash)); + END_LOGGING_REQUEST(); + + // Send the hash-only request + LOG(std::cerr << "Socket " << index << " sending hashOnly request" + << std::endl); + CHECK_MSG_NOT_TOO_LARGE(hashOnlyRequest.size()); + auto hashOnlyRequestSize = + *socket->send(zmq::message_t( + hashOnlyRequest.data(), + hashOnlyRequest.size()), + zmq::send_flags::none); + auto hashOnlyRequestSize2 = hashOnlyRequest.size(); + CHECK_MSG_SIZE(hashOnlyRequestSize, hashOnlyRequestSize2); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_REQUEST_TIMER_NAME, true); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, RECEIVING_RESPONSE_TIMER_NAME, true); + + // Wait for and retrieve the response + zmq::message_t hashOnlyResponse; + socket->recv(hashOnlyResponse, zmq::recv_flags::none); + + // Process the response + // Response data format = + // Response::NeedsFull + // | from makeResponse() + START_LOGGING_RESPONSE(); + ByteBuffer hashOnlyResponseBuffer((uint8_t*)hashOnlyResponse.data(), hashOnlyResponse.size()); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, RECEIVING_RESPONSE_TIMER_NAME, true); + auto hashOnlyResponseMagic = (Response)hashOnlyResponseBuffer.peekLong(); + if (hashOnlyResponseMagic != Response::NeedsFull) { + LOG(std::cerr << "Socket " << index + << " received memoized hashOnly response" + << std::endl); + return makeResponse(hashOnlyResponseBuffer); + } + LOG(std::cerr << "Socket " << index << " needs to send full request" + << std::endl); + LOG_RESPONSE("Response::NeedsFull"); + END_LOGGING_RESPONSE(); + } + + // Send the request + LOG(std::cerr << "Socket " << index << " sending request" << std::endl); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_REQUEST_TIMER_NAME, true); + CHECK_MSG_NOT_TOO_LARGE(request.size()); + auto requestSize = + *socket->send(zmq::message_t( + request.data(), + request.size()), + zmq::send_flags::none); + auto requestSize2 = request.size(); + CHECK_MSG_SIZE(requestSize, requestSize2); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_REQUEST_TIMER_NAME, true); + + // Wait for and receive the response + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, RECEIVING_RESPONSE_TIMER_NAME, true); + zmq::message_t response; + socket->recv(response, zmq::recv_flags::none); + ByteBuffer responseBuffer((uint8_t*)response.data(), response.size()); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, RECEIVING_RESPONSE_TIMER_NAME, true); + + // Handle retrieve requests + auto responseMagic = (Response)responseBuffer.peekLong(); + while (responseMagic == Response::NeedsRetrieve) { + response = handleRetrieveServerRequest(index, socket, responseBuffer); + responseBuffer = ByteBuffer((uint8_t*)response.data(), response.size()); + responseMagic = (Response)responseBuffer.peekLong(); + } + + // Process the response + // Response data format = + // from makeResponse() + LOG(std::cerr << "Socket " << index << " received response" << std::endl); + return makeResponse(responseBuffer); + }; +#ifdef MULTI_THREADED_COMPILER_CLIENT + std::shared_ptr socketIndexRef(new int(-1)); + return new CompilerClient::Handle{socketIndexRef, threads->push([=](index) { + *socketIndexRef = index; + return getResponse(index); + })}; +#else + auto response = getResponse(0); + return new CompilerClient::Handle{response}; +#endif +} + +CompilerClient::CompiledHandle* CompilerClient::pirCompile(SEXP what, const Context& assumptions, const std::string& name, const pir::DebugOptions& debug) { + CompilerClient::CompiledHandle* handle = nullptr; + + auto function = DispatchTable::unpack(BODY(what))->baseline(); + auto decompiled = Compiler::decompileClosure(what); + auto compilerClientOptions = SerialOptions::CompilerClient(PIR_CLIENT_INTERN, function, decompiled); + // TODO: Is this preserve necessary? + R_PreserveObject(function->container()); + + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, "CompilerClient.cpp: pirCompile", what, [&]{ + auto innerHandle = request( + [=](ByteBuffer& request) { + // Request data format = + // Request::Compile +#if COMPILER_CLIENT_SEND_SOURCE_AND_FEEDBACK + // + bool PIR_CLIENT_INTERN + // + serialize(Compiler::decompileClosure(what), CompilerClient(...)) + // + DispatchTable::unpack(BODY(what))->baseline()->fullSignature() + // + serialize(DispatchTable::unpack(BODY(what))->baseline()->typeFeedback()->container(), CompilerClient(...)) + // + (uintptr_t)DispatchTable::unpack(BODY(what))->baseline()->body() + // + DispatchTable::unpack(BODY(what))->baseline()->body()->extraPoolSize +#endif +#if COMPILER_CLIENT_SEND_FULL + // + serialize(what, SourceAndFeedback) +#endif + // + sizeof(assumptions) (always 8) + // + assumptions + // + sizeof(name) + // + name + // + sizeof(debug.flags) (always 4) + // + debug.flags + // + sizeof(debug.passFilterString) + // + debug.passFilterString + // + sizeof(debug.functionFilterString) + // + debug.functionFilterString + // + sizeof(debug.style) (always 4) + // + debug.style + START_LOGGING_REQUEST(); + LOG_REQUEST("Request::Compile"); + request.putLong((uint64_t)Request::Compile); +#if COMPILER_CLIENT_SEND_SOURCE_AND_FEEDBACK + LOG_REQUEST("PIR_CLIENT_INTERN = " << PIR_CLIENT_INTERN); + request.putBool(PIR_CLIENT_INTERN); + LOG_REQUEST("serialize(" << Print::dumpSexp(decompiled) << ", CompilerClient(...))"); + serialize(decompiled, request, compilerClientOptions); + auto baseline = DispatchTable::unpack(BODY(what))->baseline(); + LOG_REQUEST("full signature"); + baseline->serializeFullSignature(request); + auto feedback = baseline->typeFeedback(); + serialize(feedback->container(), request, compilerClientOptions); + request.putInt(function->body()->extraPoolSize); + request.putInt(function->nargs()); + for (unsigned defaultArgIdx = 0; + defaultArgIdx < function->nargs(); defaultArgIdx++) { + auto defaultArg = function->defaultArg(defaultArgIdx); + request.putInt(defaultArg ? defaultArg->extraPoolSize : 0); + } +#endif +#if COMPILER_CLIENT_SEND_FULL + LOG_REQUEST("serialize(" << Print::dumpSexp(what) << ", SourceAndFeedback)"); + serialize(what, request, SerialOptions::SourceAndFeedback); +#endif + LOG_REQUEST("assumptions = " << assumptions); + request.putLong(sizeof(Context)); + request.putBytes((uint8_t*)&assumptions, sizeof(Context)); + LOG_REQUEST("name = " << name); + request.putLong(name.size()); + request.putBytes((uint8_t*)name.c_str(), name.size()); + LOG_REQUEST("debug = " << debug); + request.putLong(sizeof(debug.flags)); + request.putBytes((uint8_t*)&debug.flags, sizeof(debug.flags)); + request.putLong(debug.passFilterString.size()); + request.putBytes((uint8_t*)debug.passFilterString.c_str(), + debug.passFilterString.size()); + request.putLong(debug.functionFilterString.size()); + request.putBytes((uint8_t*)debug.functionFilterString.c_str(), + debug.functionFilterString.size()); + request.putLong(sizeof(debug.style)); + request.putBytes((uint8_t*)&debug.style, sizeof(debug.style)); + END_LOGGING_REQUEST(); + }, + [=](const ByteBuffer& response) { + // Response data format = + // Response::Compiled + // + sizeof(pirPrint) + // + pirPrint + // + serialize(what, CompilerClient(...)) + START_LOGGING_RESPONSE(); + auto responseMagic = (Response)response.getLong(); + assert(responseMagic == Response::Compiled); + LOG_RESPONSE("Response::Compiled"); + auto pirPrintSize = response.getLong(); + std::string pirPrint; + pirPrint.resize(pirPrintSize); + response.getBytes((uint8_t*)pirPrint.data(), pirPrintSize); + LOG_RESPONSE("pirPrint = (size = " << pirPrint.size() << ")"); + SEXP responseWhat = deserialize(response, compilerClientOptions); + LOG_RESPONSE("serialize(" << Print::dumpSexp(responseWhat) + << ", CompilerServer)"); + END_LOGGING_RESPONSE(); + + // TODO: Is the above preserve necessary? + R_ReleaseObject(function->container()); + return CompilerClient::CompiledResponseData{responseWhat, std::move(pirPrint)}; + } + ); + if (innerHandle) { + handle = new CompilerClient::CompiledHandle{innerHandle}; + } + }); + return handle; +} + +SEXP CompilerClient::retrieve(const rir::UUID& hash) { + assert(PIR_CLIENT_INTERN && "interning disabled for this session"); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, RETRIEVE_TIMER_NAME, true); + auto handle = request( + [=](ByteBuffer& request) { + // Request data format = + // Request::Retrieve + // + bool PIR_CLIENT_INTERN + // + hash + START_LOGGING_REQUEST(); + LOG_REQUEST("Request::Retrieve"); + request.putLong((uint64_t)Request::Retrieve); + LOG_REQUEST("PIR_CLIENT_INTERN = " << PIR_CLIENT_INTERN); + request.putBool(PIR_CLIENT_INTERN); + LOG_REQUEST("hash = " << hash); + request.putBytes((uint8_t*)&hash, sizeof(hash)); + END_LOGGING_REQUEST(); + }, + [=](const ByteBuffer& response) -> SEXP { + // Response data format = + // Response::Retrieved + // + serialize(what, CompilerServer) + // | Response::RetrieveFailed + START_LOGGING_RESPONSE(); + auto responseMagic = (Response)response.getLong(); + switch (responseMagic) { + case Response::Retrieved: { + LOG_RESPONSE("Response::Retrieved"); + auto what = deserialize(response, SerialOptions::CompilerServer(PIR_CLIENT_INTERN), hash); + LOG_RESPONSE("serialize(" << Print::dumpSexp(what) << ", CompilerServer)"); + END_LOGGING_RESPONSE(); + return what; + } + case Response::RetrieveFailed: + LOG_RESPONSE("Response::RetrieveFailed"); + END_LOGGING_RESPONSE(); + return nullptr; + default: + assert(false && "Unexpected response magic"); + } + } + ); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, RETRIEVE_TIMER_NAME, true); +#ifdef MULTI_THREADED_COMPILER_CLIENT +#error "TODO create closure which blocks until the response is ready" +#else + auto response = handle ? handle->response : nullptr; + delete handle; + return response; +#endif +} + +void CompilerClient::killServers() { + assert(isRunning() && "Can't kill servers, the client isn't running"); +#ifdef MULTI_THREADED_COMPILER_CLIENT + std::cerr << "Waiting for active server requests to end" << std::endl; + threads->stop(true); +#endif + std::cerr << "Killing connected servers" << std::endl; + // Send the request PIR_COMPILE_KILL_MAGIC to all servers, and check the + // acknowledgement (we do this synchronously) + for (size_t i = 0; i < sockets->size(); i++) { + auto& socket = (*sockets)[i]; + // Send the request + START_LOGGING_REQUEST(); + LOG_REQUEST("Request::Kill"); + END_LOGGING_REQUEST(); + auto request = Request::Kill; + socket->send(zmq::message_t(&request, sizeof(request)), + zmq::send_flags::none); + // Check the acknowledgement + zmq::message_t response; + socket->recv(response, zmq::recv_flags::none); + if (response.size() == sizeof(Response::Killed) && + *(Response*)response.data() == Response::Killed) { + START_LOGGING_RESPONSE(); + LOG_RESPONSE("Response::Killed"); + END_LOGGING_RESPONSE(); + } else { + std::cerr << "Error: server " << i << " didn't acknowledge kill request" + << std::endl; + } + } + // Close all sockets + for (auto& socket : *sockets) { + socket->close(); + } + std::fill(socketsConnected->begin(), socketsConnected->end(), false); + // Mark that we've stopped running + _isRunning = false; + std::cerr << "Done killing connected servers, client is no longer running" << std::endl; +} + +#ifdef MULTI_THREADED_COMPILER_CLIENT +const CompiledResponseData& CompilerClient::CompiledHandle::getResponse() { + // Wait for the response, with timeout if set + if (PIR_CLIENT_TIMEOUT == std::chrono::milliseconds(0)) { + response.wait(); + } else { + switch (response.wait_for(PIR_CLIENT_TIMEOUT)) { + case std::future_status::ready: + break; + case std::future_status::timeout: { + LOG_WARN(std::cerr << console::with_red("Timeout waiting for remote PIR") + << std::endl); + // Disconnect because the server probably crashed, and we want + // to be able to restart without restarting the client; it will + // attempt to reconnect before sending the next request + auto socketIndex = *socketIndexRef; + if (socketIndex != -1) { + LOG_WARN(std::cerr << "Disconnecting " << socketIndex + << ", will reconnect on next request" + << std::endl); + auto socket = (*sockets)[socketIndex]; + auto socketAddr = (*serverAddrs)[socketIndex]; + socket->disconnect(socketAddr); + (*socketsConnected)[socketIndex] = false; + } + return; + } + case std::future_status::deferred: + assert(false); + } + } + // Get the response which is ready now + return response.get(); +} +#endif + +static void normalizePir(std::string& pir) { + // Replace addresses with 0xXXXXXXXX, since they will be different + static const std::regex ADDRESS_REGEX("0x[0-9a-fA-F]+"); + static const char* ADDRESS_REPLACE = "0xXXXXXXXX"; + pir = std::regex_replace(pir, ADDRESS_REGEX, ADDRESS_REPLACE); +} + +static void checkDiscrepancy(std::string&& localPir, std::string&& remotePir) { + if (PIR_CLIENT_SKIP_DISCREPANCY_CHECK) { + return; + } + normalizePir(localPir); + normalizePir(remotePir); + // Don't need to log if there's no discrepancy. + if (localPir == remotePir) { + return; + } + std::cerr << console::with_red("Discrepancy between local and remote PIR") + << std::endl; + // Print a fancy line-by-line diff + std::istringstream localPirStream(localPir); + std::istringstream remotePirStream(remotePir); + size_t lineNum = 0; + std::string localLine; + std::string remoteLine; + while (std::getline(localPirStream, localLine) && + std::getline(remotePirStream, remoteLine)) { + if (localLine == remoteLine) { + std::cerr << std::setw(4) << lineNum << localLine << std::endl; + } else { + std::cerr << std::setw(4) << lineNum << console::with_red(localLine) << std::endl; + std::cerr << std::setw(4) << lineNum << console::with_green(remoteLine) << std::endl; + } + lineNum++; + } + while (std::getline(localPirStream, localLine)) { + std::cerr << std::setw(4) << lineNum << console::with_red(localLine) << std::endl; + lineNum++; + } + while (std::getline(remotePirStream, remoteLine)) { + std::cerr << std::setw(4) << lineNum << console::with_green(remoteLine) << std::endl; + lineNum++; + } +} + +void CompilerClient::CompiledHandle::compare(pir::ClosureVersion* version) const { + auto localPir = printClosureVersionForCompilerServerComparison(version); +#ifdef MULTI_THREADED_COMPILER_CLIENT + // Tried using a second thread-pool here but it causes "mutex lock failed: + // Invalid argument" for `response` (and `shared_future` doesn't fix it) + (void)std::async(std::launch::async, [=]() { + auto resp = inner->getResponse(); + auto remotePir = resp.finalPir; + checkDiscrepancy(std::move(localPir), std::move(remotePir)); + }); +#else + auto remotePir = inner->response.finalPir; + checkDiscrepancy(std::move(localPir), std::move(remotePir)); +#endif +} + +SEXP CompilerClient::CompiledHandle::getSexp() const { +#ifdef MULTI_THREADED_COMPILER_CLIENT + auto& response = inner->getResponse(); +#else + const auto& response = inner->response; +#endif + return response.sexp; +} + +const std::string& CompilerClient::CompiledHandle::getFinalPir() const { +#ifdef MULTI_THREADED_COMPILER_CLIENT + auto& response = inner->getResponse(); +#else + const auto& response = inner->response; +#endif + return response.finalPir; +} + +} // namespace rir diff --git a/rir/src/compilerClientServer/CompilerClient.h b/rir/src/compilerClientServer/CompilerClient.h new file mode 100644 index 000000000..0560d6ab5 --- /dev/null +++ b/rir/src/compilerClientServer/CompilerClient.h @@ -0,0 +1,112 @@ +// +// Created by Jakob Hain on 5/25/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "compiler/log/debug.h" +#include "compiler/log/loggers.h" +#include "compiler/log/log.h" +#include "compiler/pir/closure_version.h" +#include "runtime/Context.h" +#include +#include + +class ByteBuffer; + +namespace rir { + +class UUID; + +/** + * Compiler server client. + * On startup, attempts to connect to a compile-server on PIR_CLIENT_ADDR (weird + * naming because we reserve PIR_SERVER_ADDR to be set only for the compiler- + * server) with a zeromq "request" socket. If successful, it will use the server t + * compile RIR to PIR (currently just compares to check for discrepancies). + */ +class CompilerClient { + struct CompiledResponseData { + SEXP sexp; + std::string finalPir; + + CompiledResponseData(SEXP sexp, const std::string&& finalPir) + : sexp(sexp), finalPir(finalPir) { + R_PreserveObject(sexp); + } + + ~CompiledResponseData() { + R_ReleaseObject(sexp); + } + }; + template + class Handle { + friend class CompilerClient; +#ifdef MULTI_THREADED_COMPILER_CLIENT + std::shared_ptr socketIndexRef; + std::future response; + CompiledHandle(const std::shared_ptr& socketIndexRef, + std::future response) + : socketIndexRef(socketIndexRef), response(std::move(response)) {} + /// Block and get the response data + T getResponse() const; +#else + T response; + explicit Handle(T response) : response(std::move(response)) {} +#endif + }; + + static bool _isRunning; + + template + static Handle* request( + const std::function&& makeRequest, + const std::function&& makeResponse); + public: + class CompiledHandle { + friend class CompilerClient; + Handle* inner; + explicit CompiledHandle(Handle* inner) + : inner(inner) {} + public: + ~CompiledHandle() { delete inner; } + + /// When we get response PIR, compares it with given locally-compiled + /// closure PIR and logs any discrepancies. + void compare(pir::ClosureVersion* version) const; + /// Block and get the SEXP + SEXP getSexp() const; + /// Block and get the final PIR debug print + const std::string& getFinalPir() const; + }; + + /// Returns if the client was initialized + static bool isRunning() { return _isRunning; } + + /// Initializes if PIR_CLIENT_ADDR is set + static void tryInit(); + /// "Asynchronously" (not currently, maybe in the future) sends the closure + /// to the compile server and returns a handle to use the result. + /// Then interns the result, + static CompiledHandle* pirCompile(SEXP what, const Context& assumptions, + const std::string& name, + const pir::DebugOptions& debug); + /// Synchronously retrieves the closure with the given hash from the server. + /// If in the future we make this asynchronous, should still return a + /// closure SEXP but make it block while we're waiting for the response. + /// + /// The SEXP is also interned. It must actually be interned before we finish + /// deserializing for recursive retrievals (a -> retrieve b -> retrieve a -> + /// ...). + /// + /// Returns `nullptr` if the server doesn't have the closure. + static SEXP retrieve(const UUID& hash); + + /// Send a message from the compiler client (this) to each connected + /// compiler server, which kills the server (exit 0) on receive. Then stops + /// the client for the remainder of the session + static void killServers(); +}; + +} // namespace rir diff --git a/rir/src/compilerClientServer/CompilerServer.cpp b/rir/src/compilerClientServer/CompilerServer.cpp new file mode 100644 index 000000000..ddce259d5 --- /dev/null +++ b/rir/src/compilerClientServer/CompilerServer.cpp @@ -0,0 +1,534 @@ +// +// Created by Jakob Hain on 5/25/23. +// + +#include "CompilerServer.h" +#include "R/Printing.h" +#include "api.h" +#include "bc/Compiler.h" +#include "compiler_server_client_shared_utils.h" +#include "runtime/PoolStub.h" +#include "serializeHash/hash/UUID.h" +#include "serializeHash/hash/UUIDPool.h" +#include "serializeHash/hash/hashAst.h" +#include "serializeHash/serialize/serialize.h" +#include "utils/ByteBuffer.h" +#include "utils/measuring.h" +#include +#include + +namespace rir { + +#define COMPARE_SOURCE_AND_FEEDBACK_WITH_FULL COMPILER_CLIENT_SEND_SOURCE_AND_FEEDBACK && COMPILER_CLIENT_SEND_FULL + +#define SOFT_ASSERT(x, msg) if (!(x)) \ + LOG_WARN(std::cerr << "Assertion failed (client issue): " << msg \ + << " (" << #x ")" << std::endl) + +#define LOG(stmt) if (pir::Parameter::PIR_TRACE_COMPILER_PEER || pir::Parameter::PIR_LOG_COMPILER_PEER) stmt +#define LOG_WARN(stmt) if (pir::Parameter::PIR_TRACE_COMPILER_PEER || pir::Parameter::PIR_LOG_COMPILER_PEER || pir::Parameter::PIR_WARN_COMPILER_PEER) stmt +#define LOG_DETAILED(stmt) if (pir::Parameter::PIR_TRACE_COMPILER_PEER) stmt +#define START_LOGGING_REQUEST() LOG_DETAILED(do { \ + logDetailedDepth++; \ + logDetailedIndent = std::string(logDetailedDepth * 2, ' '); \ + } while (0)) +#define END_LOGGING_REQUEST() LOG_DETAILED(do { \ + logDetailedDepth--; \ + logDetailedIndent = std::string(logDetailedDepth * 2, ' '); \ + } while (0)) +#define START_LOGGING_RESPONSE() START_LOGGING_REQUEST() +#define END_LOGGING_RESPONSE() END_LOGGING_REQUEST() +#define START_LOGGING_SERVER_REQUEST() START_LOGGING_REQUEST() +#define END_LOGGING_SERVER_REQUEST() END_LOGGING_REQUEST() +#define START_LOGGING_CLIENT_RESPONSE() START_LOGGING_REQUEST() +#define END_LOGGING_CLIENT_RESPONSE() END_LOGGING_REQUEST() +static int logDetailedDepth = 0; +static std::string logDetailedIndent; +// Arrows are different directions than CompilerClient.cpp, since we receive +// requests and send responses, send server requests and receive client +// responses +#define LOG_REQUEST(message) LOG_DETAILED(std::cerr << logDetailedIndent << "<< " << message << std::endl) +#define LOG_RESPONSE(message) LOG_DETAILED(std::cerr << logDetailedIndent << ">> " << message << std::endl) +#define LOG_SERVER_REQUEST(message) LOG_DETAILED(std::cerr << logDetailedIndent << ">>> " << message << std::endl) +#define LOG_CLIENT_RESPONSE(message) LOG_DETAILED(std::cerr << logDetailedIndent << "<<< " << message << std::endl) + +static const char* PROCESSING_REQUEST_TIMER_NAME = "CompilerServer.cpp: processing request (not sending, receiving, compiling, or interning)"; +static const char* SENDING_RESPONSE_TIMER_NAME = "CompilerServer.cpp: sending response"; + +bool CompilerServer::_isRunning = false; +static zmq::socket_t* socket; +static std::unordered_map* memoizedRequests; + +void CompilerServer::tryRun() { + // Get the server address from the environment + const char* serverAddr = getenv("PIR_SERVER_ADDR"); + if (serverAddr) { + std::cerr << "PIR_SERVER_ADDR=" << serverAddr + << ", CompilerServer initializing..." << std::endl; + } else { +#ifdef FORCE_LOG_COMPILER_SERVER + std::cerr << "PIR_SERVER_ADDR not set, CompilerServer won't initialize" << std::endl; +#endif + return; + } + + // Initialize the zmq context + zmq::context_t context( + // Only 1 thread and socket because PIR is currently single-threaded + 1, + 1 + ); + socket = new zmq::socket_t(context, zmq::socket_type::rep); + socket->bind(serverAddr); + + // Initialize memoized requests + memoizedRequests = new std::unordered_map(); + + _isRunning = true; + pir::Parameter::SERIALIZE_LLVM = true; + // _isRunning is used because of nested calls in the for loop, but CLion + // doesn't see + (void)_isRunning; + // Won't return + for (;;) { + LOG(std::cerr << "Waiting for next request..." << std::endl); + // Receive the request + zmq::message_t request; + socket->recv(request, zmq::recv_flags::none); + LOG(std::cerr << "Got request (" << request.size() << " bytes)" << std::endl); + + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, PROCESSING_REQUEST_TIMER_NAME, true); + // Deserialize the request. + // Request data format = + // - Request + // + ... + START_LOGGING_REQUEST(); + ByteBuffer requestBuffer((uint8_t*)request.data(), request.size()); + auto magic = (Request)requestBuffer.getLong(); + + // Handle Kill, Retrieved, and RetrieveFailed (not memoized) or Memoize + switch (magic) { + case Request::Kill: { + std::cerr << "Received kill request" << std::endl; + LOG_REQUEST("Request::Kill"); + // ... (end of request) + END_LOGGING_REQUEST(); + // Send Response::Killed + START_LOGGING_RESPONSE(); + LOG_RESPONSE("Response::Killed"); + END_LOGGING_RESPONSE(); + auto response = Response::Killed; + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, PROCESSING_REQUEST_TIMER_NAME, true); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + socket->send(zmq::message_t(&response, sizeof(response)), + zmq::send_flags::none); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + LOG(std::cerr << "Sent kill acknowledgement, will die" << std::endl); + _isRunning = false; + exit(0); + } + case Request::Retrieved: + case Request::RetrieveFailed: + LOG_REQUEST("Request::Retrieved | Request::RetrieveFailed"); + END_LOGGING_REQUEST(); + LOG_WARN(std::cerr << "Unexpected client-side response (" << (uint64_t)magic + << ") server shouldn't have or didn't send a request. " + << "Ignoring" << std::endl); + continue; + case Request::Memoize: { + LOG_REQUEST("Request::Memoize"); + // ... + // + UUID hash + UUID hash; + requestBuffer.getBytes((uint8_t*)&hash, sizeof(UUID)); + LOG_REQUEST("hash = " << hash); + END_LOGGING_REQUEST(); + START_LOGGING_RESPONSE(); + if (memoizedRequests->count(hash)) { + LOG(std::cerr << "Found memoized result for hash (hash-only) " + << hash << std::endl); + // Send the response (memoized) + auto result = (*memoizedRequests)[hash]; + LOG_RESPONSE("(memoized full response)"); + END_LOGGING_RESPONSE(); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, PROCESSING_REQUEST_TIMER_NAME, true); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + CHECK_MSG_NOT_TOO_LARGE(result.size()); + socket->send(zmq::message_t(result.data(), result.size()), + zmq::send_flags::none); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + LOG(std::cerr << "Sent memoized result for hash (hash-only) " + << hash << std::endl); + } else { + LOG(std::cerr << "No memoized result for hash (hash-only) " << hash + << std::endl); + // Send Response::NeedsFull + auto response = Response::NeedsFull; + LOG_RESPONSE("Response::NeedsFull"); + END_LOGGING_RESPONSE(); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, PROCESSING_REQUEST_TIMER_NAME, true); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + socket->send(zmq::message_t(&response, sizeof(response)), + zmq::send_flags::none); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + LOG(std::cerr << "Sent request full for hash (hash-only) " << hash + << std::endl); + } + continue; + } + default: + break; + } + + // Handle if we memoized + UUID requestHash = UUID::hash(request.data(), request.size()); + if (memoizedRequests->count(requestHash)) { + END_LOGGING_REQUEST(); + LOG(std::cerr << "Found memoized result for hash " << requestHash + << std::endl); + // Send the response (memoized) + auto result = (*memoizedRequests)[requestHash]; + START_LOGGING_RESPONSE(); + LOG_RESPONSE("(memoized full response)"); + END_LOGGING_RESPONSE(); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, PROCESSING_REQUEST_TIMER_NAME, true); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + CHECK_MSG_NOT_TOO_LARGE(result.size()); + socket->send(zmq::message_t( + result.data(), + result.size()), + zmq::send_flags::none); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + LOG(std::cerr << "Sent memoized result for hash " << requestHash + << std::endl); + continue; + } else { + LOG(std::cerr << "No memoized result for hash " << requestHash + << std::endl); + } + + // Handle other request types + SEXP what; + ByteBuffer response; + switch (magic) { + case Request::Compile: { + LOG(std::cerr << "Received compile request" << std::endl); + LOG_REQUEST("Request::Compile"); + // ... +#if COMPILER_CLIENT_SEND_SOURCE_AND_FEEDBACK + // + bool PIR_CLIENT_INTERN + // + serialize(Compiler::decompileClosure(what), CompilerClient(...)) + // + DispatchTable::unpack(BODY(what))->baseline()->fullSignature() + // + serialize(DispatchTable::unpack(BODY(what))->baseline()->typeFeedback()->container(), CompilerClient(...)) + // + (uintptr_t)DispatchTable::unpack(BODY(what))->baseline()->body() + // + DispatchTable::unpack(BODY(what))->baseline()->body()->extraPoolSize +#endif +#if COMPILER_CLIENT_SEND_FULL + // + serialize(what, SourceAndFeedback) +#endif + // + sizeof(assumptions) (always 8) + // + assumptions + // + sizeof(name) + // + name + // + sizeof(debug.flags) (always 4) + // + debug.flags + // + sizeof(debug.passFilterString) + // + debug.passFilterString + // + sizeof(debug.functionFilterString) + // + debug.functionFilterString + // + sizeof(debug.style) (always 4) + // + debug.style + + // Client won't send hashed SEXPs because it doesn't necessarily + // remember them, and because the server doesn't care about + // connected SEXPs like the client. However, client will send hashed + // record_call_ SEXPs, because those are very large and we can + // handle the case where they are forgotten by just not speculating + // on them. +#if COMPILER_CLIENT_SEND_SOURCE_AND_FEEDBACK + auto intern = requestBuffer.getBool(); + LOG_REQUEST("PIR_CLIENT_INTERN = " << intern); + what = deserialize(requestBuffer, SerialOptions::CompilerServer(intern)); + SOFT_ASSERT(TYPEOF(what) == CLOSXP, + "deserialized source closure to compile isn't actually a closure"); + PROTECT(what); + LOG_REQUEST("serialize(" << Print::dumpSexp(what) << ", CompilerClient(...))"); + auto sourceHash = hashDecompiled(what); + Compiler::compileClosure(what); + DispatchTable::unpack(BODY(what))->baseline()->deserializeFullSignature(requestBuffer); + LOG_REQUEST("full signature"); + auto feedback = deserialize(requestBuffer, SerialOptions::CompilerServer(intern)); + SOFT_ASSERT(TypeFeedback::check(feedback), + "deserialized type feedback isn't actually type feedback"); + DispatchTable::unpack(BODY(what))->baseline()->typeFeedback(TypeFeedback::unpack(feedback)); + auto sourceBodyPoolSize = requestBuffer.getInt(); + std::vector sourceDefaultArgPoolSizes(requestBuffer.getInt(), 0); + for (auto& sourceDefaultArgPoolSize : sourceDefaultArgPoolSizes) { + sourceDefaultArgPoolSize = requestBuffer.getInt(); + } + PoolStub::pad(sourceHash, sourceBodyPoolSize, + sourceDefaultArgPoolSizes, + DispatchTable::unpack(BODY(what))->baseline()); + UNPROTECT(1); +#endif +#if COMPARE_SOURCE_AND_FEEDBACK_WITH_FULL + auto what2 = what; + PROTECT(what2); +#endif +#if COMPILER_CLIENT_SEND_FULL + what = deserialize(requestBuffer, SerialOptions::SourceAndFeedback); + SOFT_ASSERT(TYPEOF(what) == CLOSXP && DispatchTable::check(BODY(what)), + "deserialized rir closure to compile isn't actually a rir closure"); + LOG_REQUEST("serialize(" << Print::dumpSexp(what) << ", SourceAndFeedback)"); +#endif +#if COMPARE_SOURCE_AND_FEEDBACK_WITH_FULL + PROTECT(what); + std::stringstream differencesStream; + DispatchTable::debugCompare( + DispatchTable::unpack(BODY(what)), + DispatchTable::unpack(BODY(what2)), + differencesStream, + false + ); + auto differences = differencesStream.str(); + if (!differences.empty()) { + LOG_WARN(std::cerr << "Differences when we encode code via AST " + "and bytecode without recorded calls:" + << std::endl << differences << std::endl); + } + + // No longer need to protect what, and what2 is no longer used + UNPROTECT(2); +#endif + auto assumptionsSize = requestBuffer.getLong(); + SOFT_ASSERT(assumptionsSize == sizeof(Context), + "Invalid assumptions size"); + Context assumptions; + requestBuffer.getBytes((uint8_t*)&assumptions, assumptionsSize); + LOG_REQUEST("assumptions = " << assumptions); + auto nameSize = requestBuffer.getLong(); + std::string name; + name.resize(nameSize); + requestBuffer.getBytes((uint8_t*)name.data(), nameSize); + LOG_REQUEST("name = " << name); + auto debugFlagsSize = requestBuffer.getLong(); + SOFT_ASSERT(debugFlagsSize == sizeof(pir::DebugOptions::DebugFlags), + "Invalid debug flags size"); + pir::DebugOptions::DebugFlags debugFlags; + requestBuffer.getBytes((uint8_t*)&debugFlags, debugFlagsSize); + auto passFilterStringSize = requestBuffer.getLong(); + std::string passFilterString; + passFilterString.resize(passFilterStringSize); + requestBuffer.getBytes((uint8_t*)passFilterString.data(), + passFilterStringSize); + auto functionFilterStringSize = requestBuffer.getLong(); + std::string functionFilterString; + functionFilterString.resize(functionFilterStringSize); + requestBuffer.getBytes((uint8_t*)functionFilterString.data(), + functionFilterStringSize); + auto debugStyleSize = requestBuffer.getLong(); + SOFT_ASSERT(debugStyleSize == sizeof(pir::DebugStyle), + "Invalid debug style size"); + pir::DebugStyle debugStyle; + requestBuffer.getBytes((uint8_t*)&debugStyle, debugStyleSize); + pir::DebugOptions debug(debugFlags, passFilterString, + functionFilterString, debugStyle); + LOG_REQUEST("debug = " << debug); + END_LOGGING_REQUEST(); + + // It's a bit confusing that debug options are passed from the + // client. We may want this to be the case, but we also want server + // debug options; the current solution is to merge them and take + // whatever's overridden from either. + debug.flags = debug.flags | pir::DebugOptions::DefaultDebugOptions.flags; + if (pir::DebugOptions::DefaultDebugOptions.passFilterString != ".*") { + debug.passFilterString = pir::DebugOptions::DefaultDebugOptions.passFilterString; + debug.passFilter = pir::DebugOptions::DefaultDebugOptions.passFilter; + } + if (pir::DebugOptions::DefaultDebugOptions.functionFilterString != ".*") { + debug.functionFilterString = pir::DebugOptions::DefaultDebugOptions.functionFilterString; + debug.functionFilter = pir::DebugOptions::DefaultDebugOptions.functionFilter; + } + if (pir::DebugOptions::DefaultDebugOptions.style != pir::DebugStyle::Standard) { + debug.style = pir::DebugOptions::DefaultDebugOptions.style; + } + + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, PROCESSING_REQUEST_TIMER_NAME, true); + std::string pirPrint; + what = pirCompile(what, assumptions, name, debug, &pirPrint, SerialOptions::CompilerServer(intern)); + + // Intern, not because we'll have reused it (highly unlikely since + // we memoize requests, and it doesn't affect anything anyways), but + // because we want to store it in the UUID pool for Retrieve requests + // (since we memoize requests) so that compiler client can retrieve + // it later + // UUIDPool::intern(what, true, true); + + // Serialize the response + // Response data format = + // Response::Compiled + // + sizeof(pirPrint) + // + pirPrint + // + serialize(what, CompilerClient(...)) + START_LOGGING_RESPONSE(); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, PROCESSING_REQUEST_TIMER_NAME, true); + LOG_RESPONSE("Response::Compiled"); + response.putLong((uint64_t)Response::Compiled); + LOG_RESPONSE("pirPrint = (size = " << pirPrint.size() << ")"); + auto pirPrintSize = pirPrint.size(); + response.putLong(pirPrintSize); + response.putBytes((uint8_t*)pirPrint.data(), pirPrintSize); + // TODO: Send only the closure body (since formals and environment + // are redundant), but first send the body's hash so we can reuse + // and skip deserialization if possible (see commit tagged + // cant-send-compiled-hash) + LOG_RESPONSE("serialize(" << Print::dumpSexp(what) << ", CompilerClient(...))"); + serialize(what, response, SerialOptions::CompilerServer(intern)); + END_LOGGING_RESPONSE(); + break; + } + case Request::Retrieve: { + LOG(std::cerr << "Received retrieve request" << std::endl); + LOG_REQUEST("Request::Retrieve"); + // ... + // + bool PIR_CLIENT_INTERN + // + UUID hash + auto intern = requestBuffer.getBool(); + LOG_REQUEST("PIR_CLIENT_INTERN = " << intern); + UUID hash; + requestBuffer.getBytes((uint8_t*)&hash, sizeof(UUID)); + LOG_REQUEST("hash = " << hash); + END_LOGGING_REQUEST(); + + // Get SEXP + what = UUIDPool::get(hash); + + // Serialize the response + LOG(std::cerr << "Retrieve " << hash << " = "); + START_LOGGING_RESPONSE(); + if (what) { + LOG(std::cerr << what << " " << Print::dumpSexp(what) << std::endl); + + // Response data format = + // Response::Retrieved + // + serialize(what, CompilerServer) + LOG_RESPONSE("Response::Retrieved"); + response.putLong((uint64_t)Response::Retrieved); + LOG_RESPONSE("serialize(" << Print::dumpSexp(what) << ", CompilerServer)"); + serialize(what, response, SerialOptions::CompilerServer(intern)); + } else { + LOG(std::cerr << "(not found)" << std::endl); + // Response data format = + // Response::RetrieveFailed + LOG_RESPONSE("Response::RetrieveFailed"); + response.putLong((uint64_t)Response::RetrieveFailed); + } + END_LOGGING_RESPONSE(); + break; + } + case Request::Kill: + case Request::Memoize: + case Request::Retrieved: + case Request::RetrieveFailed: + assert(false); + default: + LOG_WARN(std::cerr << "Unhandled magic: " << (uint64_t)magic + << std::endl); + continue; + } + + // Memoize the response + (*memoizedRequests)[requestHash] = response; + + // Send the response; + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, PROCESSING_REQUEST_TIMER_NAME, true); + Measuring::startTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + size_t responseSize; + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER && what, "CompilerServer.cpp: sending new response with SEXP", what, [&]{ + CHECK_MSG_NOT_TOO_LARGE(response.size()); + responseSize = *socket->send(zmq::message_t{ + response.data(), + response.size()}, + zmq::send_flags::none); + }); + auto responseSize2 = response.size(); + SOFT_ASSERT(responseSize == responseSize2, + "Client didn't receive the full response"); + Measuring::countTimerIf(pir::Parameter::PIR_MEASURE_CLIENT_SERVER, SENDING_RESPONSE_TIMER_NAME, true); + + LOG(std::cerr << "Sent response (" << responseSize << " bytes)" + << std::endl); + } +} + +SEXP CompilerServer::retrieve(const rir::UUID& hash) { + LOG(std::cerr << "Retrieving from client " << hash << std::endl); + // Build the server-side request + // Data format = + // Response::NeedsRetrieve + // + UUID hash + START_LOGGING_SERVER_REQUEST(); + ByteBuffer serverRequest; + LOG_SERVER_REQUEST("Response::NeedsRetrieve"); + serverRequest.putLong((uint64_t)Response::NeedsRetrieve); + LOG_SERVER_REQUEST("hash = " << hash); + serverRequest.putBytes((uint8_t*)&hash, sizeof(UUID)); + END_LOGGING_SERVER_REQUEST(); + + // Send the server-side request + auto serverRequestSize = serverRequest.size(); + CHECK_MSG_NOT_TOO_LARGE(serverRequest.size()); + auto serverRequestSize2 = *socket->send(zmq::message_t( + serverRequest.data(), + serverRequest.size()), + zmq::send_flags::none); + SOFT_ASSERT(serverRequestSize == serverRequestSize2, + "Client didn't receive the full request"); + + // Receive the client-side response + zmq::message_t clientResponse; + socket->recv(clientResponse, zmq::recv_flags::none); + LOG(std::cerr << "Got client-side response (" << clientResponse.size() + << " bytes)" << std::endl); + + // Deserialize the client-side response + // Data format = + // - Response + // + ... + START_LOGGING_CLIENT_RESPONSE(); + ByteBuffer clientResponseBuffer((uint8_t*)clientResponse.data(), clientResponse.size()); + auto magic = (Request)clientResponseBuffer.getLong(); + switch (magic) { + case Request::Retrieved: { + LOG_CLIENT_RESPONSE("Request::Retrieved"); + // ... + // + serialize(what, CompilerClientRetrieve) + SEXP what = deserialize(clientResponseBuffer, + SerialOptions::CompilerClientRetrieve, hash); + PROTECT(what); + LOG_CLIENT_RESPONSE("serialize(" << Print::dumpSexp(what) << ", CompilerClientRetrieve)"); + // We've already recursively interned and preserved (deserialize with + // useHashes causes children to be interned, and retrieveHash causes + // `what` itself to be interned. Both have preserve=true because they + // are explicitly coded to do that when the compiler server is running) + UNPROTECT(1); + END_LOGGING_CLIENT_RESPONSE(); + return what; + } + case Request::RetrieveFailed: + // ... + // (no data) + LOG_CLIENT_RESPONSE("Request::RetrieveFailed"); + LOG(std::cerr << "Client doesn't have the SEXP" << std::endl); + END_LOGGING_CLIENT_RESPONSE(); + return nullptr; + default: + LOG_WARN(std::cerr << "Unexpected client request or client-side " + "response (" << (uint64_t)magic << "). Ignoring" + << std::endl); + END_LOGGING_CLIENT_RESPONSE(); + return nullptr; + } +} + +} // namespace rir diff --git a/rir/src/compilerClientServer/CompilerServer.h b/rir/src/compilerClientServer/CompilerServer.h new file mode 100644 index 000000000..87f06697a --- /dev/null +++ b/rir/src/compilerClientServer/CompilerServer.h @@ -0,0 +1,44 @@ +// +// Created by Jakob Hain on 5/25/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "compiler/log/debug.h" +#include "compiler/log/loggers.h" +#include "compiler/log/log.h" +#include "compiler/pir/closure_version.h" +#include "runtime/Context.h" +#include + +namespace rir { + +/** + * Compiler server. + * On startup, attempts to bind to PIR_SERVER_ADDR with a zeromq "reply" socket. + * If successful, it will wait for incoming "compile" requests and process them + * by calling pirCompile. + */ +class CompilerServer { + static bool _isRunning; + + public: + /// Is this Ř instance a compiler server? + static bool isRunning() { return _isRunning; } + /// If PIR_SERVER_ADDR is set, initializes and starts handling requests + static void tryRun(); + + /// Synchronously retrieves the closure with the given hash from the client. + /// If in the future we make this asynchronous, should still return a + /// closure SEXP but make it block while we're waiting for the response. + /// + /// The SEXP is also interned. It must actually be interned before we finish + /// deserializing for recursive retrievals (a -> retrieve b -> retrieve a -> + /// ...). + /// + /// Returns `nullptr` if the client doesn't have the closure. + static SEXP retrieve(const UUID& hash); +}; + +} // namespace rir \ No newline at end of file diff --git a/rir/src/compilerClientServer/compiler_server_client_shared_utils.cpp b/rir/src/compilerClientServer/compiler_server_client_shared_utils.cpp new file mode 100644 index 000000000..129c6ffcf --- /dev/null +++ b/rir/src/compilerClientServer/compiler_server_client_shared_utils.cpp @@ -0,0 +1,54 @@ +// +// Created by Jakob Hain on 5/25/23. +// + +#include "compiler_server_client_shared_utils.h" +#include "compiler/log/debug.h" +#include "zmq.h" + +namespace rir { + +bool PIR_CLIENT_DRY_RUN = + getenv("PIR_CLIENT_DRY_RUN") != nullptr && + strcmp(getenv("PIR_CLIENT_DRY_RUN"), "") != 0 && + strcmp(getenv("PIR_CLIENT_DRY_RUN"), "0") != 0 && + strcmp(getenv("PIR_CLIENT_DRY_RUN"), "false") != 0; + +size_t PIR_CLIENT_COMPILE_SIZE_TO_HASH_ONLY = + getenv("PIR_CLIENT_COMPILE_SIZE_TO_HASH_ONLY") + ? strtol(getenv("PIR_CLIENT_COMPILE_SIZE_TO_HASH_ONLY"), nullptr, 10) + : 1024 * 1024; + +bool pir::Parameter::PIR_TRACE_COMPILER_PEER = + getenv("PIR_TRACE_COMPILER_PEER") != nullptr && + strcmp(getenv("PIR_TRACE_COMPILER_PEER"), "") != 0 && + strcmp(getenv("PIR_TRACE_COMPILER_PEER"), "0") != 0; + +bool pir::Parameter::PIR_LOG_COMPILER_PEER = + getenv("PIR_LOG_COMPILER_PEER") != nullptr && + strcmp(getenv("PIR_LOG_COMPILER_PEER"), "") != 0 && + strcmp(getenv("PIR_LOG_COMPILER_PEER"), "0") != 0; + +bool pir::Parameter::PIR_WARN_COMPILER_PEER = + getenv("PIR_WARN_COMPILER_PEER") != nullptr && + strcmp(getenv("PIR_WARN_COMPILER_PEER"), "") != 0 && + strcmp(getenv("PIR_WARN_COMPILER_PEER"), "0") != 0; + + +bool pir::Parameter::PIR_MEASURE_CLIENT_SERVER = + getenv("PIR_MEASURE_CLIENT_SERVER") != nullptr && + strtol(getenv("PIR_MEASURE_CLIENT_SERVER"), nullptr, 10); + +std::string printClosureVersionForCompilerServerComparison(pir::ClosureVersion* version) { + std::stringstream pir; + version->print(pir::DebugStyle::Standard, pir, false, false); + return pir.str(); +} + +__attribute__((unused)) NORET void zeromq_error(const char* func) { + printCBacktrace(); + std::cerr << "zeromq error in " << func << ": " << zmq_strerror(zmq_errno()) << std::endl; + std::abort(); +} + +} // namespace rir diff --git a/rir/src/compilerClientServer/compiler_server_client_shared_utils.h b/rir/src/compilerClientServer/compiler_server_client_shared_utils.h new file mode 100644 index 000000000..b0d679640 --- /dev/null +++ b/rir/src/compilerClientServer/compiler_server_client_shared_utils.h @@ -0,0 +1,67 @@ +// +// Created by Jakob Hain on 5/25/23. +// + +#include "compiler/pir/closure_version.h" +#include "compiler/parameter.h" +#include + +#pragma once + +#define COMPILER_CLIENT_SEND_SOURCE_AND_FEEDBACK 1 +#define COMPILER_CLIENT_SEND_FULL 0 + +#define CHECK_MSG_NOT_TOO_LARGE(size) do { \ + if (size > (unsigned)INT_MAX) { \ + std::cerr << "Message too large for zeromq: " << #size << "=" \ + << size << " (> INT_MAX)" << std::endl; \ + assert(false); \ + } \ + } while (0) + +namespace rir { + +enum class Request : uint64_t { + /// For large requests, we send the hash. If the server already received + /// the same request it will serve the cached response. Otherwise it will + /// send `Response::NeedsFull` + Memoize = 0x217A25432A462D4B, + /// Compile a function with assumptions and debug options + Compile = 0x217A25432A462D4A, + /// Retrieve an SEXP on the server referenced by an SEXP on the client + Retrieve = 0x217A25432A462D4D, + /// Kill the server + Kill = 0x217A25432A462D4C, + /// Retrieved SEXP + Retrieved = 0x217A25432A462D4E, + /// SEXP isn't in client + RetrieveFailed = 0x217A25432A462D4F, +}; + +enum class Response : uint64_t { + /// Memoized request - needs the full response + NeedsFull = 0x9BEEB1E5356F1A37, + /// Compiled closure + Compiled = 0x9BEEB1E5356F1A36, + /// Retrieved SEXP + Retrieved = 0x9BEEB1E5356F1A3D, + /// SEXP isn't in server + RetrieveFailed = 0x9BEEB1E5356F1A3E, + /// Acknowledge that the server has been killed + Killed = 0x9BEEB1E5356F1A38, + /// Retrieve an SEXP on the client referenced by an SEXP on the server + NeedsRetrieve = 0x9BEEB1E5356F1A3C, +}; + +/// If set, we still compile on the client and only compare the compiler server +/// and client results, instead of replacing the SEXP with the compiled version. +extern bool PIR_CLIENT_DRY_RUN; +extern size_t PIR_CLIENT_COMPILE_SIZE_TO_HASH_ONLY; + +std::string printClosureVersionForCompilerServerComparison(pir::ClosureVersion* version); + +/// Alternative to `throw error_t()` in zeromq, since we don't allow exceptions. +/// Used by external/zeromq so it may be unused before setup. +__attribute__((unused)) NORET void zeromq_error(const char* func); + +} // namespace rir diff --git a/rir/src/interpreter/builtins.cpp b/rir/src/interpreter/builtins.cpp index f646d1a77..26e9bf52d 100644 --- a/rir/src/interpreter/builtins.cpp +++ b/rir/src/interpreter/builtins.cpp @@ -945,8 +945,10 @@ SEXP tryFastBuiltinCall1(const CallContext& call, size_t nargs, bool hasAttrib, case blt("islistfactor"): { if (nargs != 2) return nullptr; - auto n = XLENGTH(args[0]); - if (n == 0 || !Rf_isVectorList(args[0])) + if (!Rf_isVectorList(args[0])) + return R_FalseValue; + auto n = LENGTH(args[0]); + if (n == 0) return R_FalseValue; int recursive = Rf_asLogical(args[1]); if (recursive) diff --git a/rir/src/interpreter/instance.cpp b/rir/src/interpreter/instance.cpp index 59ea2365a..7b5137c79 100644 --- a/rir/src/interpreter/instance.cpp +++ b/rir/src/interpreter/instance.cpp @@ -4,6 +4,10 @@ namespace rir { +#ifdef DO_INTERN +static std::unordered_map src_pool_interned; +#endif + void initializeResizeableList(ResizeableList* l, size_t capacity, SEXP parent, size_t index) { l->capacity = capacity; diff --git a/rir/src/interpreter/interp.cpp b/rir/src/interpreter/interp.cpp index 7e25478d7..cd7f02768 100644 --- a/rir/src/interpreter/interp.cpp +++ b/rir/src/interpreter/interp.cpp @@ -9,11 +9,14 @@ #include "compiler/osr.h" #include "compiler/parameter.h" #include "compiler/pir/continuation_context.h" +#include "compilerClientServer/CompilerClient.h" #include "runtime/Deoptimization.h" #include "runtime/LazyArglist.h" #include "runtime/LazyEnvironment.h" #include "runtime/TypeFeedback_inl.h" #include "safe_force.h" +#include "serializeHash/serialize/serialize.h" +#include "serializeHash/serialize/serializeR.h" #include "utils/Pool.h" #include "utils/measuring.h" @@ -30,10 +33,15 @@ extern Rboolean R_Visible; namespace rir { +bool INTERPRETER_IS_ACTIVE = true; + static SEXP evalRirCode(Code* c, SEXP env, const CallContext* callContext, Opcode* initialPc = nullptr, BindingCache* cache = nullptr); +#define COMPARE_SERIALIZATION_DIFFERENCES 0 +#define COMPARE_SERIALIZATION_DIFFERENCES_DETAILED 0 + // #define PRINT_INTERP // #define PRINT_STACK_SIZE 10 #ifdef PRINT_INTERP @@ -823,6 +831,8 @@ const unsigned pir::Parameter::PIR_REOPT_TIME = getenv("PIR_REOPT_TIME") ? atoi(getenv("PIR_REOPT_TIME")) : 5e7; const unsigned pir::Parameter::DEOPT_ABANDON = getenv("PIR_DEOPT_ABANDON") ? atoi(getenv("PIR_DEOPT_ABANDON")) : 12; +const unsigned pir::Parameter::PIR_OPT_BC_SIZE = + getenv("PIR_OPT_BC_SIZE") ? atoi(getenv("PIR_OPT_BC_SIZE")) : 20; static unsigned serializeCounter = 0; @@ -970,7 +980,99 @@ SEXP doCall(CallContext& call, bool popArgs) { if (pir::Parameter::RIR_SERIALIZE_CHAOS) { serializeCounter++; if (serializeCounter == pir::Parameter::RIR_SERIALIZE_CHAOS) { - body = copyBySerial(body); + auto body0 = body; + PROTECT(body0); + auto body1 = copyBySerial(body); + PROTECT(body1); + auto body2 = copyBySerialR(body); + PROTECT(body2); +#if COMPARE_SERIALIZATION_DIFFERENCES_DETAILED + auto body3 = copyBySerialR(body1); + PROTECT(body3); + auto body4 = copyBySerial(body2); + PROTECT(body4); +#endif + body = body1; + // TODO: Disabling this for now, but there's an issue where + // function invocation times and flags are different from body0 + // to body1 and body2. With the old R serialization algorithm + // they body2's were identical to body0, so it's weird... +#if COMPARE_SERIALIZATION_DIFFERENCES || COMPARE_SERIALIZATION_DIFFERENCES_DETAILED + disableInterpreter([&]{ + std::stringstream differencesStream; + DispatchTable::debugCompare( + DispatchTable::unpack(body0), + DispatchTable::unpack(body1), + differencesStream + ); + auto differences = differencesStream.str(); + if (!differences.empty()) { + std::cout << "WARNING: Serialization differences between 0 and 1:\n" + << differences << "\n"; + } + differencesStream = std::stringstream(); + DispatchTable::debugCompare( + DispatchTable::unpack(body1), + DispatchTable::unpack(body2), + differencesStream + ); + differences = differencesStream.str(); + if (!differences.empty()) { + std::cout << "WARNING: Serialization differences between 1 and 2:\n" + << differences << "\n"; + } +#if COMPARE_SERIALIZATION_DIFFERENCES_DETAILED + differencesStream = std::stringstream(); + DispatchTable::debugCompare( + DispatchTable::unpack(body2), + DispatchTable::unpack(body3), + differencesStream + ); + differences = differencesStream.str(); + if (!differences.empty()) { + std::cout << "WARNING: Serialization differences between 2 and 3:\n" + << differences << "\n"; + } + differencesStream = std::stringstream(); + DispatchTable::debugCompare( + DispatchTable::unpack(body3), + DispatchTable::unpack(body4), + differencesStream + ); + differences = differencesStream.str(); + if (!differences.empty()) { + std::cout << "WARNING: Serialization differences between 3 and 4:\n" + << differences << "\n"; + } + differencesStream = std::stringstream(); + DispatchTable::debugCompare( + DispatchTable::unpack(body4), + DispatchTable::unpack(body1), + differencesStream + ); + differences = differencesStream.str(); + if (!differences.empty()) { + std::cout << "WARNING: Serialization differences between 4 and 1:\n" + << differences << "\n"; + } + differencesStream = std::stringstream(); + DispatchTable::debugCompare( + DispatchTable::unpack(body1), + DispatchTable::unpack(body1), + differencesStream + ); + differences = differencesStream.str(); + if (!differences.empty()) { + std::cout << "!!! WARNING: Serialization differences between 1 and 1:\n" + << differences << "\n"; + } +#endif + }); +#endif + UNPROTECT(3); +#if COMPARE_SERIALIZATION_DIFFERENCES_DETAILED + UNPROTECT(2); +#endif serializeCounter = 0; } PROTECT(body); @@ -1001,9 +1103,21 @@ SEXP doCall(CallContext& call, bool popArgs) { call.caller->function()->invocationCount() > 0 && !call.caller->isCompiled() && !call.caller->function()->disabled() && - call.caller->size() < pir::Parameter::MAX_INPUT_SIZE && - fun->body()->codeSize < 20) { - call.triggerOsr = true; + call.caller->size() < pir::Parameter::MAX_INPUT_SIZE) { + if (fun->body()->codeSize < + pir::Parameter::PIR_OPT_BC_SIZE) { + // std::cerr << "***** CODE SIZE " + // << fun->body()->codeSize << " < " + // << pir::Parameter::PIR_OPT_BC_SIZE + // << "\n"; + call.triggerOsr = true; + } else { + // std::cerr + // << "!!!!! CODE SIZE " << + // fun->body()->codeSize + // << " >= " << pir::Parameter::PIR_OPT_BC_SIZE + // << "\n"; + } } DoRecompile(fun, call.ast, call.callee, given); fun = dispatch(call, table); @@ -1013,7 +1127,7 @@ SEXP doCall(CallContext& call, bool popArgs) { bool needsEnv = fun->signature().envCreation == FunctionSignature::Environment::CallerProvided; - if (fun->flags.contains(Function::DepromiseArgs)) { + if (fun->flags().contains(Function::DepromiseArgs)) { // Force arguments and depromise call.depromiseArgs(); } @@ -1767,7 +1881,7 @@ bool isColonFastcase(SEXP lhs, SEXP rhs) { // TODO(o): // I don't like this part of the condition. It prevents us from constant // folding the colonEffects instruction. Can we do this differently? - if (XLENGTH(lhs) == 0 || XLENGTH(rhs) == 0) + if (RAW_LENGTH(lhs) == 0 || RAW_LENGTH(rhs) == 0) return true; switch (TYPEOF(lhs)) { @@ -1881,6 +1995,8 @@ SEXP colonCastRhs(SEXP newLhs, SEXP rhs) { bool pir::Parameter::ENABLE_OSR = !getenv("PIR_OSR") || *getenv("PIR_OSR") != '0'; +bool pir::Parameter::FORCE_ENABLE_OSR = + getenv("PIR_OSR") && *getenv("PIR_OSR") == '1'; static size_t osrLimit = getenv("PIR_OSR_LIMIT") ? std::atoi(getenv("PIR_OSR_LIMIT")) : 5000; static SEXP osr(const CallContext* callCtxt, R_bcstack_t* basePtr, SEXP env, @@ -1892,13 +2008,13 @@ static SEXP osr(const CallContext* callCtxt, R_bcstack_t* basePtr, SEXP env, auto l = Rf_length(FRAME(env)); auto dt = DispatchTable::check(BODY(callCtxt->callee)); if (dt && - !dt->baseline()->flags.includes(Function::Flag::NotOptimizable) && + !dt->baseline()->flags().includes(Function::Flag::NotOptimizable) && size <= (long)pir::ContinuationContext::MAX_STACK && l <= (long)pir::ContinuationContext::MAX_ENV) { pir::ContinuationContext ctx(pc, env, true, basePtr, size); if (auto fun = pir::OSR::compile(callCtxt->callee, c, ctx)) { PROTECT(fun->container()); - dt->baseline()->flags.set(Function::Flag::MarkOpt); + dt->baseline()->setFlag(Function::Flag::MarkOpt); auto code = fun->body(); auto nc = code->nativeCode(); auto res = nc(code, basePtr, env, callCtxt->callee); @@ -1915,6 +2031,10 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, Opcode* initialPC, BindingCache* cache) { assert(env != symbol::delayedEnv || (callCtxt != nullptr)); + if (!INTERPRETER_IS_ACTIVE) { + std::cerr << "TODO: Interpreting code during serialization or comparison\n"; + } + checkUserInterrupt(); auto native = c->nativeCode(); assert((!initialPC || !native) && "Cannot jump into native code"); @@ -1990,17 +2110,22 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, } else { // This is a lazy loading stub, it replaces the promise with the // actual value. From now on it will be a value... - if (CAR(PREXPR(s)) == symbol::lazyLoadDBfetch) + if ((s)->u.listsxp.carval == symbol::lazyLoadDBfetch) state = ObservedValues::StateBeforeLastForce::value; else state = ObservedValues::StateBeforeLastForce::promise; } - ObservedValues* feedback = (ObservedValues*)(pc + 1); - if (feedback->stateBeforeLastForce < state) - feedback->stateBeforeLastForce = state; + ObservedValues& feedback = + c->function()->typeFeedback()->types(*(Immediate*)(pc + 1)); + if (feedback.stateBeforeLastForce < state) + feedback.stateBeforeLastForce = state; }; + // TODO: move above + auto function = c->function(); + auto typeFeedback = function->typeFeedback(); + // main loop BEGIN_MACHINE { @@ -2306,26 +2431,26 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, } INSTRUCTION(record_call_) { - ObservedCallees* feedback = (ObservedCallees*)pc; + Immediate idx = readImmediate(); + advanceImmediate(); SEXP callee = ostack_top(); - feedback->record(c, callee); - pc += sizeof(ObservedCallees); + typeFeedback->callees(idx).record(function, callee); NEXT(); } INSTRUCTION(record_test_) { - ObservedTest* feedback = (ObservedTest*)pc; + Immediate idx = readImmediate(); + advanceImmediate(); SEXP t = ostack_top(); - feedback->record(t); - pc += sizeof(ObservedTest); + typeFeedback->test(idx).record(t); NEXT(); } INSTRUCTION(record_type_) { - ObservedValues* feedback = (ObservedValues*)pc; + Immediate idx = readImmediate(); + advanceImmediate(); SEXP t = ostack_top(); - feedback->record(t); - pc += sizeof(ObservedValues); + typeFeedback->types(idx).record(t); NEXT(); } @@ -3052,13 +3177,13 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, INSTRUCTION(asbool_) { SEXP val = ostack_top(); int cond = NA_LOGICAL; - if (XLENGTH(val) > 1) + if (RAW_LENGTH(val) > 1) Rf_warningcall( getSrcAt(c, pc - 1), "the condition has length > 1 and only the first " "element will be used"); - if (XLENGTH(val) > 0) { + if (RAW_LENGTH(val) > 0) { switch (TYPEOF(val)) { case LGLSXP: cond = LOGICAL(val)[0]; @@ -3074,7 +3199,7 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, if (cond == NA_LOGICAL) { const char* msg = - XLENGTH(val) + RAW_LENGTH(val) ? (Rf_isLogical(val) ? ("missing value where TRUE/FALSE needed") : ("argument is not interpretable as logical")) @@ -3195,8 +3320,19 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, checkUserInterrupt(); pc += offset; PC_BOUNDSCHECK(pc, c); + // We enable OSR if: + // - We are NOT in serialize chaos mode (deserialization breaks OSR) + // - AND we are NOT in preserve mode (deserialization breaks OSR) + // - AND the compiler-client is NOT running (deserialization breaks + // OSR; but even if it worked, we don't want to compile anything + // locally, and OSR on the compiler-server isn't implemented) + // - OR any of the above is true, but we're forcing OSR regardless + // (e.g. for testing) // TODO: why does osr-in deserialized code break? - if (!pir::Parameter::RIR_SERIALIZE_CHAOS) { + if ((!pir::Parameter::RIR_SERIALIZE_CHAOS && + !pir::Parameter::RIR_PRESERVE && + !CompilerClient::isRunning()) || + pir::Parameter::FORCE_ENABLE_OSR) { static size_t loopCounter = 0; if (offset < 0 && ++loopCounter >= osrLimit) { loopCounter = 0; @@ -3321,7 +3457,7 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, goto fallback; } - if (i >= XLENGTH(val) || i < 0) + if (i >= RAW_LENGTH(val) || i < 0) goto fallback; switch (TYPEOF(val)) { @@ -3871,7 +4007,7 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, pc += offset; checkUserInterrupt(); assert(*pc == Opcode::endloop_); - advanceOpcode(); + (void)advanceOpcode(); NEXT(); } @@ -3888,7 +4024,11 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, INSTRUCTION(ret_) { goto eval_done; } INSTRUCTION(int3_) { +#ifdef __ARM_ARCH + __builtin_debugtrap(); +#else asm("int3"); +#endif NEXT(); } diff --git a/rir/src/interpreter/interp.h b/rir/src/interpreter/interp.h index 8b7a9b7da..6f94d290d 100644 --- a/rir/src/interpreter/interp.h +++ b/rir/src/interpreter/interp.h @@ -58,11 +58,13 @@ inline RCNTXT* findFunctionContextFor(SEXP e) { inline bool RecompileHeuristic(Function* fun, Function* funMaybeDisabled = nullptr) { - auto flags = fun->flags; + auto flags = fun->flags(); if (flags.contains(Function::MarkOpt)) return true; if (flags.contains(Function::NotOptimizable)) return false; + if (fun->isOptimized()) + return false; if (!funMaybeDisabled) funMaybeDisabled = fun; @@ -70,15 +72,13 @@ inline bool RecompileHeuristic(Function* fun, auto abandon = funMaybeDisabled->deoptCount() >= pir::Parameter::DEOPT_ABANDON; - auto wt = fun->isOptimized() ? pir::Parameter::PIR_REOPT_TIME - : pir::Parameter::PIR_OPT_TIME; - if (fun->invocationCount() >= 3 && fun->invocationTime() > wt) { - fun->clearInvocationTime(); - return !abandon; - } + // auto wt = fun->isOptimized() ? pir::Parameter::PIR_REOPT_TIME + // : pir::Parameter::PIR_OPT_TIME; + // if (fun->invocationCount() >= 3 && fun->invocationTime() > wt) { + // fun->clearInvocationTime(); + // return !abandon; + // } - if (fun->isOptimized()) - return false; auto wu = pir::Parameter::PIR_WARMUP; if (wu == 0) return !abandon; @@ -91,17 +91,17 @@ inline bool RecompileHeuristic(Function* fun, inline bool RecompileCondition(DispatchTable* table, Function* fun, const Context& context) { - return (fun->flags.contains(Function::MarkOpt) || !fun->isOptimized() || + return (fun->flags().contains(Function::MarkOpt) || !fun->isOptimized() || (context.smaller(fun->context()) && context.isImproving(fun) > table->size()) || - fun->flags.contains(Function::Reoptimize)); + fun->flags().contains(Function::Reoptimize)); } inline void DoRecompile(Function* fun, SEXP ast, SEXP callee, Context given) { // We have more assumptions available, let's recompile // More assumptions are available than this version uses. Let's // try compile a better matching version. - auto flags = fun->flags; + auto flags = fun->flags(); #ifdef DEBUG_DISPATCH std::cout << "Optimizing for new context " << fun->invocationCount() << ": "; @@ -113,8 +113,8 @@ inline void DoRecompile(Function* fun, SEXP ast, SEXP callee, Context given) { if (TYPEOF(lhs) == SYMSXP) name = lhs; if (flags.contains(Function::MarkOpt)) - fun->flags.reset(Function::MarkOpt); - globalContext()->closureOptimizer(callee, given, name); + fun->resetFlag(Function::MarkOpt); + SET_BODY(callee, BODY(globalContext()->closureOptimizer(callee, given, name))); } inline bool matches(const CallContext& call, Function* f) { diff --git a/rir/src/interpreter/interp_incl.h b/rir/src/interpreter/interp_incl.h index 477795e37..3cb9fdd23 100644 --- a/rir/src/interpreter/interp_incl.h +++ b/rir/src/interpreter/interp_incl.h @@ -14,6 +14,15 @@ struct Code; struct CallContext; class Configurations; +extern bool INTERPRETER_IS_ACTIVE; + +template inline void disableInterpreter(F f) { + bool wasActive = INTERPRETER_IS_ACTIVE; + INTERPRETER_IS_ACTIVE = false; + f(); + INTERPRETER_IS_ACTIVE = wasActive; +} + bool isValidClosureSEXP(SEXP closure); void initializeRuntime(); @@ -46,11 +55,6 @@ SEXP rirDecompile(SEXP s); void rirPrint(SEXP s); -void serializeRir(SEXP s, SEXP refTable, R_outpstream_t out); -SEXP deserializeRir(SEXP refTable, R_inpstream_t inp); -// Will serialize and deserialize the SEXP, returning a deep copy. -SEXP copyBySerial(SEXP x); - SEXP materialize(SEXP rirDataWrapper); SEXP evaluatePromise(SEXP e, Opcode* pc, bool delayNamed = false); diff --git a/rir/src/interpreter/profiler.cpp b/rir/src/interpreter/profiler.cpp index edfd921c2..ad50331c4 100644 --- a/rir/src/interpreter/profiler.cpp +++ b/rir/src/interpreter/profiler.cpp @@ -98,7 +98,7 @@ void RuntimeProfiler::sample(int signal) { // at least one slot justifies re-opt. if (goodValues >= (slotCount / 2) && needReopt) { // set global re-opt flag - code->function()->flags.set(Function::Reoptimize); + code->function()->setFlag(Function::Reoptimize); compilations++; } } @@ -169,6 +169,7 @@ void RuntimeProfiler::initProfiler() { #else void RuntimeProfiler::initProfiler() {} +bool RuntimeProfiler::enabled() { return false; } #endif } // namespace rir diff --git a/rir/src/interpreter/runtime.cpp b/rir/src/interpreter/runtime.cpp index 4c835ab9b..3d6d79408 100644 --- a/rir/src/interpreter/runtime.cpp +++ b/rir/src/interpreter/runtime.cpp @@ -1,8 +1,12 @@ #include "api.h" #include "interp.h" #include "profiler.h" +#include "serializeHash/globals.h" +#include "serializeHash/serialize/serializeR.h" +#include "serializeHash/hash/hashAst.h" +#include "serializeHash/serialize/native/SerialRepr.h" -#include +#include "compilerClientServer/CompilerClient.h" namespace rir { @@ -28,9 +32,12 @@ void initializeRuntime() { globalContext_ = new InterpreterInstance; context_init(); registerExternalCode(rirEval, rirApplyClosure, rirForcePromise, rirCompile, - rirDecompile, rirPrint, deserializeRir, serializeRir, + rirDecompile, rirPrint, rirDeserializeHook, rirSerializeHook, materialize); + initGlobals(); + initAstHashCache(); RuntimeProfiler::initProfiler(); + CompilerClient::tryInit(); } InterpreterInstance* globalContext() { return globalContext_; } diff --git a/rir/src/interpreter/serialize.cpp b/rir/src/interpreter/serialize.cpp deleted file mode 100644 index c6420f56c..000000000 --- a/rir/src/interpreter/serialize.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "R/r.h" -#include "compiler/parameter.h" -#include "interp_incl.h" -#include "runtime/DispatchTable.h" - -namespace rir { - -bool pir::Parameter::RIR_PRESERVE = - getenv("RIR_PRESERVE") ? atoi(getenv("RIR_PRESERVE")) : false; -unsigned pir::Parameter::RIR_SERIALIZE_CHAOS = - getenv("RIR_SERIALIZE_CHAOS") ? atoi(getenv("RIR_SERIALIZE_CHAOS")) : 0; - -static bool oldPreserve = false; - -// Will serialize s if it's an instance of CLS -template -static bool trySerialize(SEXP s, SEXP refTable, R_outpstream_t out) { - if (CLS* b = CLS::check(s)) { - OutInteger(out, b->info.magic); - b->serialize(refTable, out); - return true; - } else { - return false; - } -} - -void serializeRir(SEXP s, SEXP refTable, R_outpstream_t out) { - if (pir::Parameter::RIR_PRESERVE) { - OutInteger(out, EXTERNALSXP); - if (!trySerialize(s, refTable, out) && - !trySerialize(s, refTable, out) && - !trySerialize(s, refTable, out)) { - std::cerr << "couldn't deserialize EXTERNALSXP: "; - Rf_PrintValue(s); - assert(false); - } - } else { - WriteItem(rirDecompile(s), refTable, out); - } -} - -SEXP deserializeRir(SEXP refTable, R_inpstream_t inp) { - unsigned code = InInteger(inp); - switch (code) { - case DISPATCH_TABLE_MAGIC: - return DispatchTable::deserialize(refTable, inp)->container(); - case CODE_MAGIC: - return Code::deserialize(refTable, inp)->container(); - case FUNCTION_MAGIC: - return Function::deserialize(refTable, inp)->container(); - default: - std::cerr << "couldn't deserialize EXTERNALSXP with code: 0x" - << std::hex << code << "\n"; - assert(false); - return nullptr; - } -} - -SEXP copyBySerial(SEXP x) { - if (!pir::Parameter::RIR_SERIALIZE_CHAOS) - return x; - - oldPreserve = pir::Parameter::RIR_PRESERVE; - pir::Parameter::RIR_PRESERVE = true; - SEXP data = R_serialize(x, R_NilValue, R_NilValue, R_NilValue, R_NilValue); - PROTECT(data); - SEXP copy = R_unserialize(data, R_NilValue); - UNPROTECT(1); - pir::Parameter::RIR_PRESERVE = oldPreserve; - return copy; -} - -} // namespace rir diff --git a/rir/src/runtime/ArglistOrder.cpp b/rir/src/runtime/ArglistOrder.cpp new file mode 100644 index 000000000..ebefc1fd1 --- /dev/null +++ b/rir/src/runtime/ArglistOrder.cpp @@ -0,0 +1,41 @@ +#include "ArglistOrder.h" +#include "R/Protect.h" +#include "R/Serialize.h" + +namespace rir { + +ArglistOrder* ArglistOrder::deserialize(AbstractDeserializer& deserializer) { + Protect p; + auto size = deserializer.readBytesOf(); + auto store = p(Rf_allocVector(EXTERNALSXP, size)); + // Doesn't need ref since this is never used alone + auto arglistOrder = new (DATAPTR(store)) ArglistOrder(deserializer.readBytesOf()); + for (int i = 0, offset = sizeof(ArglistOrder); offset < size; i++, offset += sizeof(*data)) { + arglistOrder->data[i] = (ArglistOrder::ArgIdx)deserializer.readBytesOf(); + } + return arglistOrder; +} + +void ArglistOrder::serialize(AbstractSerializer& serializer) const { + auto size = (R_xlen_t)this->size(); + serializer.writeBytesOf(size); + serializer.writeBytesOf((int)nCalls); + for (int i = 0, offset = sizeof(ArglistOrder); offset < size; i++, offset += sizeof(*data)) { + serializer.writeBytesOf((int)data[i]); + } +} + +void ArglistOrder::hash(HasherOld& hasher) const { + auto size = (int)this->size(); + hasher.hashBytesOf(nCalls); + for (int i = 0, offset = sizeof(ArglistOrder); offset < size; i++, offset += sizeof(*data)) { + hasher.hashBytesOf(data[i]); + } +} + +void ArglistOrder::addConnected(__attribute__((unused)) + ConnectedCollectorOld& collector) const { + // No connected SEXPs in ArglistOrder +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/ArglistOrder.h b/rir/src/runtime/ArglistOrder.h index 2eea9ddd4..b69fe63d6 100644 --- a/rir/src/runtime/ArglistOrder.h +++ b/rir/src/runtime/ArglistOrder.h @@ -2,6 +2,9 @@ #define ARGLIST_ORDER_H #include "RirRuntimeObject.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/hash/hashRootOld.h" +#include "serializeHash/serializeUni.h" #include #include @@ -54,6 +57,14 @@ struct ArglistOrder (2 * reordering.size() + sz) * sizeof(*data); } + size_t size() const { + size_t sz = 0; + for (size_t i = 0; i < nCalls; i++) { + sz += originalArglistLength(i); + } + return sizeof(ArglistOrder) + (2 * nCalls + sz) * sizeof(*data); + } + static ArglistOrder* New(std::vector const& reordering) { SEXP cont = Rf_allocVector(EXTERNALSXP, size(reordering)); ArglistOrder* res = new (DATAPTR(cont)) ArglistOrder(reordering); @@ -61,7 +72,7 @@ struct ArglistOrder } explicit ArglistOrder(std::vector const& reordering) - : RirRuntimeObject(0, 0), nCalls(reordering.size()) { + : ArglistOrder(reordering.size()) { auto offset = nCalls * 2; for (size_t i = 0; i < nCalls; i++) { data[2 * i] = offset; @@ -84,12 +95,22 @@ struct ArglistOrder return data[callId * 2 + 1]; } + static ArglistOrder* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& deserializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + /* * Layout of data[] is nCalls * (offset, length), followed by * nCalls * (variable length list of indices) */ size_t nCalls; ArgIdx data[]; + + private: + // cppcheck-suppress uninitMemberVarPrivate + explicit ArglistOrder(size_t nCalls) + : RirRuntimeObject(0, 0), nCalls(nCalls) {} }; #pragma pack(pop) diff --git a/rir/src/runtime/Code.cpp b/rir/src/runtime/Code.cpp index b7ca453cd..56d02f946 100644 --- a/rir/src/runtime/Code.cpp +++ b/rir/src/runtime/Code.cpp @@ -1,26 +1,35 @@ #include "Code.h" #include "Function.h" #include "R/Printing.h" -#include "R/Serialize.h" #include "bc/BC.h" +#include "bc/BC_inc.h" #include "compiler/native/pir_jit_llvm.h" +#include "compiler/parameter.h" +#include "rirObjectMagic.h" +#include "runtime/log/printPrettyGraph.h" +#include "runtime/TypeFeedback.h" +#include "serializeHash/hash/UUIDPool.h" +#include "serializeHash/hash/hashAst.h" +#include "serializeHash/serialize/serialize.h" +#include "utils/HTMLBuilder/escapeHtml.h" #include "utils/Pool.h" +#include "utils/measuring.h" #include -#include #include #include namespace rir { +static const unsigned PRETTY_GRAPH_CODE_NAME_MAX_LENGTH = 25; + // cppcheck-suppress uninitMemberVar; symbol=data Code::Code(Kind kind, FunctionSEXP fun, SEXP src, unsigned srcIdx, unsigned cs, unsigned sourceLength, size_t localsCnt, size_t bindingsCnt) : RirRuntimeObject( // GC area starts just after the header (intptr_t)&locals_ - (intptr_t)this, - // GC area has only 1 pointer NumLocals), kind(kind), nativeCode_(nullptr), src(srcIdx), trivialExpr(nullptr), stackLength(0), localsCount(localsCnt), bindingCacheSize(bindingsCnt), @@ -51,15 +60,38 @@ Code* Code::NewNative(Immediate ast) { return New(Kind::Native, ast, 0, 0, 0, 0); } -Code::~Code() { - // TODO: Not sure if this is actually called - // Otherwise the pointer will leak a few bytes +void Code::lazyCode(const std::string& handle, const SerialModule* module) { + if (module) { + PROTECT(module->container()); + } + assert(!handle.empty()); + assert(handle.size() < MAX_CODE_HANDLE_LENGTH); + assert(kind == Kind::Native); + assert(lazyCodeHandle[0] == '\0' && !getEntry(4)); + strncpy(lazyCodeHandle, handle.c_str(), MAX_CODE_HANDLE_LENGTH - 1); + if (module) { + UNPROTECT(1); + setLazyCodeModule(module); + } + UUIDPool::reintern(container()); } void Code::function(Function* fun) { setEntry(3, fun->container()); } +rir::Function* Code::functionOpt() const { + auto f = getEntry(3); + if (!f && kind == Kind::Deserializing) { + return nullptr; + } + assert(f && "no function, but code is not being deserialized"); + return rir::Function::check(f); +} + rir::Function* Code::function() const { auto f = getEntry(3); + if (!f && kind == Kind::Deserializing) { + assert(false && "can't access function of code while it's being deserialized"); + } assert(f); return rir::Function::unpack(f); } @@ -103,91 +135,232 @@ unsigned Code::getSrcIdxAt(const Opcode* pc, bool allowMissing) const { return sidx; } -Code* Code::deserialize(SEXP refTable, R_inpstream_t inp) { - size_t size = InInteger(inp); - SEXP store = Rf_allocVector(EXTERNALSXP, size); - PROTECT(store); - Code* code = new (DATAPTR(store)) Code; - code->nativeCode_ = nullptr; // not serialized for now - code->src = InInteger(inp); - bool hasTr = InInteger(inp); - if (hasTr) - code->trivialExpr = ReadItem(refTable, inp); - code->stackLength = InInteger(inp); - *const_cast(&code->localsCount) = InInteger(inp); - *const_cast(&code->bindingCacheSize) = InInteger(inp); - code->codeSize = InInteger(inp); - code->srcLength = InInteger(inp); - code->extraPoolSize = InInteger(inp); - SEXP extraPool = ReadItem(refTable, inp); - PROTECT(extraPool); - auto hasArgReorder = InInteger(inp); - SEXP argReorder = nullptr; - if (hasArgReorder) { - argReorder = ReadItem(refTable, inp); - PROTECT(argReorder); - } - SEXP rirFunction = ReadItem(refTable, inp); - PROTECT(rirFunction); +Code* Code::deserialize(AbstractDeserializer& deserializer) { + Protect p; + auto size = deserializer.readBytesOf(SerialFlags::CodeMisc); + auto store = p(Rf_allocVector(EXTERNALSXP, size)); + auto code = new (DATAPTR(store)) Code; + // Magic is already set + deserializer.addRef(store); + + // Header + DESERIALIZE(code->src, readSrc, SerialFlags::CodeAst); + DESERIALIZE(code->trivialExpr, readNullable, SerialFlags::CodeAst); + DESERIALIZE(code->stackLength, readBytesOf, SerialFlags::CodeMisc); + DESERIALIZE(*const_cast(&code->localsCount), readBytesOf, SerialFlags::CodeMisc); + DESERIALIZE(*const_cast(&code->bindingCacheSize), readBytesOf, SerialFlags::CodeMisc); + DESERIALIZE(code->codeSize, readBytesOf, SerialFlags::CodeMisc); + DESERIALIZE(code->srcLength, readBytesOf, SerialFlags::CodeMisc); + DESERIALIZE(code->extraPoolSize, readBytesOf, SerialFlags::CodeMisc); + auto argReorder = deserializer.readNullable(SerialFlags::CodeArglistOrder); + auto outer = p.nullable(deserializer.read(SerialFlags::CodeOuterFun)); + // Can't check magic because it may not be assigned yet + assert((!outer || TYPEOF(outer) == EXTERNALSXP) && + "sanity check failed: code's outer is not a Function"); // Bytecode - BC::deserialize(refTable, inp, code->code(), code->codeSize, code); + BC::deserialize(deserializer, code->code(), code->codeSize, code); + + // Extra pool + SEXP extraPool = Rf_allocVector(VECSXP, code->extraPoolSize); + for (unsigned i = 0; i < code->extraPoolSize; ++i) { + auto extraPoolFlag = SerialFlags::ById[deserializer.readBytesOf(SerialFlags::CodeMisc)]; + SET_VECTOR_ELT(extraPool, i, deserializer.read(extraPoolFlag)); + } // Srclist - for (unsigned i = 0; i < code->srcLength; i++) { - code->srclist()[i].pcOffset = InInteger(inp); - code->srclist()[i].srcIdx = src_pool_add(ReadItem(refTable, inp)); + if (deserializer.willRead(SerialFlags::CodeMisc)) { + for (unsigned i = 0; i < code->srcLength; i++) { + code->srclist()[i].pcOffset = deserializer.readBytesOf(SerialFlags::CodeMisc); + code->srclist()[i].srcIdx = deserializer.readSrc(SerialFlags::CodeAst); + } } code->info = {// GC area starts just after the header (uint32_t)((intptr_t)&code->locals_ - (intptr_t)code), - // GC area has only 1 pointer NumLocals, CODE_MAGIC}; code->setEntry(0, extraPool); - code->setEntry(3, rirFunction); - if (hasArgReorder) { + if (outer) { + code->setEntry(3, outer); + } + if (argReorder) { code->setEntry(2, argReorder); - UNPROTECT(1); } - UNPROTECT(3); + + // Native code + if (deserializer.willRead(SerialFlags::CodeNative)) { + code->kind = deserializer.readBytesOf(SerialFlags::CodeNative); + if (code->kind == Kind::Native) { + auto lazyCodeHandleLen = + deserializer.readBytesOf(SerialFlags::CodeNative); + deserializer.readBytes(code->lazyCodeHandle, lazyCodeHandleLen, + SerialFlags::CodeNative); + code->lazyCodeHandle[lazyCodeHandleLen] = '\0'; + if (deserializer.readBytesOf(SerialFlags::CodeNative)) { + auto lazyCodeModule = pir::PirJitLLVM::deserializeModule( + deserializer, code,deserializer.serialOptions()); + code->setLazyCodeModule(lazyCodeModule); + } + } + } + // Native code is always null here because it's lazy + code->nativeCode_ = nullptr; return code; } -void Code::serialize(SEXP refTable, R_outpstream_t out) const { - OutInteger(out, size()); - // Header - OutInteger(out, src); - OutInteger(out, trivialExpr != nullptr); - if (trivialExpr) - WriteItem(trivialExpr, refTable, out); - OutInteger(out, stackLength); - OutInteger(out, localsCount); - OutInteger(out, bindingCacheSize); - OutInteger(out, codeSize); - OutInteger(out, srcLength); - OutInteger(out, extraPoolSize); - WriteItem(getEntry(0), refTable, out); - OutInteger(out, getEntry(2) != nullptr); - if (getEntry(2)) - WriteItem(getEntry(2), refTable, out); - WriteItem(getEntry(3), refTable, out); +void Code::serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf((R_xlen_t)size(), SerialFlags::CodeMisc); + + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: serialize source", container(), [&]{ + serializer.writeSrc(src, SerialFlags::CodeAst); + serializer.writeNullable(trivialExpr, SerialFlags::CodeAst); + }); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: serialize numbers", container(), [&]{ + serializer.writeBytesOf((unsigned)stackLength, SerialFlags::CodeMisc); + serializer.writeBytesOf((unsigned)localsCount, SerialFlags::CodeMisc); + serializer.writeBytesOf((unsigned)bindingCacheSize, SerialFlags::CodeMisc); + serializer.writeBytesOf((unsigned)codeSize, SerialFlags::CodeMisc); + serializer.writeBytesOf((unsigned)srcLength, SerialFlags::CodeMisc); + serializer.writeBytesOf((unsigned)extraPoolSize, SerialFlags::CodeMisc); + }); + + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: serialize call argument reordering metadata", container(), [&]{ + serializer.writeNullable(getEntry(2), SerialFlags::CodeArglistOrder); + }); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: serialize outer function", container(), [&]{ + serializer.write(getEntry(3), SerialFlags::CodeOuterFun); + }); + + std::vector extraPoolFlags(extraPoolSize, SerialFlags::CodePoolUnknown); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: serialize bytecode", container(), [&]{ + // One might think we can skip serializing entries which are just + // recorded calls, but it breaks semantics and causes a test failure + BC::serialize(serializer, extraPoolFlags, code(), codeSize, this); + }); + + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: serialize extra pool", container(), [&]{ + for (unsigned i = 0; i < extraPoolSize; ++i) { + serializer.writeBytesOf(extraPoolFlags[i].id(), SerialFlags::CodeMisc); + serializer.write(getExtraPoolEntry(i), extraPoolFlags[i]); + } + }); - // Bytecode - BC::serialize(refTable, out, code(), codeSize, this); + if (serializer.willWrite(SerialFlags::CodeMisc)) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: serialize srclist", container(), [&]{ + for (unsigned i = 0; i < srcLength; i++) { + serializer.writeBytesOf(srclist()[i].pcOffset, SerialFlags::CodeMisc); + serializer.writeSrc(srclist()[i].srcIdx, SerialFlags::CodeAst); + } + }); + } - // Srclist - for (unsigned i = 0; i < srcLength; i++) { - OutInteger(out, srclist()[i].pcOffset); - WriteItem(src_pool_at(srclist()[i].srcIdx), refTable, out); + if (serializer.willWrite(SerialFlags::CodeNative)) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: serialize native", container(), [&]{ + serializer.writeBytesOf(kind, SerialFlags::CodeNative); + assert((kind != Kind::Native || lazyCodeHandle[0] != '\0') && + "Code in bad pending state"); + if (kind == Kind::Native && lazyCodeHandle[0] != '\0') { + assert(lazyCodeHandle[0] != '\0'); + auto lazyCodeHandleLen = (unsigned)strlen(lazyCodeHandle); + serializer.writeBytesOf(lazyCodeHandleLen, SerialFlags::CodeNative); + serializer.writeBytes(lazyCodeHandle, lazyCodeHandleLen, SerialFlags::CodeNative); + auto lcm = lazyCodeModule(); + serializer.writeBytesOf(lcm != nullptr, SerialFlags::CodeNative); + if (lcm) { + serializer.write(lcm->container(), SerialFlags::CodeNative); + } + } + }); } } +void Code::hash(HasherOld& hasher) const { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: hash source", container(), [&]{ + hasher.hashSrc(src); + }); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: hash numbers", container(), [&]{ + hasher.hashBytesOf(stackLength); + hasher.hashBytesOf(localsCount); + hasher.hashBytesOf(bindingCacheSize); + hasher.hashBytesOf(codeSize); + hasher.hashBytesOf(srcLength); + hasher.hashBytesOf(extraPoolSize); + }); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: hash call argument reordering metadata", container(), [&]{ + hasher.hashNullable(getEntry(2)); + }); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: hash outer function", container(), [&]{ + hasher.hash(function()->container()); + }); + + std::vector extraPoolIgnored; + extraPoolIgnored.resize(extraPoolSize); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: hash bytecode", container(), [&]{ + BC::hash(hasher, extraPoolIgnored, code(), codeSize, this); + }); + + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: hash extra pool", container(), [&]{ + hasher.hashBytesOf(extraPoolSize); + for (unsigned i = 0; i < extraPoolSize; ++i) { + if (!extraPoolIgnored[i]) { + hasher.hash(getExtraPoolEntry(i)); + } + } + }); + + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: hash srclist", container(), [&]{ + for (unsigned i = 0; i < srcLength; i++) { + hasher.hashBytesOf(srclist()[i].pcOffset); + hasher.hashSrc(srclist()[i].srcIdx); + } + }); + + // Don't hash native code +} + +void Code::addConnected(ConnectedCollectorOld& collector) const { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: add connected in source", container(), [&]{ + collector.addSrc(src); + }); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: add connected in call argument reordering metadata", container(), [&]{ + collector.addNullable(getEntry(2), false); + }); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: add connected in outer function", container(), [&]{ + collector.add(function()->container(), false); + }); + + std::vector extraPoolChildren; + extraPoolChildren.resize(extraPoolSize); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: add connected in bytecode", container(), [&]{ + BC::addConnected(extraPoolChildren, collector, code(), codeSize, this); + }); + + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: add connected in extra pool", container(), [&]{ + for (unsigned i = 0; i < extraPoolSize; ++i) { + collector.add(getExtraPoolEntry(i), extraPoolChildren[i]); + } + }); + + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "Code.cpp: add connected in srclist", container(), [&]{ + for (unsigned i = 0; i < srcLength; i++) { + collector.addSrc(srclist()[i].srcIdx); + } + }); + + // No connected in SEXPs native code +} + void Code::disassemble(std::ostream& out, const std::string& prefix) const { + if (kind == Kind::Deserializing) { + out << "(code is being deserialized)\n"; + return; + } + if (auto map = pirTypeFeedback()) { map->forEachSlot( [&](size_t i, const PirTypeFeedback::MDEntry& mdEntry) { auto feedback = mdEntry.feedback; - out << " - slot #" << i << ": " << mdEntry.offset << " : ["; + out << " - slot #" << i << ": " << mdEntry.rirIdx << " : ["; feedback.print(out); out << "] (" << mdEntry.sampleCount << " records - " << (mdEntry.readyForReopt ? "ready" : "not ready") @@ -197,6 +370,8 @@ void Code::disassemble(std::ostream& out, const std::string& prefix) const { switch (kind) { case Kind::Bytecode: { + auto fun = functionOpt(); + auto typeFeedback = fun && !fun->isDeserializing() ? fun->typeFeedback() : nullptr; Opcode* pc = code(); size_t label = 0; std::map targets; @@ -257,10 +432,35 @@ void Code::disassemble(std::ostream& out, const std::string& prefix) const { bc.printOpcode(out); formatLabel(targets[BC::jmpTarget(pc)]); out << "\n"; + } else if (bc.isRecord()) { + out << " " + << "[ "; + if (bc.bc == Opcode::record_call_) { + if (typeFeedback) { + typeFeedback->callees(bc.immediate.i).print(out, fun); + } else { + out << ""; + } + out << " ] Call#"; + } else if (bc.bc == Opcode::record_test_) { + if (typeFeedback) { + typeFeedback->test(bc.immediate.i).print(out); + } else { + out << ""; + } + out << " ] Test#"; + } else { + if (typeFeedback) { + typeFeedback->types(bc.immediate.i).print(out); + } else { + out << ""; + } + out << " ] Type#"; + } + out << bc.immediate.i << "\n"; } else { bc.print(out); } - pc = BC::next(pc); } @@ -275,7 +475,16 @@ void Code::disassemble(std::ostream& out, const std::string& prefix) const { } case Kind::Native: { if (nativeCode_) { - out << "nativeCode " << (void*)nativeCode_ << "\n"; + out << "nativeCode " << nativeCode_ << ", module:"; + auto lcm = lazyCodeModule(); + if (lcm) { + out << "\n" << lcm << " (" + << lcm->size() << " bytes)\n"; + lcm->print(out); + } else { + out << " (elided)"; + } + out << "\n"; } else { out << "nativeCode (compilation pending)\n"; } @@ -298,16 +507,20 @@ void Code::disassemble(std::ostream& out, const std::string& prefix) const { } } -void Code::print(std::ostream& out) const { +void Code::print(std::ostream& out, bool isDetailed) const { out << "Code object\n"; out << std::left << std::setw(20) << " Source: " << src << " (index into src pool)\n"; - out << std::left << std::setw(20) << " Magic: " << std::hex << info.magic - << std::dec << " (hex)\n"; + out << std::left << std::setw(20) << " Magic: " << std::hex + << info.magic << std::dec << " (hex)\n"; out << std::left << std::setw(20) << " Stack (o): " << stackLength << "\n"; out << std::left << std::setw(20) << " Code size: " << codeSize << "[B]\n"; + if (isDetailed) { + out << std::left << std::setw(20) << " Size: " << size() + << "[B]\n"; + } if (info.magic != CODE_MAGIC) { out << "Wrong magic number -- corrupted IR bytecode"; @@ -316,6 +529,262 @@ void Code::print(std::ostream& out) const { out << "\n"; disassemble(out); + + if (isDetailed) { + out << "extra pool = \n" + << Print::dumpSexp(getEntry(0), SIZE_MAX) << "\n"; + out << "src = \n" + << Print::dumpSexp(src_pool_at(src), SIZE_MAX) + << ", hash = " << hashAst(src_pool_at(src)) << "\n"; + for (unsigned i = 0; i < srcLength; i++) { + out << "src[" << i << "] @ " << srclist()[i].pcOffset + << " = \n"; + out << Print::dumpSexp(src_pool_at(i), SIZE_MAX) + << ", hash = " << hashAst(src_pool_at(i)) << "\n"; + } + } +} + + +static bool isEqualOrInExtraPool(const Code* outer, const Code* inner) { // NOLINT(*-no-recursion) + if (outer == inner) { + return true; + } + for (unsigned i = 0; i < outer->extraPoolSize; ++i) { + auto codeEntry = Code::check(outer->getExtraPoolEntry(i)); + if (codeEntry && isEqualOrInExtraPool(codeEntry, inner)) { + return true; + } + } + return false; +} + +static bool isInFunction(const Function* outer, const Code* inner) { + if (isEqualOrInExtraPool(outer->body(), inner)) { + return true; + } + for (unsigned i = 0; i < outer->nargs(); ++i) { + auto arg = outer->defaultArg(i); + if (arg && isEqualOrInExtraPool(arg, inner)) { + return true; + } + } + return false; +} + +void Code::printPrettyGraphContent(const PrettyGraphInnerPrinter& print) const { + auto srcPrint = Print::dumpSexp(src_pool_at(src), SIZE_MAX); + print.addName([&](std::ostream& s) { + if (srcPrint.length() < PRETTY_GRAPH_CODE_NAME_MAX_LENGTH) { + s << srcPrint; + } else { + s << srcPrint.substr(0, PRETTY_GRAPH_CODE_NAME_MAX_LENGTH) + << "..."; + } + }); + print.addBody([&](std::ostream& s) { + if (srcPrint.length() >= PRETTY_GRAPH_CODE_NAME_MAX_LENGTH) { + s << "
" << escapeHtml(srcPrint) << "
\n"; + } + std::stringstream str; + disassemble(str); + s << "
" << escapeHtml(str.str()) << "
"; + }); + auto addSourceEdge = [&](SEXP sexp, const char* type, size_t index = SIZE_MAX){ + if (sexp && sexp != R_NilValue && TYPEOF(sexp) != SYMSXP && + TYPEOF(sexp) != LANGSXP && TYPEOF(sexp) != INTSXP && + TYPEOF(sexp) != LGLSXP && TYPEOF(sexp) != REALSXP && + TYPEOF(sexp) != CPLXSXP && TYPEOF(sexp) != CHARSXP && + TYPEOF(sexp) != STRSXP) { + print.addEdgeTo(sexp, false, "unexpected-ast", [&](std::ostream& s){ + s << type; + if (index != SIZE_MAX) { + s << " " << index; + } + s << " isn't a source type!"; + }); + } + }; + addSourceEdge(src_pool_at(src), "source"); + addSourceEdge(trivialExpr, "trivial-expr"); + for (unsigned i = 0; i < srcLength; i++) { + addSourceEdge(src_pool_at(i), "src-pool entry", i); + } + if (arglistOrderContainer()) { + print.addEdgeTo(arglistOrderContainer(), true, "arglist-order", [&](std::ostream& s) { + s << "arglist order"; + }); + } + auto fun = functionOpt(); + if (fun && !isInFunction(fun, this)) { + print.addEdgeTo(fun->container(), false, "unexpected", [&](std::ostream& s) { + s << "function, its not this code's parent!"; + }); + } + std::vector addedExtraPoolEntries; + addedExtraPoolEntries.resize(extraPoolSize); + BC::addToPrettyGraph(print, addedExtraPoolEntries, code(), codeSize, this); + for (unsigned i = 0; i < extraPoolSize; i++) { + if (!addedExtraPoolEntries[i]) { + print.addEdgeTo(getExtraPoolEntry(i), false, "unknown-extra-pool", [&](std::ostream& s) { + s << "pool " << i; + }); + } + } +} + +static void compareAsts(SEXP ast1, SEXP ast2, + const char* prefix, const char* srcPrefix, + std::stringstream& differences) { + // Asts can be compared via printing + auto print1 = ast1 ? Print::dumpSexp(ast1, SIZE_MAX) : "(null)"; + auto print2 = ast1 ? Print::dumpSexp(ast2, SIZE_MAX) : "(null)"; + if (print1 != print2) { + differences << prefix << " " << srcPrefix << " asts differ:\n"; + differences << prefix << " " << srcPrefix << "1: " << print1 << "\n"; + differences << prefix << " " << srcPrefix << "2: " << print2 << "\n"; + } +} + +// Can probably be compared for equivalency by comparing the debug prints (no +// pointers in debug prints). This is used for debugging so doesn't have to be +// 100% accurate +static bool isProbablyDirectlyComparable[] = { + /* NILSXP */ true, + /* SYMSXP */ true, + /* LISTSXP */ true, + /* CLOSXP */ false, + /* ENVSXP */ false, + /* PROMSXP */ false, + /* LANGSXP */ true, + /* SPECIALSXP */ true, + /* BUILTINSXP */ true, + /* CHARSXP */ true, + /* LGLSXP */ true, + /* unused */ false, + /* unused */ false, + /* INTSXP */ true, + /* REALSXP */ true, + /* CPLXSXP */ true, + /* STRSXP */ true, + /* DOTSXP */ true, + /* ANYSXP */ false, + /* VECSXP */ true, + /* EXPRSXP */ true, + /* BCODESXP */ false, + /* EXTPTRSXP */ false, + /* WEAKREFSXP */ false, + /* RAWSXP */ false, + /* S4SXP */ false, + /* EXTERNALSXP */ false +}; + +static void compareSexps(SEXP sexp1, SEXP sexp2, + const char* prefix, const char* srcPrefix, + std::stringstream& differences, + bool compareExtraPoolRBytecodes) { + if (TYPEOF(sexp1) != TYPEOF(sexp2)) { + if (compareExtraPoolRBytecodes || + TYPEOF(sexp1) != BCODESXP || TYPEOF(sexp2) != NILSXP) { + differences << prefix << " " << srcPrefix + << " types differ: " << Rf_type2char(TYPEOF(sexp1)) + << " vs " << Rf_type2char(TYPEOF(sexp2)) << "\n"; + } + return; + } + if (TYPEOF(sexp1) == EXTERNALSXP && + rirObjectMagic(sexp1) != rirObjectMagic(sexp2)) { + differences << prefix << " " << srcPrefix << " rir types differ: " + << rirObjectMagic(sexp1) << " vs " + << rirObjectMagic(sexp2) << "\n"; + return; + } + + if (Code::check(sexp1)) { + auto poolPrefix = std::string(prefix) + " " + srcPrefix; + + Code::debugCompare( + Code::unpack(sexp1), + Code::unpack(sexp2), + poolPrefix.c_str(), + differences, + compareExtraPoolRBytecodes + ); + } else if (TYPEOF(sexp1) == RAWSXP) { + auto raw1 = RAW(sexp1); + auto raw2 = RAW(sexp2); + auto len1 = XLENGTH(sexp1); + auto len2 = XLENGTH(sexp2); + if (len1 != len2) { + differences << prefix << " " << srcPrefix << " raw lengths differ: " + << len1 << " vs " << len2 << "\n"; + } + if (memcmp(raw1, raw2, len1) != 0) { + differences << prefix << " " << srcPrefix << " raws differ\n"; + } + } else if (isProbablyDirectlyComparable[TYPEOF(sexp1)]) { + compareAsts(sexp1, sexp2, prefix, srcPrefix, differences); + } +} + +static void compareSrcs(unsigned src1, unsigned src2, + const char* prefix, const char* srcPrefix, + std::stringstream& differences) { + compareAsts(src_pool_at(src1), src_pool_at(src2), prefix, + srcPrefix, differences); +} + +void Code::debugCompare(const Code* c1, const Code* c2, const char* prefix, + std::stringstream& differences, bool compareExtraPoolRBytecodes) { + compareSrcs(c1->src, c2->src, prefix, "src", differences); + compareAsts(c1->trivialExpr, c2->trivialExpr, prefix, "trivialExpr", differences); + if (c1->srcLength != c2->srcLength) { + differences << prefix << " srcLengths differ: " << c1->srcLength + << " vs " << c2->srcLength << "\n"; + } + if (c1->codeSize != c2->codeSize) { + differences << prefix << " codeSizes differ: " << c1->codeSize << " vs " + << c2->codeSize << "\n"; + } + if (c1->stackLength != c2->stackLength) { + differences << prefix << " stackLengths differ: " << c1->stackLength + << " vs " << c2->stackLength << "\n"; + } + // c1 may have extra pool R-bytecodes than c2, + // if it was from a closure with them and c2 was from an AST-only closure + if (compareExtraPoolRBytecodes ? + c1->extraPoolSize != c2->extraPoolSize : + c1->extraPoolSize < c2->extraPoolSize) { + differences << prefix << " extraPoolSizes differ: " << c1->extraPoolSize + << " vs " << c2->extraPoolSize << "\n"; + } + if (c1->bindingCacheSize != c2->bindingCacheSize) { + differences << prefix << " bindingCacheSizes differ: " + << c1->bindingCacheSize << " vs " << c2->bindingCacheSize + << "\n"; + } + for (unsigned i = 0; i < std::min(c1->srcLength, c2->srcLength); i++) { + auto src1 = c1->srclist()[i]; + auto src2 = c2->srclist()[i]; + if (src1.pcOffset != src2.pcOffset) { + differences << prefix << " src " << i << " pcOffsets differ: " + << src1.pcOffset << " vs " << src2.pcOffset << "\n"; + } + char srcPrefix[100]; + sprintf(srcPrefix, "src %u", i); + compareSrcs(src1.srcIdx, src2.srcIdx, prefix, + srcPrefix, differences); + } + BC::debugCompare(c1->code(), c2->code(), c1->codeSize, c2->codeSize, c1, c2, + prefix, differences); + for (unsigned i = 0; i < std::min(c1->extraPoolSize, c2->extraPoolSize); i++) { + auto pool1 = c1->getExtraPoolEntry(i); + auto pool2 = c2->getExtraPoolEntry(i); + + char poolPrefix[100]; + sprintf(poolPrefix, "entry %u", i); + compareSexps(pool1, pool2, prefix, poolPrefix, differences, compareExtraPoolRBytecodes); + } } unsigned Code::addExtraPoolEntry(SEXP v) { @@ -338,10 +807,23 @@ unsigned Code::addExtraPoolEntry(SEXP v) { llvm::ExitOnError ExitOnErr; +const SerialModule* Code::lazyCodeModule() const { + auto module = getEntry(4) ? SerialModule::unpack(getEntry(4)) : nullptr; + assert((!module || (kind == Kind::Native && *lazyCodeHandle != '\0')) && + "If code has a lazy module, it should be native code with a handle"); + return module; +} + +void Code::setLazyCodeModule(const rir::SerialModule* module) { + assert(kind == Kind::Native && *lazyCodeHandle != '\0' && + "Can only set lazy code module for native code with a handle"); + setEntry(4, module->container()); +} + NativeCode Code::lazyCompile() { assert(kind == Kind::Native); - assert(*lazyCodeHandle_ != '\0'); - auto symbol = ExitOnErr(pir::PirJitLLVM::JIT->lookup(lazyCodeHandle_)); + assert(*lazyCodeHandle != '\0'); + auto symbol = ExitOnErr(pir::PirJitLLVM::JIT->lookup(lazyCodeHandle)); nativeCode_ = (NativeCode)symbol.getAddress(); return nativeCode_; } diff --git a/rir/src/runtime/Code.h b/rir/src/runtime/Code.h index a15e0a437..c113a816f 100644 --- a/rir/src/runtime/Code.h +++ b/rir/src/runtime/Code.h @@ -5,18 +5,32 @@ #include "PirTypeFeedback.h" #include "RirRuntimeObject.h" #include "bc/BC_inc.h" +#include "runtime/log/RirObjectPrintStyle.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/hash/hashRootOld.h" +#include "serializeHash/serialize/native/SerialModule.h" +#include "utils/ByteBuffer.h" #include #include #include #include +#ifndef __ARM_ARCH #include +#endif + +namespace llvm { + +class Function; + +} // namespace llvm namespace rir { typedef SEXP FunctionSEXP; typedef SEXP CodeSEXP; +typedef SEXP (*NativeCode)(Code*, void*, SEXP, SEXP); #define CODE_MAGIC 0xc0de0000 #define NATIVE_CODE_MAGIC 0xc0deffff @@ -45,24 +59,23 @@ typedef SEXP CodeSEXP; struct InterpreterInstance; struct Code; -typedef SEXP (*NativeCode)(Code*, void*, SEXP, SEXP); struct Code : public RirRuntimeObject { friend class FunctionWriter; friend class CodeVerifier; - enum class Kind { Bytecode, Native } kind; + enum class Kind { Bytecode, Native, Deserializing } kind; - // extra pool, pir type feedback, arg reordering info - static constexpr size_t NumLocals = 4; + // extra pool, pir type feedback, arg reordering info, rir function, + // lazy code module + static constexpr size_t NumLocals = 5; Code(Kind kind, FunctionSEXP fun, SEXP src, unsigned srcIdx, unsigned codeSize, unsigned sourceSize, size_t localsCnt, size_t bindingsCacheSize); - ~Code(); private: - Code() : Code(Kind::Bytecode, nullptr, 0, 0, 0, 0, 0, 0) {} + Code() : Code(Kind::Deserializing, nullptr, 0, 0, 0, 0, 0, 0) {} static Code* New(Kind kind, Immediate ast, size_t codeSize, size_t sources, size_t locals, size_t bindingCache); /* @@ -72,6 +85,7 @@ struct Code : public RirRuntimeObject { * 1 : pir type feedback * 2 : call argument reordering metadata * 3 : rir function + * 4 : lazy code module */ SEXP locals_[NumLocals]; @@ -83,26 +97,18 @@ struct Code : public RirRuntimeObject { constexpr static size_t MAX_CODE_HANDLE_LENGTH = 64; private: - char lazyCodeHandle_[MAX_CODE_HANDLE_LENGTH] = "\0"; + char lazyCodeHandle[MAX_CODE_HANDLE_LENGTH] = "\0"; + const SerialModule* lazyCodeModule() const; + void setLazyCodeModule(const SerialModule* module); NativeCode nativeCode_; NativeCode lazyCompile(); public: - void lazyCodeHandle(const std::string& h) { - assert(h != ""); - assert(kind == Kind::Native); - auto l = h.length() + 1; - if (l > MAX_CODE_HANDLE_LENGTH) { - assert(false); - l = MAX_CODE_HANDLE_LENGTH; - } - memcpy(&lazyCodeHandle_, h.c_str(), l); - lazyCodeHandle_[MAX_CODE_HANDLE_LENGTH - 1] = '\0'; - } + void lazyCode(const std::string& handle, const SerialModule* module); NativeCode nativeCode() { if (nativeCode_) return nativeCode_; - if (kind == Kind::Bytecode || *lazyCodeHandle_ == '\0') + if (kind == Kind::Bytecode || lazyCodeHandle[0] == '\0') return nullptr; return lazyCompile(); } @@ -114,7 +120,7 @@ struct Code : public RirRuntimeObject { // evaluated (if we trigger some code in the backend, eg. during printing). // The current workaround is to skip them during dispatch. bool pendingCompilation() const { - return kind == Kind::Native && *lazyCodeHandle_ == '\0'; + return kind == Kind::Native && lazyCodeHandle[0] == '\0'; } static unsigned pad4(unsigned sizeInBytes) { @@ -202,6 +208,10 @@ struct Code : public RirRuntimeObject { return ArglistOrder::unpack(data); } + private: + // Only used when code may not be fully deserialized + rir::Function* functionOpt() const; + public: rir::Function* function() const; void function(rir::Function*); @@ -218,11 +228,24 @@ struct Code : public RirRuntimeObject { unsigned getSrcIdxAt(const Opcode* pc, bool allowMissing) const; - static Code* deserialize(SEXP refTable, R_inpstream_t inp); - void serialize(SEXP refTable, R_outpstream_t out) const; + static Code* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& deserializer) const; + + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + void disassemble(std::ostream&, const std::string& promPrefix) const; void disassemble(std::ostream& out) const { disassemble(out, ""); } - void print(std::ostream&) const; + void print(std::ostream&, bool isDetailed = false) const; + void printPrettyGraphContent(const PrettyGraphInnerPrinter& print) const; + + /// Check if 2 code objects are the same, for validation and sanity check + /// (before we do operations which will cause weird errors otherwise). If + /// not, will add each difference to differences, prefixing with `prefix` + /// (the code type, either body or default arg). + static void debugCompare(const Code* c1, const Code* c2, const char* prefix, + std::stringstream& differences, + bool compareExtraPoolRBytecodes = true); static size_t extraPtrOffset() { static Code* c = (Code*)malloc(sizeof(Code)); diff --git a/rir/src/runtime/Context.cpp b/rir/src/runtime/Context.cpp index 271e008dd..b9e83923f 100644 --- a/rir/src/runtime/Context.cpp +++ b/rir/src/runtime/Context.cpp @@ -6,16 +6,6 @@ namespace rir { -Context Context::deserialize(SEXP refTable, R_inpstream_t inp) { - Context as; - InBytes(inp, &as, sizeof(Context)); - return as; -} - -void Context::serialize(SEXP refTable, R_outpstream_t out) const { - OutBytes(out, this, sizeof(Context)); -} - std::ostream& operator<<(std::ostream& out, Assumption a) { switch (a) { case Assumption::NoExplicitlyMissingArgs: diff --git a/rir/src/runtime/Context.h b/rir/src/runtime/Context.h index e30ca5182..61da04cdc 100644 --- a/rir/src/runtime/Context.h +++ b/rir/src/runtime/Context.h @@ -252,9 +252,6 @@ struct Context { unsigned isImproving(const Context& other, bool hasDotsFormals, bool hasDefaultArgs) const; - static Context deserialize(SEXP refTable, R_inpstream_t inp); - void serialize(SEXP refTable, R_outpstream_t out) const; - friend struct std::hash; friend std::ostream& operator<<(std::ostream& out, const Context& a); diff --git a/rir/src/runtime/Deoptimization.cpp b/rir/src/runtime/Deoptimization.cpp index 33e262ea6..a9ab41351 100644 --- a/rir/src/runtime/Deoptimization.cpp +++ b/rir/src/runtime/Deoptimization.cpp @@ -1,9 +1,69 @@ #include "Deoptimization.h" -#include "R/Serialize.h" #include "runtime/Code.h" +#include "serializeHash/hash/UUID.h" +#include "serializeHash/hash/UUIDPool.h" +#include "utils/ByteBuffer.h" namespace rir { +void FrameInfo::deserialize(const ByteBuffer& buf, + const SerialOptions& serialOpts) { + code = Code::unpack(rir::deserialize(buf, serialOpts)); + pc = code->code() + buf.getInt(); + stackSize = (size_t)buf.getInt(); + inPromise = (bool)buf.getInt(); +} + +void FrameInfo::serialize(ByteBuffer& buf, + const SerialOptions& serialOpts) const { + rir::serialize(code->container(), buf, serialOpts); + buf.putInt((uint32_t)(pc - code->code())); + buf.putInt((uint32_t)stackSize); + buf.putInt((uint32_t)inPromise); +} + +void FrameInfo::gcAttach(Code* outer) const { + outer->addExtraPoolEntry(code->container()); +} + +SEXP DeoptMetadata::container() const { + // cppcheck-suppress thisSubtraction + SEXP result = (SEXP)((uintptr_t)this - sizeof(VECTOR_SEXPREC)); + assert(TYPEOF(result) == RAWSXP && "DeoptMetadata not embedded in container, or corrupt."); + return result; +} + +DeoptMetadata* DeoptMetadata::deserialize(const ByteBuffer& buf, + const SerialOptions& serialOpts) { + auto numFrames = (size_t)buf.getInt(); + auto size = sizeof(DeoptMetadata) + numFrames * sizeof(FrameInfo); + SEXP store = Rf_allocVector(RAWSXP, (int)size); + PROTECT(store); + auto m = new (DATAPTR(store)) DeoptMetadata; + m->numFrames = numFrames; + for (size_t i = 0; i < numFrames; ++i) { + m->frames[i].deserialize(buf, serialOpts); + PROTECT(m->frames[i].code->container()); + } + UNPROTECT(1 + m->numFrames); + return m; +} + +void DeoptMetadata::serialize(ByteBuffer& buf, + const SerialOptions& serialOpts) const { + buf.putInt((uint32_t)numFrames); + for (size_t i = 0; i < numFrames; ++i) { + frames[i].serialize(buf, serialOpts); + } +} + +void DeoptMetadata::gcAttach(Code* outer) const { + outer->addExtraPoolEntry(this->container()); + for (size_t i = 0; i < numFrames; ++i) { + frames[i].gcAttach(outer); + } +} + void DeoptMetadata::print(std::ostream& out) const { for (size_t i = 0; i < numFrames; ++i) { auto f = frames[i]; diff --git a/rir/src/runtime/Deoptimization.h b/rir/src/runtime/Deoptimization.h index b62104c3b..de01b2724 100644 --- a/rir/src/runtime/Deoptimization.h +++ b/rir/src/runtime/Deoptimization.h @@ -1,9 +1,12 @@ #ifndef RIR_DEOPTIMIZATION_H #define RIR_DEOPTIMIZATION_H +#include "serializeHash/serialize/serialize.h" #include #include +class ByteBuffer; + namespace rir { #pragma pack(push) #pragma pack(1) @@ -17,12 +20,21 @@ struct FrameInfo { size_t stackSize; bool inPromise; - FrameInfo() {} - FrameInfo(Opcode* pc, Code* code, size_t stackSize, bool promise) - : pc(pc), code(code), stackSize(stackSize), inPromise(promise) {} + void deserialize(const ByteBuffer& buf, const SerialOptions& serialOpts); + void serialize(ByteBuffer& buf, const SerialOptions& serialOpts) const; + /// Adds the code object's container to the code's extra pool, so it gets + /// gc-collected when the SEXP does + void gcAttach(Code* outer) const; }; struct DeoptMetadata { + SEXP container() const; + static DeoptMetadata* deserialize(const ByteBuffer& buf, + const SerialOptions& serialOpts); + void serialize(ByteBuffer& buf, const SerialOptions& serialOpts) const; + /// Adds the container and the frame code objects' containers to the code's + /// extra pool, so it gets gc-collected when the SEXP does + void gcAttach(Code* outer) const; void print(std::ostream& out) const; size_t numFrames; FrameInfo frames[]; diff --git a/rir/src/runtime/DispatchTable.cpp b/rir/src/runtime/DispatchTable.cpp new file mode 100644 index 000000000..78a2e9d5f --- /dev/null +++ b/rir/src/runtime/DispatchTable.cpp @@ -0,0 +1,144 @@ +#include "DispatchTable.h" +#include "R/Protect.h" +#include "runtime/log/printPrettyGraph.h" +#include "serializeHash/serialize/serialize.h" + +namespace rir { + +DispatchTable* DispatchTable::deserialize(AbstractDeserializer& deserializer) { + Protect p; + auto dt = create(); + p(dt->container()); + // Magic is already set + deserializer.addRef(dt->container()); + if (deserializer.willRead(SerialFlags::DtContext)) { + dt->userDefinedContext_ = Context( + deserializer.readBytesOf(SerialFlags::DtContext)); + } + dt->size_ = deserializer.willRead(SerialFlags::DtOptimized) + ? deserializer.readBytesOf(SerialFlags::DtOptimized) + : 1; + for (size_t i = 0; i < dt->size(); i++) { + auto version = deserializer.read(i == 0 ? SerialFlags::DtBaseline : SerialFlags::DtOptimized); + Function::unpack(version)->dispatchTable(dt); + dt->setEntry(i, version); + } + return dt; +} + +void DispatchTable::serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf(userDefinedContext_.toI(), SerialFlags::DtContext); + serializer.writeBytesOf((int)size(), SerialFlags::DtOptimized); + size_t n = serializer.willWrite(SerialFlags::DtOptimized) ? size() : 1; + for (size_t i = 0; i < n; i++) { + serializer.write(getEntry(i), i == 0 ? SerialFlags::DtBaseline : SerialFlags::DtOptimized); + } +} + +void DispatchTable::hash(HasherOld& hasher) const { + assert(size() > 0); + // Only hash baseline so the hash doesn't change when new entries get added + // (since semantics won't, and other rir objects will reference optimized + // versions directly when they rely on them) + hasher.hash(getEntry(0)); +} + +void DispatchTable::addConnected(ConnectedCollectorOld& collector) const { + assert(size() > 0); + for (size_t i = 0; i < size(); i++) { + collector.add(getEntry(i), false); + } +} + +void DispatchTable::print(std::ostream& out, bool isDetailed) const { // NOLINT(*-no-recursion) + std::cout << "== dispatch table " << this << " ==\n"; + + for (size_t entry = 0; entry < size(); ++entry) { + Function* f = get(entry); + std::cout << "= version " << entry << " (" << f << ") =\n"; + f->disassemble(std::cout); + } + + if (isDetailed && !baseline()->isDeserializing()) { + auto code = baseline()->body(); + auto pc = code->code(); + auto printHeader = true; + + Opcode* prev = nullptr; + Opcode* pprev = nullptr; + + while (pc < code->endCode()) { + auto bc = BC::decode(pc, code); + if (bc.bc == Opcode::close_) { + if (printHeader) { + out << "== nested closures ==\n"; + printHeader = false; + } + + // prev is the push_ of srcref + // pprev is the push_ of body + auto body = BC::decodeShallow(pprev).immediateConst(); + auto dt = DispatchTable::unpack(body); + dt->print(std::cout, isDetailed); + } + pprev = prev; + prev = pc; + pc = BC::next(pc); + } + } +} + +void DispatchTable::printPrettyGraphContent(const PrettyGraphInnerPrinter& print) const { + print.addName([&](std::ostream& s) { s << "DispatchTable(" << size() << ")"; }); + for (size_t i = 0; i < size(); i++) { + print.addEdgeTo(getEntry(i), true, "entry", [&](std::ostream& s) { + s << "entry " << i; + }); + } + + // Add edges to nested closures + { + auto code = baseline()->body(); + auto pc = code->code(); + + Opcode* prev = nullptr; + Opcode* pprev = nullptr; + + while (pc < code->endCode()) { + auto bc = BC::decode(pc, code); + if (bc.bc == Opcode::close_) { + // prev is the push_ of srcref + // pprev is the push_ of body + auto childBody = BC::decodeShallow(pprev).immediateConst(); + print.addEdgeTo(childBody, true, "nested-closure"); + } + pprev = prev; + prev = pc; + pc = BC::next(pc); + } + } +} + +void DispatchTable::debugCompare(const rir::DispatchTable* dt1, + const rir::DispatchTable* dt2, + std::stringstream& differences, + bool compareExtraPoolRBytecodes) { + if (dt1->size() != dt2->size()) { + differences << "DispatchTable size differs: " << dt1->size() << " vs " << dt2->size() << "\n"; + } + for (size_t i = 0; i < dt1->size() && i < dt2->size(); i++) { + std::stringstream funDifferencesStream; + Function::debugCompare( + Function::unpack(dt1->getEntry(i)), + Function::unpack(dt2->getEntry(i)), + funDifferencesStream, + compareExtraPoolRBytecodes + ); + std::string funDifferences = funDifferencesStream.str(); + if (!funDifferences.empty()) { + differences << "DispatchTable entry " << i << " differs:\n" << funDifferences; + } + } +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/DispatchTable.h b/rir/src/runtime/DispatchTable.h index 7043f603e..45ad405d2 100644 --- a/rir/src/runtime/DispatchTable.h +++ b/rir/src/runtime/DispatchTable.h @@ -4,7 +4,14 @@ #include "Function.h" #include "R/Serialize.h" #include "RirRuntimeObject.h" +#include "TypeFeedback.h" +#include "compilerClientServer/CompilerClient.h" +#include "runtime/log/RirObjectPrintStyle.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/hash/hashRootOld.h" +#include "utils/ByteBuffer.h" #include "utils/random.h" +#include namespace rir { @@ -92,20 +99,32 @@ struct DispatchTable assert(baseline()->signature().optimization == FunctionSignature::OptimizationLevel::Baseline); setEntry(0, f->container()); + f->dispatchTable(this); } bool contains(const Context& assumptions) const { + auto i = indexOf(assumptions); + return i != SIZE_MAX && !get(i)->disabled(); + } + + private: + // Note: Also returns index if disabled + size_t indexOf(const Context& assumptions) const { for (size_t i = 0; i < size(); ++i) if (get(i)->context() == assumptions) - return !get(i)->disabled(); - return false; + return i; + return SIZE_MAX; } + public: void remove(Code* funCode) { size_t i = 1; for (; i < size(); ++i) { - if (get(i)->body() == funCode) + auto fun = get(i); + if (fun->body() == funCode) { + fun->dispatchTable(nullptr); break; + } } if (i == size()) return; @@ -122,6 +141,7 @@ struct DispatchTable assert(size() > 0); assert(fun->signature().optimization != FunctionSignature::OptimizationLevel::Baseline); + fun->dispatchTable(this); auto assumptions = fun->context(); size_t i; for (i = size() - 1; i > 0; --i) { @@ -134,6 +154,7 @@ struct DispatchTable setEntry(i, fun->container()); assert(get(i) == fun); } + old->dispatchTable(nullptr); return; } if (!(assumptions < get(i)->context())) { @@ -141,7 +162,19 @@ struct DispatchTable } } i++; - assert(!contains(fun->context())); + if (CompilerClient::isRunning()) { + // Not sure if this even happens or is the right approach, but in + // theory, since only DT baselines are hashed, the compiler server + // could return a DT with an already optimized closure. In this + // case, replacing should be ok + auto indexOfSameContext = indexOf(fun->context()); + if (indexOfSameContext != SIZE_MAX) { + setEntry(indexOfSameContext, fun->container()); + return; + } + } else { + assert(!contains(fun->context())); + } if (size() == capacity()) { #ifdef DEBUG_DISPATCH std::cout << "Tried to insert into a full Dispatch table. Have: \n"; @@ -193,24 +226,19 @@ struct DispatchTable size_t capacity() const { return info.gc_area_length; } - static DispatchTable* deserialize(SEXP refTable, R_inpstream_t inp) { - DispatchTable* table = create(); - PROTECT(table->container()); - AddReadRef(refTable, table->container()); - table->size_ = InInteger(inp); - for (size_t i = 0; i < table->size(); i++) { - table->setEntry(i, - Function::deserialize(refTable, inp)->container()); - } - UNPROTECT(1); - return table; - } + static DispatchTable* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& deserializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + void print(std::ostream&, bool isDetailed = false) const; + void printPrettyGraphContent(const PrettyGraphInnerPrinter& print) const; + /// Check if 2 dispatch tables are the same, for validation and sanity check + /// (before we do operations which will cause weird errors otherwise). If + /// not, will add each difference to differences. + static void debugCompare(const DispatchTable* dt1, const DispatchTable* dt2, + std::stringstream& differences, + bool compareExtraPoolRBytecodes = true); - void serialize(SEXP refTable, R_outpstream_t out) const { - HashAdd(container(), refTable); - OutInteger(out, 1); - baseline()->serialize(refTable, out); - } Context userDefinedContext() const { return userDefinedContext_; } DispatchTable* newWithUserContext(Context udc) { @@ -237,12 +265,12 @@ struct DispatchTable private: DispatchTable() = delete; - explicit DispatchTable(size_t cap) + explicit DispatchTable(size_t capacity) : RirRuntimeObject( // GC area starts at the end of the DispatchTable sizeof(DispatchTable), // GC area is just the pointers in the entry array - cap) {} + capacity) {} size_t size_ = 0; Context userDefinedContext_; diff --git a/rir/src/runtime/Function.cpp b/rir/src/runtime/Function.cpp index 1e6d9ba50..f38c7873c 100644 --- a/rir/src/runtime/Function.cpp +++ b/rir/src/runtime/Function.cpp @@ -1,67 +1,182 @@ #include "Function.h" -#include "R/Serialize.h" +#include "R/Protect.h" +#include "Rinternals.h" #include "compiler/compiler.h" +#include "interpreter/instance.h" +#include "runtime/TypeFeedback.h" +#include "runtime/log/printPrettyGraph.h" namespace rir { -Function* Function::deserialize(SEXP refTable, R_inpstream_t inp) { - size_t functionSize = InInteger(inp); - const FunctionSignature sig = FunctionSignature::deserialize(refTable, inp); - const Context as = Context::deserialize(refTable, inp); - SEXP store = Rf_allocVector(EXTERNALSXP, functionSize); - void* payload = DATAPTR(store); - Function* fun = new (payload) Function(functionSize, nullptr, {}, sig, as); - fun->numArgs_ = InInteger(inp); - fun->info.gc_area_length += fun->numArgs_; - for (unsigned i = 0; i < fun->numArgs_ + 1; i++) { - fun->setEntry(i, R_NilValue); - } - PROTECT(store); - AddReadRef(refTable, store); - SEXP body = Code::deserialize(refTable, inp)->container(); - fun->body(body); - PROTECT(body); - int protectCount = 2; - for (unsigned i = 0; i < fun->numArgs_; i++) { - if ((bool)InInteger(inp)) { - SEXP arg = Code::deserialize(refTable, inp)->container(); - PROTECT(arg); - protectCount++; - fun->setEntry(Function::NUM_PTRS + i, arg); - } else - fun->setEntry(Function::NUM_PTRS + i, nullptr); - } - fun->flags = EnumSet(InInteger(inp)); - UNPROTECT(protectCount); +void Function::setFlag(rir::Function::Flag f) { + // UUIDPool::reintern(container()); + flags_.set(f); +} + +void Function::resetFlag(rir::Function::Flag f) { + // UUIDPool::reintern(container()); + flags_.reset(f); +} + +void Function::deserializeFullSignature(const ByteBuffer& buf) { + signature_.deserializeFrom(buf); + context_ = Context(buf.getLong()); + buf.getBytes((uint8_t*)&flags_, sizeof(flags_)); + invocationCount_ = buf.getInt(); + deoptCount_ = buf.getInt(); + deadCallReached_ = buf.getInt(); + // invoked = buf.getLong(); + // execTime = buf.getLong(); +} + +void Function::serializeFullSignature(ByteBuffer& buf) const { + signature_.serialize(buf); + buf.putLong(context_.toI()); + buf.putBytes((uint8_t*)&flags_, sizeof(flags_)); + // Misc bytes = whether counts exceed certain values checked by rir2pir. + // Stats = actual counts and invocation time + buf.putInt(std::min(invocationCount_, 2u)); + buf.putInt(std::min(deoptCount_, 2u)); + buf.putInt(std::min(deadCallReached_, 4u)); + // buf.putLong(invoked); + // buf.putLong(execTime); +} + +Function* Function::deserialize(AbstractDeserializer& deserializer) { + Protect p; + auto funSize = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + auto sig = FunctionSignature::deserialize(deserializer); + auto ctx = Context(deserializer.readBytesOf(SerialFlags::FunMiscBytes)); + auto flags = EnumSet(deserializer.readBytesOf(SerialFlags::FunMiscBytes)); + // Misc bytes = whether counts exceed certain values checked by rir2pir. + // Stats = actual counts and invocation time + auto invocationCount_ = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + auto deoptCount_ = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + auto deadCallReached_ = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + if (deserializer.willRead(SerialFlags::FunStats)) { + invocationCount_ = + deserializer.readBytesOf(SerialFlags::FunStats); + deoptCount_ = deserializer.readBytesOf(SerialFlags::FunStats); + deadCallReached_ = + deserializer.readBytesOf(SerialFlags::FunStats); + } + auto invoked = deserializer.readBytesOf(SerialFlags::FunStats); + auto execTime = deserializer.readBytesOf(SerialFlags::FunStats); + SEXP store = p(Rf_allocVector(EXTERNALSXP, funSize)); + // There's an interesting situation where we start using the function WHILE + // it's being deserialized (recursive deserialization madness), so we have + // to make `Function::unpack` not crash by making `store` have the function + // magic, and we have to make fun->typeFeedback() return nullptr. + // + // That's what these assignments do. Fortunately we don't try to use + // anything else... + *((rir_header*)STDVEC_DATAPTR(store)) = + {sizeof(Function) - NUM_PTRS * sizeof(SEXP), + NUM_PTRS, + FUNCTION_MAGIC}; + for (unsigned i = 0; i < NUM_PTRS; i++) { + EXTERNALSXP_SET_ENTRY(store, (int)i, nullptr); + } + // Also needed to set FUNCTION_MAGIC for addRef + deserializer.addRef(store); + + auto feedback = p(deserializer.read(SerialFlags::FunFeedback)); + auto body = p(deserializer.read(SerialFlags::FunBody)); + std::vector defaultArgs(sig.numArguments, nullptr); + for (unsigned i = 0; i < sig.numArguments; i++) { + if (deserializer.readBytesOf(SerialFlags::FunDefaultArg)) { + defaultArgs[i] = p(deserializer.read(SerialFlags::FunDefaultArg)); + } + } + + auto fun = new (DATAPTR(store)) + Function(funSize, body, defaultArgs, sig, ctx, + TypeFeedback::unpack(feedback)); + fun->flags_ = flags; + fun->invocationCount_ = invocationCount_; + fun->deoptCount_ = deoptCount_; + fun->deadCallReached_ = deadCallReached_; + fun->invoked = invoked; + fun->execTime = execTime; return fun; } -void Function::serialize(SEXP refTable, R_outpstream_t out) const { - OutInteger(out, size); - signature().serialize(refTable, out); - context_.serialize(refTable, out); - OutInteger(out, numArgs_); - HashAdd(container(), refTable); - body()->serialize(refTable, out); +void Function::serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf((R_xlen_t)size, SerialFlags::FunMiscBytes); + signature().serialize(serializer); + serializer.writeBytesOf(context_.toI(), SerialFlags::FunMiscBytes); + serializer.writeBytesOf(flags_.to_i(), SerialFlags::FunMiscBytes); + // Misc bytes = whether counts exceed certain values checked by rir2pir. + // Stats = actual counts and invocation time + serializer.writeBytesOf(std::min(invocationCount_, 2u), SerialFlags::FunMiscBytes); + serializer.writeBytesOf(std::min(deoptCount_, 2u), SerialFlags::FunMiscBytes); + serializer.writeBytesOf(std::min(deadCallReached_, 4u), SerialFlags::FunMiscBytes); + serializer.writeBytesOf(invocationCount_, SerialFlags::FunStats); + serializer.writeBytesOf(deoptCount_, SerialFlags::FunStats); + serializer.writeBytesOf(deadCallReached_, SerialFlags::FunStats); + serializer.writeBytesOf(invoked, SerialFlags::FunStats); + serializer.writeBytesOf(execTime, SerialFlags::FunStats); + + serializer.write(typeFeedback()->container(), SerialFlags::FunFeedback); + serializer.write(body()->container(), SerialFlags::FunBody); for (unsigned i = 0; i < numArgs_; i++) { - Code* arg = defaultArg(i); - OutInteger(out, (int)(arg != nullptr)); - if (arg) - defaultArg(i)->serialize(refTable, out); + serializer.writeBytesOf(defaultArg_[i] != nullptr, SerialFlags::FunDefaultArg); + if (defaultArg_[i]) { + serializer.write(defaultArg_[i], SerialFlags::FunDefaultArg); + } } - OutInteger(out, flags.to_i()); } -void Function::disassemble(std::ostream& out) { - out << "[sigature] "; +void Function::hash(HasherOld& hasher) const { + hasher.hashBytesOf(signature()); + hasher.hashBytesOf(context_); + hasher.hashBytesOf(numArgs_); + // TODO: why are body and args not set sometimes when we hash + // deserialized value to check hash consistency? It probably has + // something to do with cyclic references in serialization, but why? + // (This is one of the reasons we use SEXP instead of unpacking Code + // for body and default args, also because we are going to serialize + // the SEXP anyways to properly handle cyclic references) + hasher.hash(getEntry(0)); + + for (unsigned i = 0; i < numArgs_; i++) { + CodeSEXP arg = defaultArg_[i]; + hasher.hashNullable(arg); + } + + // Don't hash flags because they change +} + +void Function::addConnected(ConnectedCollectorOld& collector) const { + collector.add(getEntry(0), false); + + for (unsigned i = 0; i < numArgs_; i++) { + CodeSEXP arg = defaultArg_[i]; + collector.addNullable(arg, false); + } +} + +void Function::disassemble(std::ostream& out) const { + print(out); +} + +void Function::print(std::ostream& out, bool isDetailed) const { + if (isDeserializing()) { + out << "(function is being deserialized)\n"; + return; + } + if (isDetailed) { + out << "[size]" << size << "\n[numArgs] " << numArgs_ << "\n"; + } + out << "[signature] "; signature().print(out); if (!context_.empty()) out << "| context: [" << context_ << "]"; out << "\n"; out << "[flags] "; #define V(F) \ - if (flags.includes(F)) \ - out << #F << " "; +if (flags_.includes(F)) \ + out << #F << " "; RIR_FUNCTION_FLAGS(V) #undef V out << "\n"; @@ -70,7 +185,138 @@ void Function::disassemble(std::ostream& out) { << ", time: " << ((double)invocationTime() / 1e6) << "ms, deopt: " << deoptCount(); out << "\n"; - body()->disassemble(out); + if (isDetailed) { + body()->print(out, isDetailed); + for (unsigned i = 0; i < numArgs_; i++) { + CodeSEXP arg = defaultArg_[i]; + if (arg) { + out << "[default arg " << i << "]\n"; + Code::unpack(arg)->print(out, isDetailed); + } + } + } else { + body()->disassemble(out); + } + out << "[feedback]\n"; + typeFeedback()->print(out); +} + +void Function::printPrettyGraphContent(const PrettyGraphInnerPrinter& print) const { + print.addName([&](std::ostream& s) { + auto ast = src_pool_at(body()->src); + auto headAst = TYPEOF(ast) == LANGSXP ? CAR(ast) : R_NilValue; + if (TYPEOF(headAst) == SYMSXP) { + s << CHAR(PRINTNAME(headAst)); + } else { + s << ""; + } + }); + print.addBody([&](std::ostream& s) { + s << "

("; + signature().print(s); + s << ")

"; + if (!context_.empty()) { + s << "

[" << context_ + << "]

"; + } + if (!flags_.empty()) { + s << "

{"; + } +#define V(F) \ + if (flags_.includes(F)) \ + s << #F << " "; + RIR_FUNCTION_FLAGS(V) +#undef V + if (!flags_.empty()) { + s << "}

"; + } + s << "

" + << "invoked: " << invocationCount() + << ", time: " << ((double)invocationTime() / 1e6) + << "ms, deopt: " << deoptCount() + << "

"; + }); + print.addEdgeTo(body()->container(), true, "body"); + for (unsigned i = 0; i < numArgs_; i++) { + CodeSEXP arg = defaultArg_[i]; + if (arg) { + print.addEdgeTo(arg, true, "default-arg", [&](std::ostream& s) { + s << "arg " << i << " default"; + }); + } + } +} + +void Function::debugCompare(const Function* f1, const Function* f2, + std::stringstream& differences, + bool compareExtraPoolRBytecodes) { + FunctionSignature::debugCompare(f1->signature(), f2->signature(), differences); + if (f1->context() != f2->context()) { + differences << "context: " << f1->context() << " != " << f2->context() + << "\n"; + } + if (f1->flags() != f2->flags()) { + differences << "flags: "; +#define V(F) \ + if (f1->flags_.includes(F)) \ + differences << #F << " "; + RIR_FUNCTION_FLAGS(V) +#undef V + differences << " != "; +#define V(F) \ + if (f2->flags_.includes(F)) \ + differences << #F << " "; + RIR_FUNCTION_FLAGS(V) +#undef V + differences << "\n"; + } + if (f1->size != f2->size) { + differences << "size: " << f1->size << " != " << f2->size << "\n"; + } + if (f1->numArgs_ != f2->numArgs_) { + differences << "numArgs: " << f1->numArgs_ << " != " << f2->numArgs_ + << "(note: signature also has numArgs)\n"; + } + // TODO: invocationCount, invoked, and execTime are frequently different, + // even when doing a deep copy. Why? + if (f1->invocationCount_ != f2->invocationCount_) { + differences << "invocationCount: " << f1->invocationCount_ + << " != " << f2->invocationCount_ << "\n"; + } + if (f1->deoptCount_ != f2->deoptCount_) { + differences << "deoptCount: " << f1->deoptCount_ + << " != " << f2->deoptCount_ << "\n"; + } + if (f1->deadCallReached_ != f2->deadCallReached_) { + differences << "deadCallReached: " << f1->deadCallReached_ + << " != " << f2->deadCallReached_ << "\n"; + } + if (f1->invoked != f2->invoked) { + differences << "invoked: " << f1->invoked << " != " << f2->invoked + << "\n"; + } + if (f1->execTime != f2->execTime) { + differences << "invocationTime: " << f1->execTime + << " != " << f2->execTime << "\n"; + } + Code::debugCompare(f1->body(), f2->body(), "body", differences, + compareExtraPoolRBytecodes); + for (unsigned i = 0; i < std::min(f1->numArgs_, f2->numArgs_); i++) { + auto arg1 = f1->defaultArg_[i]; + auto arg2 = f2->defaultArg_[i]; + auto hasArg1 = (arg1 != nullptr); + auto hasArg2 = (arg2 != nullptr); + if (hasArg1 != hasArg2) { + differences << "defaultArg[" << i << "] != nullptr: " << hasArg1 + << " != " << hasArg2 << "\n"; + } + if (hasArg1 && hasArg2) { + char prefix[100]; + sprintf(prefix, "defaultArg[%u]", i); + Code::debugCompare(Code::unpack(arg1), Code::unpack(arg2), + prefix, differences, compareExtraPoolRBytecodes); + } + } } static int GLOBAL_SPECIALIZATION_LEVEL = @@ -78,11 +324,11 @@ static int GLOBAL_SPECIALIZATION_LEVEL = ? atoi(getenv("PIR_GLOBAL_SPECIALIZATION_LEVEL")) : 100; void Function::clearDisabledAssumptions(Context& given) const { - if (flags.contains(Function::DisableArgumentTypeSpecialization)) + if (flags_.contains(Function::DisableArgumentTypeSpecialization)) given.clearTypeFlags(); - if (flags.contains(Function::DisableNumArgumentsSpezialization)) + if (flags_.contains(Function::DisableNumArgumentsSpezialization)) given.clearNargs(); - if (flags.contains(Function::DisableAllSpecialization)) + if (flags_.contains(Function::DisableAllSpecialization)) given.clearExcept(pir::Compiler::minimalContext); if (GLOBAL_SPECIALIZATION_LEVEL < 100) diff --git a/rir/src/runtime/Function.h b/rir/src/runtime/Function.h index 04d3ea0d3..22c5b8ada 100644 --- a/rir/src/runtime/Function.h +++ b/rir/src/runtime/Function.h @@ -5,13 +5,18 @@ #include "FunctionSignature.h" #include "R/r.h" #include "RirRuntimeObject.h" +#include "runtime/log/RirObjectPrintStyle.h" +#include "serializeHash/hash/hashRootOld.h" +#include "utils/ByteBuffer.h" +#include "runtime/TypeFeedback.h" namespace rir { +struct DispatchTable; + /** * Aliases for readability. */ -typedef SEXP FunctionSEXP; // Function magic constant is designed to help to distinguish between Function // objects and normal EXTERNALSXPs. Normally this is not necessary, but a very @@ -39,28 +44,72 @@ struct Function : public RirRuntimeObject { friend class FunctionCodeIterator; friend class ConstFunctionCodeIterator; - static constexpr size_t NUM_PTRS = 1; + // In its entries, a function ows two SEXP pointers + a variable length of + // default arguments code: + static constexpr size_t NUM_PTRS = 2; + // 0: body (Code*) + static constexpr size_t BODY_IDX = 0; + // 1: type feedback (TypeFeedback*) + static constexpr size_t TYPE_FEEDBACK_IDX = 1; Function(size_t functionSize, SEXP body_, const std::vector& defaultArgs, - const FunctionSignature& signature, const Context& ctx) + const FunctionSignature& signature, const Context& ctx, + TypeFeedback* feedback) : RirRuntimeObject( // GC area starts at &locals and goes to the end of defaultArg_ - sizeof(Function) - NUM_PTRS * sizeof(FunctionSEXP), + sizeof(Function) - NUM_PTRS * sizeof(SEXP), NUM_PTRS + defaultArgs.size()), size(functionSize), numArgs_(defaultArgs.size()), signature_(signature), context_(ctx) { for (size_t i = 0; i < numArgs_; ++i) setEntry(NUM_PTRS + i, defaultArgs[i]); - body(body_); + if (body_) { + body(body_); + } else { + // Happens when we create a function in deserialization + assert(functionSize == 0); + } + if (feedback) { + typeFeedback(feedback); + } } - Code* body() const { return Code::unpack(getEntry(0)); } - void body(SEXP body) { setEntry(0, body); } + Code* body() const { return Code::unpack(getEntry(BODY_IDX)); } + void body(SEXP body) { + assert(body); + assert(Code::check(body)); + setEntry(BODY_IDX, body); + } - static Function* deserialize(SEXP refTable, R_inpstream_t inp); - void serialize(SEXP refTable, R_outpstream_t out) const; - void disassemble(std::ostream&); + bool isDeserializing() const { + return !getEntry(BODY_IDX); + } + TypeFeedback* typeFeedback() const { + return TypeFeedback::unpack(getEntry(TYPE_FEEDBACK_IDX)); + } + void typeFeedback(TypeFeedback* typeFeedback) { + typeFeedback->owner_ = this; + setEntry(TYPE_FEEDBACK_IDX, typeFeedback->container()); + } + + /// "Full signature" include context, flags, and invocation info + void serializeFullSignature(ByteBuffer& buf) const; + /// "Full signature" include context, flags, and invocation info + void deserializeFullSignature(const ByteBuffer& buf); + static Function* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& deserializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + void disassemble(std::ostream&) const; + void print(std::ostream&, bool isDetailed = false) const; + void printPrettyGraphContent(const PrettyGraphInnerPrinter& print) const; + /// Check if 2 functions are the same, for validation and sanity check + /// (before we do operations which will cause weird errors otherwise). If + /// not, will add each difference to differences. + static void debugCompare(const Function* f1, const Function* f2, + std::stringstream& differences, + bool compareExtraPoolRBytecodes = true); bool isOptimized() const { return signature_.optimization != @@ -74,15 +123,21 @@ struct Function : public RirRuntimeObject { return Code::unpack(defaultArg_[i]); } - size_t invocationCount() { return invocationCount_; } + size_t invocationCount() const { return invocationCount_; } - size_t deoptCount() { return deoptCount_; } + size_t deoptCount() const { return deoptCount_; } void addDeoptCount(size_t n) { deoptCount_ += n; } static inline unsigned long rdtsc() { +#ifdef __ARM_ARCH + uint64_t val; + asm volatile("mrs %0, cntvct_el0" : "=r" (val)); + return val; +#else unsigned low, high; asm volatile("rdtsc" : "=a"(low), "=d"(high)); return ((low) | ((uint64_t)(high) << 32)); +#endif } static constexpr unsigned long MAX_TIME_MEASURE = 1e9; @@ -110,7 +165,7 @@ struct Function : public RirRuntimeObject { invoked = 0; } } - unsigned long invocationTime() { return execTime; } + unsigned long invocationTime() const { return execTime; } void clearInvocationTime() { execTime = 0; } unsigned size; /// Size, in bytes, of the function and its data @@ -135,10 +190,15 @@ struct Function : public RirRuntimeObject { RIR_FUNCTION_FLAGS(V) #undef V - FIRST = Deopt, + FIRST = Deopt, LAST = DisableNumArgumentsSpezialization }; - EnumSet flags; + private: + EnumSet flags_; + public: + const EnumSet& flags() const { return flags_; } + void setFlag(Flag f); + void resetFlag(Flag f); void inheritFlags(const Function* other) { static Flag inherited[] = {ForceInline, @@ -147,10 +207,10 @@ struct Function : public RirRuntimeObject { DisableArgumentTypeSpecialization, DisableNumArgumentsSpezialization, DepromiseArgs}; - auto f = other->flags; + auto f = other->flags_; for (auto flag : inherited) if (f.contains(flag)) - flags.set(flag); + setFlag(flag); } void clearDisabledAssumptions(Context& given) const; @@ -161,13 +221,13 @@ struct Function : public RirRuntimeObject { const FunctionSignature& signature() const { return signature_; } const Context& context() const { return context_; } - bool disabled() const { return flags.contains(Flag::Deopt); } + bool disabled() const { return flags_.contains(Flag::Deopt); } bool pendingCompilation() const { return body()->pendingCompilation(); } void registerDeopt() { // Deopt counts are kept on the optimized versions assert(isOptimized()); - flags.set(Flag::Deopt); + setFlag(Flag::Deopt); if (deoptCount_ < UINT_MAX) deoptCount_++; } @@ -178,7 +238,7 @@ struct Function : public RirRuntimeObject { if (r == DeoptReason::DeadCall) deadCallReached_++; if (r == DeoptReason::EnvStubMaterialized) - flags.set(NeedsFullEnv); + setFlag(NeedsFullEnv); } size_t deadCallReached() const { @@ -186,6 +246,9 @@ struct Function : public RirRuntimeObject { return deadCallReached_; } + void dispatchTable(DispatchTable* dt) { dispatchTable_ = dt; } + DispatchTable* dispatchTable() { return dispatchTable_; } + private: unsigned numArgs_; @@ -199,11 +262,12 @@ struct Function : public RirRuntimeObject { FunctionSignature signature_; /// pointer to this version's signature Context context_; + DispatchTable* dispatchTable_; // !!! SEXPs traceable by the GC must be declared here !!! - // locals contains: body - CodeSEXP locals[NUM_PTRS]; - CodeSEXP defaultArg_[]; + // locals contains: body (BODY_IDX) and typeFeedback (TYPE_FEEDBACK_IDX) + SEXP locals[NUM_PTRS]; + SEXP defaultArg_[]; }; #pragma pack(pop) diff --git a/rir/src/runtime/FunctionSignature.h b/rir/src/runtime/FunctionSignature.h index bb286a94b..14572a7d2 100644 --- a/rir/src/runtime/FunctionSignature.h +++ b/rir/src/runtime/FunctionSignature.h @@ -2,9 +2,12 @@ #include "R/Serialize.h" #include "R/r.h" +#include "serializeHash/serializeUni.h" +#include "utils/ByteBuffer.h" #include #include +#include #include namespace rir { @@ -21,27 +24,81 @@ struct FunctionSignature { Contextual, }; - static FunctionSignature deserialize(SEXP refTable, R_inpstream_t inp) { - Environment envc = (Environment)InInteger(inp); - OptimizationLevel opt = (OptimizationLevel)InInteger(inp); - unsigned numArgs = InInteger(inp); + static FunctionSignature deserialize(__attribute__((unused)) SEXP refTable, + R_inpstream_t inp) { + auto envc = (Environment)InInteger(inp); + auto opt = (OptimizationLevel)InInteger(inp); FunctionSignature sig(envc, opt); - sig.numArguments = numArgs; - sig.dotsPosition = InInteger(inp); + sig.numArguments = InUInt(inp); + sig.dotsPosition = InU64(inp); sig.hasDotsFormals = InInteger(inp); sig.hasDefaultArgs = InInteger(inp); return sig; } - void serialize(SEXP refTable, R_outpstream_t out) const { + void serialize(__attribute__((unused)) SEXP refTable, R_outpstream_t out) const { OutInteger(out, (int)envCreation); OutInteger(out, (int)optimization); - OutInteger(out, numArguments); - OutInteger(out, dotsPosition); + OutUInt(out, numArguments); + OutU64(out, dotsPosition); OutInteger(out, hasDotsFormals); OutInteger(out, hasDefaultArgs); } + static FunctionSignature deserialize(AbstractDeserializer& deserializer) { + auto envc = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + auto opt = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + FunctionSignature sig(envc, opt); + sig.numArguments = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + sig.dotsPosition = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + sig.hasDotsFormals = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + sig.hasDefaultArgs = deserializer.readBytesOf(SerialFlags::FunMiscBytes); + return sig; + } + + void serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf(envCreation, SerialFlags::FunMiscBytes); + serializer.writeBytesOf(optimization, SerialFlags::FunMiscBytes); + serializer.writeBytesOf(numArguments, SerialFlags::FunMiscBytes); + serializer.writeBytesOf(dotsPosition, SerialFlags::FunMiscBytes); + serializer.writeBytesOf(hasDotsFormals, SerialFlags::FunMiscBytes); + serializer.writeBytesOf(hasDefaultArgs, SerialFlags::FunMiscBytes); + } + + static FunctionSignature deserialize(const ByteBuffer& buffer) { + auto envc = (Environment)buffer.getInt(); + auto opt = (OptimizationLevel)buffer.getInt(); + FunctionSignature sig(envc, opt); + sig.numArguments = buffer.getInt(); + sig.dotsPosition = buffer.getLong(); + sig.hasDotsFormals = buffer.getBool(); + sig.hasDefaultArgs = buffer.getBool(); + return sig; + } + + /// Deserialize buffer into this, and assert that const fields match. + void deserializeFrom(const ByteBuffer& buffer) { + auto envc = (Environment)buffer.getInt(); + auto opt = (OptimizationLevel)buffer.getInt(); + assert(envc == envCreation && + "FunctionSignature deserialized with different environment"); + assert(opt == optimization && + "FunctionSignature deserialized with different optimization"); + numArguments = buffer.getInt(); + dotsPosition = buffer.getLong(); + hasDotsFormals = buffer.getBool(); + hasDefaultArgs = buffer.getBool(); + } + + void serialize(ByteBuffer& buffer) const { + buffer.putInt((uint32_t)envCreation); + buffer.putInt((uint32_t)optimization); + buffer.putInt(numArguments); + buffer.putLong(dotsPosition); + buffer.putBool(hasDotsFormals); + buffer.putBool(hasDefaultArgs); + } + void pushFormal(SEXP arg, SEXP name) { if (arg != R_MissingArg) hasDefaultArgs = true; @@ -59,6 +116,36 @@ struct FunctionSignature { out << "needsEnv "; } + /// Compare two signatures and print the differences to the given stream. + static void debugCompare(const FunctionSignature& f1, + const FunctionSignature& f2, + std::stringstream& differences) { + if (f1.envCreation != f2.envCreation) { + differences << "envCreation: " << (int)f1.envCreation << " != " + << (int)f2.envCreation << std::endl; + } + if (f1.optimization != f2.optimization) { + differences << "optimization: " << (int)f1.optimization << " != " + << (int)f2.optimization << std::endl; + } + if (f1.numArguments != f2.numArguments) { + differences << "numArguments: " << f1.numArguments << " != " + << f2.numArguments << std::endl; + } + if (f1.hasDotsFormals != f2.hasDotsFormals) { + differences << "hasDotsFormals: " << f1.hasDotsFormals << " != " + << f2.hasDotsFormals << std::endl; + } + if (f1.hasDefaultArgs != f2.hasDefaultArgs) { + differences << "hasDefaultArgs: " << f1.hasDefaultArgs << " != " + << f2.hasDefaultArgs << std::endl; + } + if (f1.dotsPosition != f2.dotsPosition) { + differences << "dotsPosition: " << f1.dotsPosition << " != " + << f2.dotsPosition << std::endl; + } + } + public: FunctionSignature() = delete; FunctionSignature(Environment envCreation, OptimizationLevel optimization) diff --git a/rir/src/runtime/LazyArglist.cpp b/rir/src/runtime/LazyArglist.cpp new file mode 100644 index 000000000..3d9fc26a1 --- /dev/null +++ b/rir/src/runtime/LazyArglist.cpp @@ -0,0 +1,153 @@ +#include "LazyArglist.h" +#include "R/Protect.h" + +namespace rir { + +// cppcheck is wrong, this can't be const +// cppcheck-suppress constParameter +R_bcstack_t deserializeStackArg(Protect& p, AbstractDeserializer& deserializer) { + R_bcstack_t res; + res.tag = deserializer.readBytesOf(); + res.flags = deserializer.readBytesOf(); + auto isSexpArg = deserializer.readBytesOf(); + if (isSexpArg) { + res.u.sxpval = p(deserializer.read()); + } else { + deserializer.readBytes(&res.u, sizeof(res.u)); + } + return res; +} + +void serializeStackArg(const R_bcstack_t& stackArg, AbstractSerializer& serializer) { + auto isSexpArg = stackArg.tag == 0; + serializer.writeBytesOf(stackArg.tag); + serializer.writeBytesOf(stackArg.flags); + serializer.writeBytesOf(isSexpArg); + if (isSexpArg) { + serializer.write(stackArg.u.sxpval); + } else { + serializer.writeBytes(&stackArg.u, sizeof(stackArg.u)); + } +} + +void hashStackArg(const R_bcstack_t& stackArg, HasherOld& hasher) { + auto isSexpArg = stackArg.tag == 0; + hasher.hashBytesOf(stackArg.tag); + hasher.hashBytesOf(stackArg.flags); + hasher.hashBytesOf(isSexpArg); + if (isSexpArg) { + hasher.hash(stackArg.u.sxpval); + } else { + hasher.hashBytes(&stackArg.u, sizeof(stackArg.u)); + } +} + +void addConnectedStackArg(const R_bcstack_t& stackArg, + ConnectedCollectorOld& collector) { + auto isSexpArg = stackArg.tag == 0; + if (isSexpArg) { + collector.add(stackArg.u.sxpval, false); + } +} + +LazyArglist* LazyArglist::deserialize(AbstractDeserializer& deserializer) { + Protect p; + auto size = deserializer.readBytesOf(); + SEXP store = p(Rf_allocVector(EXTERNALSXP, size)); + // Need to ensure magic is correct so that we know this is internable + ((rir_header*)STDVEC_DATAPTR(store))->magic = LAZY_ARGS_MAGIC; + deserializer.addRef(store); + + auto callId = deserializer.readBytesOf(); + auto length = deserializer.readBytesOf(); + auto onStack = deserializer.readBytesOf(); + auto args = new R_bcstack_t[length]; + if (onStack) { + for (size_t i = 0; i < length; ++i) { + args[i] = deserializeStackArg(p, deserializer); + } + } else { + for (size_t i = 0; i < length; ++i) { + args[i] = {0, 0, {.sxpval = p(deserializer.read())}}; + } + } + auto ast = p(deserializer.read(SerialFlags::Ast)); + auto reordering = p(deserializer.read()); + + auto arglist = new (DATAPTR(store)) LazyArglist(callId, reordering, length, args, ast, onStack); + + // Otherwise it's owned by LazyArglist. But is this a leak? + if (!onStack) { + delete[] args; + } + + return arglist; +} + +void LazyArglist::serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf((R_xlen_t)size()); + serializer.writeBytesOf(callId); + serializer.writeBytesOf(length); + // actualNargs is a lazily-computed value, and we don't want laziness to + // affect serialization + serializer.writeBytesOf(stackArgs != nullptr); + if (stackArgs) { + for (size_t i = 0; i < length; ++i) { + serializeStackArg(stackArgs[i], serializer); + } + } else { + for (size_t i = 0; i < length; ++i) { + auto heapArg = heapArgs[i]; + // This invariant isn't clear but it holds + SLOWASSERT(heapArg == getEntry(i + 1)); + serializer.write(heapArg); + } + serializer.write(ast, SerialFlags::Ast); + serializer.write(reordering); + } +} + +void LazyArglist::hash(HasherOld& hasher) const { + hasher.hashBytesOf(callId); + hasher.hashBytesOf(length); + // actualNargs is a lazily-computed value, and we don't want laziness to + // affect hashing + hasher.hashBytesOf(stackArgs != nullptr); + if (stackArgs) { + for (size_t i = 0; i < length; ++i) { + hashStackArg(stackArgs[i], hasher); + } + } else { + for (size_t i = 0; i < length; ++i) { + auto heapArg = heapArgs[i]; + // This invariant isn't clear but it holds + SLOWASSERT(heapArg == getEntry(i + 1)); + hasher.hash(heapArg); + } + hasher.hash(ast, false); + hasher.hash(reordering); + } +} + +void LazyArglist::addConnected(ConnectedCollectorOld& collector) const { + if (stackArgs) { + for (size_t i = 0; i < length; ++i) { + addConnectedStackArg(stackArgs[i], collector); + } + } else { + for (size_t i = 0; i < length; ++i) { + auto heapArg = heapArgs[i]; + // This invariant isn't clear but it holds + SLOWASSERT(heapArg == getEntry(i + 1)); + collector.add(heapArg, false); + } + collector.add(ast, false); + collector.add(reordering, true); + } +} + +size_t LazyArglist::size() const { + return sizeof(LazyArglist) + (stackArgs ? 0 : length * sizeof(SEXP)); +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/LazyArglist.h b/rir/src/runtime/LazyArglist.h index 6ffe2158c..11a56980f 100644 --- a/rir/src/runtime/LazyArglist.h +++ b/rir/src/runtime/LazyArglist.h @@ -5,6 +5,7 @@ #include "runtime/RirRuntimeObject.h" #include "interpreter/interp_incl.h" +#include "serializeHash/serializeUni.h" #include #include @@ -72,6 +73,11 @@ struct LazyArglist : public RirRuntimeObject { true); } + static LazyArglist* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& deserializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + private: // cppcheck-suppress uninitMemberVarPrivate LazyArglist(ArglistOrder::CallId id, SEXP arglistOrder, size_t length, @@ -97,6 +103,8 @@ struct LazyArglist : public RirRuntimeObject { } } + size_t size() const; + friend struct LazyArglistOnHeap; friend struct LazyArglistOnStack; diff --git a/rir/src/runtime/LazyEnvironment.cpp b/rir/src/runtime/LazyEnvironment.cpp index f96088c4a..dcd5534d8 100644 --- a/rir/src/runtime/LazyEnvironment.cpp +++ b/rir/src/runtime/LazyEnvironment.cpp @@ -1,9 +1,10 @@ #include "LazyEnvironment.h" +#include "R/Protect.h" #include "utils/Pool.h" namespace rir { -size_t LazyEnvironment::getArgIdx(SEXP n) { +size_t LazyEnvironment::getArgIdx(SEXP n) const { size_t i = 0; while (i < nargs) { auto name = Pool::get(names[i]); @@ -16,23 +17,105 @@ size_t LazyEnvironment::getArgIdx(SEXP n) { return i; } -SEXP LazyEnvironment::getArg(SEXP n) { +SEXP LazyEnvironment::getArg(SEXP n) const { auto i = getArgIdx(n); if (i == nargs) return R_UnboundValue; return getArg(i); } -bool LazyEnvironment::isMissing(SEXP n) { +bool LazyEnvironment::isMissing(SEXP n) const { auto i = getArgIdx(n); if (i == nargs) return false; return isMissing(i); } -bool LazyEnvironment::isMissing(size_t i) { +bool LazyEnvironment::isMissing(size_t i) const { assert(i < nargs); return missing[i] || getArg(i) == R_MissingArg; } +LazyEnvironment* LazyEnvironment::deserialize(AbstractDeserializer& deserializer) { + Protect p; + auto size = deserializer.readBytesOf(); + SEXP store = p(Rf_allocVector(EXTERNALSXP, size)); + // Need to ensure magic is correct so that we know this is internable + ((rir_header*)STDVEC_DATAPTR(store))->magic = LAZY_ENVIRONMENT_MAGIC; + deserializer.addRef(store); + + auto nargs = deserializer.readBytesOf(); + auto missing = new char[nargs]; + auto names = new Immediate[nargs]; + for (int i = 0; i < nargs; i++) { + missing[i] = deserializer.readBytesOf(); + } + for (int i = 0; i < nargs; i++) { + names[i] = deserializer.readConst(); + } + SEXP materialized = p.nullable(deserializer.readNullable()); + SEXP parent = p.nullable(deserializer.readNullable()); + auto le = new (DATAPTR(store)) LazyEnvironment(parent, nargs, names); + le->materialized(materialized); + for (int i = 0; i < nargs; i++) { + le->missing[i] = missing[i]; + le->setArg(i, deserializer.readNullable(), false); + } + delete[] missing; + // names won't get deleted because its now owned by LazyEnvironment, + // but does LazyEnvironment free when destroyed? + return le; +} + +void LazyEnvironment::serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf((R_xlen_t)size()); + serializer.writeBytesOf((int)nargs); + for (int i = 0; i < (int)nargs; i++) { + serializer.writeBytesOf(missing[i]); + } + for (int i = 0; i < (int)nargs; i++) { + serializer.writeConst(names[i]); + } + serializer.writeNullable(materialized()); + // TODO: Why are getParent() and getArg(i) null after deopt in pir_regression_check_code.R? + serializer.writeNullable(getParent()); + for (int i = 0; i < (int)nargs; i++) { + serializer.writeNullable(getArg((size_t)i)); + } +} + +void LazyEnvironment::hash(HasherOld& hasher) const { + hasher.hashBytesOf(nargs); + for (int i = 0; i < (int)nargs; i++) { + hasher.hashBytesOf(missing[i]); + } + for (int i = 0; i < (int)nargs; i++) { + hasher.hashConstant(names[i]); + } + hasher.hashNullable(materialized()); + // TODO: Why are getParent() and getArg(i) null after deopt in pir_regression_check_code.R? + hasher.hashNullable(getParent()); + for (int i = 0; i < (int)nargs; i++) { + hasher.hashNullable(getArg((size_t)i)); + } +} + +void LazyEnvironment::addConnected(ConnectedCollectorOld& collector) const { + for (int i = 0; i < (int)nargs; i++) { + collector.addConstant(names[i]); + } + collector.addNullable(materialized(), false); + // TODO: Why are getParent() and getArg(i) null after deopt in pir_regression_check_code.R? + collector.addNullable(getParent(), false); + for (int i = 0; i < (int)nargs; i++) { + collector.addNullable(getArg((size_t)i), false); + } +} + +size_t LazyEnvironment::size() const { + return sizeof(LazyEnvironment) + sizeof(char) * nargs + + sizeof(SEXP) * (nargs + ArgOffset); +} + + } // namespace rir diff --git a/rir/src/runtime/LazyEnvironment.h b/rir/src/runtime/LazyEnvironment.h index b1967bfc0..9d33b0d2c 100644 --- a/rir/src/runtime/LazyEnvironment.h +++ b/rir/src/runtime/LazyEnvironment.h @@ -5,6 +5,7 @@ #include "interpreter/instance.h" #include "interpreter/interp_incl.h" #include "runtime/RirRuntimeObject.h" +#include "serializeHash/serializeUni.h" #include #include @@ -32,24 +33,24 @@ struct LazyEnvironment memset(missing, 0, sizeof(char) * nargs); } - SEXP materialized() { return getEntry(0); } + SEXP materialized() const { return getEntry(0); } void materialized(SEXP m) { setEntry(0, m); } size_t nargs; Immediate* names; - SEXP getArg(size_t i) { return getEntry(i + ArgOffset); } + SEXP getArg(size_t i) const { return getEntry(i + ArgOffset); } void setArg(size_t i, SEXP val, bool overrideMissing) { setEntry(i + ArgOffset, val); if (overrideMissing) missing[i] = false; } - SEXP getArg(SEXP n); - bool isMissing(SEXP n); - bool isMissing(size_t i); - size_t getArgIdx(SEXP n); + SEXP getArg(SEXP n) const; + bool isMissing(SEXP n) const; + bool isMissing(size_t i) const; + size_t getArgIdx(SEXP n) const; - SEXP getParent() { return getEntry(1); } + SEXP getParent() const { return getEntry(1); } void clear() { setEntry(1, nullptr); @@ -82,9 +83,17 @@ struct LazyEnvironment return le; } + static LazyEnvironment* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& deserializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + // This byteset remembers which slots have been overwritten, such that they // should not be considered missing anymore. char missing[]; + + private: + size_t size() const; }; } // namespace rir diff --git a/rir/src/runtime/PirTypeFeedback.cpp b/rir/src/runtime/PirTypeFeedback.cpp index 66e0e0bdb..67e911322 100644 --- a/rir/src/runtime/PirTypeFeedback.cpp +++ b/rir/src/runtime/PirTypeFeedback.cpp @@ -1,7 +1,10 @@ #include "PirTypeFeedback.h" #include "Code.h" +#include "R/Protect.h" #include "compiler/pir/instruction.h" -#include +#include "serializeHash/hash/UUIDPool.h" +#include "serializeHash/serialize/serializeR.h" +#include "runtime/TypeFeedback.h" #include namespace rir { @@ -23,46 +26,111 @@ PirTypeFeedback::PirTypeFeedback( // TODO, is this really needed? or is there any guarantee that my baseline // and all inlinee's baseline code objects stay live? also this should // probably be a weak map instead... - std::unordered_map srcCodeMap; + std::unordered_map functionMap; size_t idx = 0; for (auto c : codes) { - srcCodeMap[c] = idx; - setEntry(idx++, c->container()); + functionMap[c->function()] = idx; + setEntry(idx++, c->function()->container()); } idx = 0; - std::unordered_map reverseMapping; + std::unordered_map reverseMapping; for (auto s : slots) { auto slot = s.first; auto typeFeedback = s.second; assert(slot < MAX_SLOT_IDX); - auto e = reverseMapping.find(typeFeedback.feedbackOrigin.pc()); + auto e = reverseMapping.find(typeFeedback.feedbackOrigin); + if (e != reverseMapping.end()) { entry[slot] = e->second; assert(mdEntries()[e->second].previousType == typeFeedback.type); } else { - assert(codes.count(typeFeedback.feedbackOrigin.srcCode())); + assert(codes.count(typeFeedback.feedbackOrigin.function()->body())); new (&mdEntries()[idx]) MDEntry; - mdEntries()[idx].srcCode = - srcCodeMap.at(typeFeedback.feedbackOrigin.srcCode()); - mdEntries()[idx].offset = typeFeedback.feedbackOrigin.offset(); + mdEntries()[idx].funIdx = + functionMap.at(typeFeedback.feedbackOrigin.function()); + mdEntries()[idx].rirIdx = typeFeedback.feedbackOrigin.index(); mdEntries()[idx].previousType = typeFeedback.type; - reverseMapping[typeFeedback.feedbackOrigin.pc()] = idx; + reverseMapping[typeFeedback.feedbackOrigin] = idx; entry[slot] = idx++; } } } -Code* PirTypeFeedback::getSrcCodeOfSlot(size_t slot) { - auto code = getEntry(getMDEntryOfSlot(slot).srcCode); - return Code::unpack(code); +FeedbackIndex PirTypeFeedback::rirIdx(size_t slot) { + return getMDEntryOfSlot(slot).rirIdx; +} + +PirTypeFeedback* PirTypeFeedback::deserialize(AbstractDeserializer& deserializer) { + Protect p; + auto size = deserializer.readBytesOf(); + SEXP store = p(Rf_allocVector(EXTERNALSXP, size)); + // Need to ensure magic is correct so that we know this is internable + ((rir_header*)STDVEC_DATAPTR(store))->magic = PIR_TYPE_FEEDBACK_MAGIC; + deserializer.addRef(store); + + auto numCodes = deserializer.readBytesOf(); + auto numEntries = deserializer.readBytesOf(); + auto typeFeedback = new (DATAPTR(store)) PirTypeFeedback(numCodes); + deserializer.readBytes(typeFeedback->entry, sizeof(typeFeedback->entry)); + for (int i = 0; i < numCodes; i++) { + typeFeedback->setEntry(i, p(deserializer.read())); + } + deserializer.readBytes(typeFeedback->mdEntries(), (int)sizeof(MDEntry) * numEntries); + return typeFeedback; +} + +void PirTypeFeedback::serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf((R_xlen_t)size()); + auto numCodes = this->numCodes(); + auto numEntries = this->numEntries(); + serializer.writeBytesOf(numCodes); + serializer.writeBytesOf(numEntries); + serializer.writeBytes(entry, sizeof(entry)); + for (int i = 0; i < numCodes; i++) { + serializer.write(getEntry(i)); + } + serializer.writeBytes(mdEntries(), (int)sizeof(MDEntry) * numEntries); +} + +void PirTypeFeedback::hash(HasherOld& hasher) const { + auto numCodes = this->numCodes(); + auto numEntries = this->numEntries(); + hasher.hashBytesOf(numCodes); + hasher.hashBytesOf(numEntries); + hasher.hashBytes(entry, sizeof(entry)); + for (int i = 0; i < numCodes; i++) { + hasher.hash(getEntry(i)); + } + hasher.hashBytes(mdEntries(), (int)sizeof(MDEntry) * numEntries); +} + +void PirTypeFeedback::addConnected(ConnectedCollectorOld& collector) const { + auto numCodes = this->numCodes(); + for (int i = 0; i < numCodes; i++) { + collector.add(getEntry(i), false); + } +} + +int PirTypeFeedback::numCodes() const { + return (int)info.gc_area_length; +} + +int PirTypeFeedback::numEntries() const { + int numEntries = 0; + for (auto id : entry) { + if (id < MAX_SLOT_IDX && id > numEntries) { + numEntries = id + 1; + } + } + return numEntries; } -Opcode* PirTypeFeedback::getOriginOfSlot(size_t slot) { - return getSrcCodeOfSlot(slot)->code() + getBCOffsetOfSlot(slot); +size_t PirTypeFeedback::size() const { + return requiredSize(numCodes(), numEntries()); } } // namespace rir diff --git a/rir/src/runtime/PirTypeFeedback.h b/rir/src/runtime/PirTypeFeedback.h index 43168e640..45137ca63 100644 --- a/rir/src/runtime/PirTypeFeedback.h +++ b/rir/src/runtime/PirTypeFeedback.h @@ -4,6 +4,9 @@ #include "RirRuntimeObject.h" #include "compiler/pir/type.h" #include "runtime/TypeFeedback.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/hash/hashRootOld.h" +#include "serializeHash/serializeUni.h" #include #include @@ -21,7 +24,7 @@ struct Code; namespace pir { struct TypeFeedback; struct CallFeedback; -} +} // namespace pir struct PirTypeFeedback : public RirRuntimeObject { @@ -45,11 +48,8 @@ struct PirTypeFeedback ObservedValues& getSampleOfSlot(size_t slot) { return getMDEntryOfSlot(slot).feedback; } - unsigned getBCOffsetOfSlot(size_t slot) { - return getMDEntryOfSlot(slot).offset; - } - Code* getSrcCodeOfSlot(size_t slot); - Opcode* getOriginOfSlot(size_t slot); + + FeedbackIndex rirIdx(size_t slot); static size_t requiredSize(size_t origins, size_t entries) { return sizeof(PirTypeFeedback) + sizeof(SEXP) * origins + @@ -57,8 +57,8 @@ struct PirTypeFeedback } struct MDEntry { - uint8_t srcCode; - unsigned offset; + uint8_t funIdx; + FeedbackIndex rirIdx; ObservedValues feedback; pir::PirType previousType; unsigned sampleCount = 0; @@ -75,7 +75,16 @@ struct PirTypeFeedback } } + static PirTypeFeedback* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& deserializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + private: + explicit PirTypeFeedback(int numCodes) + : RirRuntimeObject(sizeof(*this), numCodes), + entry() {} + MDEntry& getMDEntryOfSlot(size_t slot) { assert(slot < MAX_SLOT_IDX); auto idx = entry[slot]; @@ -88,6 +97,10 @@ struct PirTypeFeedback sizeof(SEXP) * info.gc_area_length); } + int numCodes() const; + int numEntries() const; + size_t size() const; + uint8_t entry[MAX_SLOT_IDX]; }; diff --git a/rir/src/runtime/PoolStub.cpp b/rir/src/runtime/PoolStub.cpp new file mode 100644 index 000000000..a79729a6e --- /dev/null +++ b/rir/src/runtime/PoolStub.cpp @@ -0,0 +1,78 @@ +// +// Created by Jakob Hain on 10/9/23. +// + +#include "PoolStub.h" +#include "runtime/Function.h" + +namespace rir { + +PoolStub::PoolStub(const UUID& sourceHash, unsigned defaultArgIdx, size_t index) + : RirRuntimeObject(0, 0), + sourceHash(sourceHash), + defaultArgIdx(defaultArgIdx), + index(index) { + assert(sourceHash && "sourceHash must be non-null"); +} + +SEXP PoolStub::create(const UUID& sourceHash, unsigned defaultArgIdx, + size_t index) { + auto store = Rf_allocVector(EXTERNALSXP, sizeof(PoolStub)); + new (DATAPTR(store)) PoolStub(sourceHash, defaultArgIdx, index); + return store; +} + +void PoolStub::pad(const UUID& sourceHash, size_t sourceBodyPoolSize, + const std::vector& sourceDefaultArgPoolSizes, + Function* targetFunction) { + auto targetBody = targetFunction->body(); + for (auto i = (size_t)targetBody->extraPoolSize; i < sourceBodyPoolSize; + i++) { + targetBody->addExtraPoolEntry(create(sourceHash, UINT32_MAX, i)); + } + for (unsigned defaultArgIdx = 0; + defaultArgIdx < sourceDefaultArgPoolSizes.size(); defaultArgIdx++) { + auto sourceDefaultArgPoolSize = sourceDefaultArgPoolSizes[defaultArgIdx]; + if (sourceDefaultArgPoolSize > 0) { + auto targetDefaultArg = targetFunction->defaultArg(defaultArgIdx); + assert(targetDefaultArg && + "target default arg is NULL but source default arg has pool " + "entries"); + for (auto i = (size_t)targetDefaultArg->extraPoolSize; + i < sourceDefaultArgPoolSize; i++) { + targetDefaultArg->addExtraPoolEntry(create(sourceHash, defaultArgIdx, i)); + } + } + } +} + +void PoolStub::print(std::ostream& out) const { + out << "(" << sourceHash << ", " << index << ")"; +} + +PoolStub* PoolStub::deserialize(AbstractDeserializer& deserializer) { + UUID sourceHash; + deserializer.readBytes(&sourceHash, sizeof(UUID)); + auto poolType = deserializer.readBytesOf(); + auto index = deserializer.readBytesOf(); + auto store = create(sourceHash, poolType, index); + return unpack(store); +} + +void PoolStub::serialize(AbstractSerializer& serializer) const { + serializer.writeBytes(&sourceHash, sizeof(UUID)); + serializer.writeBytesOf(defaultArgIdx); + serializer.writeBytesOf(index); +} + +void PoolStub::hash(HasherOld& hasher) const { + hasher.hashBytes(&sourceHash, sizeof(UUID)); + hasher.hashBytesOf(defaultArgIdx); + hasher.hashBytesOf(index); +} + +void PoolStub::addConnected(__attribute__((unused)) ConnectedCollectorOld& collector) const { + // Nothing to add +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/PoolStub.h b/rir/src/runtime/PoolStub.h new file mode 100644 index 000000000..aa6fc4ee1 --- /dev/null +++ b/rir/src/runtime/PoolStub.h @@ -0,0 +1,59 @@ +// +// Created by Jakob Hain on 10/9/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "RirRuntimeObject.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/hash/hashRootOld.h" +#include "serializeHash/serializeUni.h" +#include +#include + +namespace rir { + +struct Function; + +#define POOL_STUB_MAGIC 0xec17a101 + +/// Stub for an SEXP in a local pool when we send it to the compiler server, +/// because the server only needs minimal information about the SEXP (like its +/// identity), so we don't want to send all of its data. For example, we +/// replace extra pool entries with stubs when sending RIR code to the server, +/// and then the server creates pushes and static calls and other bytecode +/// instructions for these SEXPs without caring about their content. When the +/// data is deserialized back to the client, we convert the stubs back into +/// their stubbed values. + +class PoolStub : + public RirRuntimeObject { + public: + /// Unique hash to identify the source + UUID sourceHash; + /// UNSIGNED_MAX if this is the function body's pool, otherwise this is the + /// default argument at the index's pool + unsigned defaultArgIdx; + size_t index; + + PoolStub(const UUID& sourceHash, unsigned defaultArgIdx, size_t index); + /// Create an SEXP stubbing the given pool entry + static SEXP create(const UUID& sourceHash, unsigned defaultArgIdx, + size_t index); + + /// Add stubs to source pool entries to the target code's pool until it's + /// `size`. + static void pad(const UUID& sourceHash, size_t sourceBodyPoolSize, + const std::vector& sourceDefaultArgPoolSizes, + Function* targetFunction); + + void print(std::ostream& out) const; + static PoolStub* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& serializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + +}; + +} // namespace rir diff --git a/rir/src/runtime/ProxyEnv.cpp b/rir/src/runtime/ProxyEnv.cpp new file mode 100644 index 000000000..3e8a61bee --- /dev/null +++ b/rir/src/runtime/ProxyEnv.cpp @@ -0,0 +1,89 @@ +// +// Created by Jakob Hain on 10/23/23. +// + +#include "ProxyEnv.h" +#include "R/Printing.h" +#include "R/r.h" +#include "serializeHash/globals.h" + +namespace rir { + +ProxyEnv::ProxyEnv(unsigned depth, unsigned depthToGlobal, SEXP global) + : RirRuntimeObject(0, 0), depth(depth), + depthToGlobal(depthToGlobal), global(global) { + assert(depth < depthToGlobal && TYPEOF(global) == ENVSXP && + isGlobalEnv(global)); +} + +SEXP ProxyEnv::create(SEXP env) { + if (globalsSet.count(env)) { + return env; + } + auto global = ENCLOS(env); + unsigned depthToGlobal = 1; + // Every env has a global eventually (R_EmptyEnv is global) + while (!globalsSet.count(global)) { + depthToGlobal++; + global = ENCLOS(global); + } + + auto store = Rf_allocVector(EXTERNALSXP, sizeof(ProxyEnv)); + new (DATAPTR(store)) ProxyEnv(0, depthToGlobal, global); + return store; +} + +SEXP ProxyEnv::parent() const { + if (depth + 1 == depthToGlobal) { + return global; + } else { + auto store = Rf_allocVector(EXTERNALSXP, sizeof(ProxyEnv)); + new (DATAPTR(store)) ProxyEnv(depth + 1, depthToGlobal, global); + return store; + } +} + +SEXP ProxyEnv::materialize(SEXP env) const { + assert(TYPEOF(env) == ENVSXP); + for (unsigned i = 0; i < depth; i++) { + env = ENCLOS(env); + } + return env; +} + +void ProxyEnv::print(std::ostream& out) const { + out << "^" << depth << ", ^" << depthToGlobal << " is " + << Print::dumpSexp(global) << ""; +} + +ProxyEnv* ProxyEnv::deserialize(AbstractDeserializer& deserializer) { + auto depth = deserializer.readBytesOf(); + auto depthToGlobal = deserializer.readBytesOf(); + auto global = deserializer.read(); + auto store = Rf_allocVector(EXTERNALSXP, sizeof(ProxyEnv)); + new (DATAPTR(store)) ProxyEnv(depth, depthToGlobal, global); + return unpack(store); +} + +void ProxyEnv::serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf(depth); + serializer.writeBytesOf(depthToGlobal); + serializer.write(global); +} + +void ProxyEnv::hash(HasherOld& hasher) const { + hasher.hashBytesOf(depth); + hasher.hashBytesOf(depthToGlobal); + hasher.hash(global); +} + +void ProxyEnv::addConnected(ConnectedCollectorOld& collector) const { + collector.add(global); +} + +bool isGlobalEnv(SEXP env) { + assert(TYPEOF(env) == ENVSXP && "only call this on environments"); + return globalsSet.count(env) || R_IsPackageEnv(env) || R_IsNamespaceEnv(env); +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/ProxyEnv.h b/rir/src/runtime/ProxyEnv.h new file mode 100644 index 000000000..ad94556a9 --- /dev/null +++ b/rir/src/runtime/ProxyEnv.h @@ -0,0 +1,58 @@ +// +// Created by Jakob Hain on 10/23/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "RirRuntimeObject.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/hash/hashRootOld.h" +#include "serializeHash/serializeUni.h" +#include + +namespace rir { + +#define PROXY_ENV_MAGIC 0xeeee1702 + +/// Proxy for an ENVSEXP that exists in the compiler client, so we don't have to +/// send the entire environment (it may be very large and have other compiled +/// closures). The main reason we use "proxy" instead of "stub" is because +/// "stub env" already refers to something else + +class ProxyEnv : + public RirRuntimeObject { + /// 0 if this is the env of the closure being compiled, 1 if its the parent, + /// 2 if ancestor, etc. + unsigned depth; + /// Depth to get to nearest global (typically `R_GlobalEnv`) + unsigned depthToGlobal; + /// Nearest global (typically `R_GlobalEnv`) + SEXP global; + + ProxyEnv(unsigned depth, unsigned depthToGlobal, SEXP global); + public: + /// Create an ENVSXP stubbing the given closure environment + static SEXP create(SEXP env); + /// The proxy's parent environment + SEXP parent() const; + + /// Convert back into a regular env, given the closure environment it was + /// originally created from (the closure environment can't be stored inside + /// this because we want to send it to the compiler server without sending + /// the closure, so we need it again) + SEXP materialize(SEXP env) const; + + void print(std::ostream& out) const; + static ProxyEnv* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& serializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; +}; + +/// Is the environment toplevel. Asserts argument is ENVSXP +/// +/// TODO: Move this somewhere else? +bool isGlobalEnv(SEXP env); + +} // namespace rir diff --git a/rir/src/runtime/RirRuntimeObject.h b/rir/src/runtime/RirRuntimeObject.h index c57b2531f..d9d64798d 100644 --- a/rir/src/runtime/RirRuntimeObject.h +++ b/rir/src/runtime/RirRuntimeObject.h @@ -71,10 +71,16 @@ struct RirRuntimeObject { return EXTERNALSXP_ENTRY(this->container(), pos); } + /// Creates an SEXP which, when the container is freed, will run finalizer + /// on it. + void makeFinalizer(R_CFinalizer_t finalizer, bool onexit) const { + return R_RegisterCFinalizerEx(container(), finalizer, (Rboolean)onexit); + } + RirRuntimeObject(uint32_t gc_area_start, uint32_t gc_area_length) : info{gc_area_start, gc_area_length, MAGIC} { uint8_t* start = (uint8_t*)this + gc_area_start; - memset(start, 0, gc_area_length * sizeof(SEXP)); + memset((void*)start, 0, gc_area_length * sizeof(SEXP)); } }; diff --git a/rir/src/runtime/TypeFeedback.cpp b/rir/src/runtime/TypeFeedback.cpp index 8a0ce4074..5e8d90113 100644 --- a/rir/src/runtime/TypeFeedback.cpp +++ b/rir/src/runtime/TypeFeedback.cpp @@ -6,17 +6,19 @@ #include "runtime/Function.h" #include +#include +#include namespace rir { -void ObservedCallees::record(Code* caller, SEXP callee, +void ObservedCallees::record(Function* function, SEXP callee, bool invalidateWhenFull) { if (taken < CounterOverflow) taken++; - if (numTargets < MaxTargets) { int i = 0; + auto caller = function->body(); for (; i < numTargets; ++i) if (caller->getExtraPoolEntry(targets[i]) == callee) break; @@ -30,66 +32,50 @@ void ObservedCallees::record(Code* caller, SEXP callee, } } -SEXP ObservedCallees::getTarget(const Code* code, size_t pos) const { +SEXP ObservedCallees::getTarget(const Function* function, size_t pos) const { assert(pos < numTargets); - return code->getExtraPoolEntry(targets[pos]); + return function->body()->getExtraPoolEntry(targets[pos]); } -FeedbackOrigin::FeedbackOrigin(rir::Code* src, Opcode* p) - : offset_((uintptr_t)p - (uintptr_t)src), srcCode_(src) { - if (p) { - assert(p >= src->code()); - assert(p < src->endCode()); - assert(pc() == p); - } +FeedbackPosition::FeedbackPosition(rir::Function* function, FeedbackIndex index) + : index_(index), function_(function) { + assert(function->typeFeedback()->isValid(index)); } -DeoptReason::DeoptReason(const FeedbackOrigin& origin, +DeoptReason::DeoptReason(const FeedbackPosition& origin, DeoptReason::Reason reason) - : reason(reason), origin(origin) { - switch (reason) { - case DeoptReason::Typecheck: - case DeoptReason::DeadCall: - case DeoptReason::CallTarget: - case DeoptReason::ForceAndCall: - case DeoptReason::DeadBranchReached: { - assert(pc()); - auto o = *pc(); - assert(o == Opcode::record_call_ || o == Opcode::record_type_ || - o == Opcode::record_test_); - break; - } - case DeoptReason::Unknown: - case DeoptReason::EnvStubMaterialized: - break; - } -} + : reason(reason), origin(origin) {} void DeoptReason::record(SEXP val) const { - srcCode()->function()->registerDeoptReason(reason); + if (origin.function()->isDeserializing() || + origin.function()->body()->kind == Code::Kind::Deserializing) { + // TODO: Is there still a way to record? We probably already have + // function in some cases, if so maybe we could set it earlier... + // Regardless, the only issue here is we just deopt again + return; + } + origin.function()->registerDeoptReason(reason); switch (reason) { case DeoptReason::Unknown: break; case DeoptReason::DeadBranchReached: { - assert(*pc() == Opcode::record_test_); - ObservedTest* feedback = (ObservedTest*)(pc() + 1); - feedback->seen = ObservedTest::Both; + auto& feedback = origin.function()->typeFeedback()->test(origin.idx()); + feedback.seen = ObservedTest::Both; break; } case DeoptReason::Typecheck: { - assert(*pc() == Opcode::record_type_); if (val == symbol::UnknownDeoptTrigger) break; - ObservedValues* feedback = (ObservedValues*)(pc() + 1); - feedback->record(val); + auto feedback = origin.function()->typeFeedback()->types(origin.idx()); + feedback.record(val); if (TYPEOF(val) == PROMSXP) { if (PRVALUE(val) == R_UnboundValue && - feedback->stateBeforeLastForce < ObservedValues::promise) - feedback->stateBeforeLastForce = ObservedValues::promise; - else if (feedback->stateBeforeLastForce < + feedback.stateBeforeLastForce < ObservedValues::promise) + feedback.stateBeforeLastForce = ObservedValues::promise; + else if (feedback.stateBeforeLastForce < ObservedValues::evaluatedPromise) - feedback->stateBeforeLastForce = + feedback.stateBeforeLastForce = ObservedValues::evaluatedPromise; } break; @@ -97,12 +83,12 @@ void DeoptReason::record(SEXP val) const { case DeoptReason::DeadCall: case DeoptReason::ForceAndCall: case DeoptReason::CallTarget: { - assert(*pc() == Opcode::record_call_); if (val == symbol::UnknownDeoptTrigger) break; - ObservedCallees* feedback = (ObservedCallees*)(pc() + 1); - feedback->record(srcCode(), val, true); - assert(feedback->taken > 0); + auto feedback = + origin.function()->typeFeedback()->callees(origin.idx()); + feedback.record(origin.function(), val, true); + assert(feedback.taken > 0); break; } case DeoptReason::EnvStubMaterialized: { @@ -111,4 +97,266 @@ void DeoptReason::record(SEXP val) const { } } +TypeFeedback* TypeFeedback::deserialize(AbstractDeserializer& deserializer) { + auto size = deserializer.readBytesOf(); + std::vector callees(size); + for (size_t i = 0; i < size; ++i) { + deserializer.readBytes(&callees[i], sizeof(ObservedCallees)); + } + + size = deserializer.readBytesOf(); + std::vector tests(size); + for (size_t i = 0; i < size; ++i) { + deserializer.readBytes(&tests[i], sizeof(ObservedTest)); + } + + size = deserializer.readBytesOf(); + std::vector types(size); + for (size_t i = 0; i < size; ++i) { + deserializer.readBytes(&types[i], sizeof(ObservedValues)); + } + + auto feedback = TypeFeedback::create(callees, tests, types); + // TypeFeedback doesn't need addRef + return feedback; +} + +void TypeFeedback::serialize(AbstractSerializer& serializer) const { + serializer.writeBytesOf(callees_size_); + for (size_t i = 0; i < callees_size_; i++) { + serializer.writeBytes(callees_ + i, sizeof(ObservedCallees)); + } + + serializer.writeBytesOf(tests_size_); + for (size_t i = 0; i < tests_size_; i++) { + serializer.writeBytes(tests_ + i, sizeof(ObservedTest)); + } + + serializer.writeBytesOf(types_size_); + for (size_t i = 0; i < types_size_; i++) { + serializer.writeBytes(types_ + i, sizeof(ObservedValues)); + } +} + +void TypeFeedback::hash(__attribute__((unused)) HasherOld& hasher) const { // NOLINT(*-convert-member-functions-to-static) + // TODO: debug why this sometimes gets called (does nothing but we want to + // sanity check that it never gets called, but it does only when running + // /bin/tests but not /bin/R (compiler client has similar issues with + // multiple instances...it's a confusing current issue) + // assert(false && "Feedback should never be hashed"); +} + +void TypeFeedback::addConnected( + __attribute__((unused)) ConnectedCollectorOld& collector) const { // NOLINT(*-convert-member-functions-to-static) + // TODO: debug why this sometimes gets called (does nothing but we want to + // sanity check that it never gets called, but it does only when running + // /bin/tests but not /bin/R (compiler client has similar issues with + // multiple instances...it's a confusing current issue) + // assert(false && "Feedback should never be hashed (don't call addConnected)"); +} + +unsigned TypeFeedback::numCallees() const { + return callees_size_; +} + +ObservedCallees& TypeFeedback::callees(uint32_t idx) { + assert(idx < this->callees_size_ && "Out of bounds callee access"); + return this->callees_[idx]; +} + +ObservedTest& TypeFeedback::test(uint32_t idx) { + assert(idx < this->tests_size_ && "Out of bounds test access"); + return this->tests_[idx]; +} + +ObservedValues& TypeFeedback::types(uint32_t idx) { + assert(idx < this->types_size_ && "Out of bounds type access"); + return this->types_[idx]; +} + +void TypeFeedback::print(std::ostream& out) const { + out << "TypeFeedback"; + if (!owner_) { + out << " (owner not set)"; + } + out << ":\n"; + + out << " " << callees_size_ << " callees:\n"; + for (size_t i = 0; i < callees_size_; ++i) { + out << " " << i << ": "; + callees_[i].print(out, owner_); + out << "\n"; + } + + out << " " << tests_size_ << " tests:\n"; + for (size_t i = 0; i < tests_size_; ++i) { + out << " " << i << ": "; + tests_[i].print(out); + out << "\n"; + } + + out << " " << types_size_ << " types:\n"; + for (size_t i = 0; i < types_size_; ++i) { + out << " " << i << ": "; + types_[i].print(out); + out << "\n"; + } +} + +void ObservedCallees::print(std::ostream& out, const Function* function) const { + if (taken == ObservedCallees::CounterOverflow) + out << "*, <"; + else + out << taken << ", <"; + if (numTargets == ObservedCallees::MaxTargets) + out << "*>, "; + else + out << numTargets << ">, "; + + out << (invalid ? "invalid" : "valid"); + out << (numTargets ? ", " : " "); + + for (unsigned i = 0; i < numTargets; ++i) { + if (function && function->body()->kind != Code::Kind::Deserializing) { + auto target = getTarget(function, i); + out << target << "(" << Rf_type2char(TYPEOF(target)) << ") "; + } else { + out << ""; + } + } +} + +void ObservedTest::print(std::ostream& out) const { + switch (seen) { + case ObservedTest::None: + out << "_"; + break; + case ObservedTest::OnlyTrue: + out << "T"; + break; + case ObservedTest::OnlyFalse: + out << "F"; + break; + case ObservedTest::Both: + out << "?"; + break; + } +} + +void ObservedValues::print(std::ostream& out) const { + if (numTypes) { + for (size_t i = 0; i < numTypes; ++i) { + out << Rf_type2char(seen[i]); + if (i != (unsigned)numTypes - 1) + out << ", "; + } + out << " (" << (object ? "o" : "") << (attribs ? "a" : "") + << (notFastVecelt ? "v" : "") << (!notScalar ? "s" : "") << ")"; + if (stateBeforeLastForce != + ObservedValues::StateBeforeLastForce::unknown) { + out << " | " + << ((stateBeforeLastForce == + ObservedValues::StateBeforeLastForce::value) + ? "value" + : (stateBeforeLastForce == + ObservedValues::StateBeforeLastForce::evaluatedPromise) + ? "evaluatedPromise" + : "promise"); + } + } else { + out << ""; + } +} + +bool FeedbackPosition::hasSlot() const { return !index_.isUndefined(); } + +uint32_t TypeFeedback::Builder::addCallee() { return ncallees_++; } + +uint32_t TypeFeedback::Builder::addTest() { return ntests_++; } + +uint32_t TypeFeedback::Builder::addType() { return ntypes_++; } + +TypeFeedback* TypeFeedback::Builder::build() { + std::vector callees(ncallees_, ObservedCallees{}); + std::vector tests(ntests_, ObservedTest{}); + std::vector types(ntypes_, ObservedValues{}); + + return TypeFeedback::create(callees, tests, types); +} + +TypeFeedback* TypeFeedback::empty() { return TypeFeedback::create({}, {}, {}); } + +void FeedbackPosition::function(Function* fun) { + assert(!hasSlot() || fun->typeFeedback()->isValid(index_)); + function_ = fun; +} +bool TypeFeedback::isValid(const FeedbackIndex& index) const { + switch (index.kind) { + case FeedbackKind::Call: + return index.idx < callees_size_; + case FeedbackKind::Test: + return index.idx < tests_size_; + case FeedbackKind::Type: + return index.idx < types_size_; + default: + return false; + } +} + +TypeFeedback* TypeFeedback::create(const std::vector& callees, + const std::vector& tests, + const std::vector& types) { + size_t dataSize = callees.size() * sizeof(ObservedCallees) + + tests.size() * sizeof(ObservedTest) + + types.size() * sizeof(ObservedValues); + + size_t objSize = sizeof(TypeFeedback) + dataSize; + + SEXP store = Rf_allocVector(EXTERNALSXP, objSize); + + TypeFeedback* res = + new (INTEGER(store)) TypeFeedback(callees, tests, types); + + return res; +} + +TypeFeedback::TypeFeedback(const std::vector& callees, + const std::vector& tests, + const std::vector& types) + : RirRuntimeObject(0, 0), owner_(nullptr), callees_size_(callees.size()), + tests_size_(tests.size()), types_size_(types.size()) { + + size_t callees_mem_size = callees_size_ * sizeof(ObservedCallees); + size_t tests_mem_size = tests_size_ * sizeof(ObservedTest); + size_t types_mem_size = types_size_ * sizeof(ObservedValues); + + callees_ = (ObservedCallees*)slots_; + tests_ = (ObservedTest*)(slots_ + callees_mem_size); + types_ = (ObservedValues*)(slots_ + callees_mem_size + tests_mem_size); + + if (callees_size_) { + memcpy(callees_, callees.data(), callees_mem_size); + } + + if (tests_size_) { + memcpy(tests_, tests.data(), tests_mem_size); + } + + if (types_size_) { + memcpy(types_, types.data(), types_mem_size); + } +} + +const char* FeedbackIndex::name() const { + switch (kind) { + case FeedbackKind::Call: + return "Call"; + case FeedbackKind::Test: + return "Test"; + case FeedbackKind::Type: + return "Type"; + default: + assert(false); + } +} } // namespace rir diff --git a/rir/src/runtime/TypeFeedback.h b/rir/src/runtime/TypeFeedback.h index ccf3821b5..a40de0e64 100644 --- a/rir/src/runtime/TypeFeedback.h +++ b/rir/src/runtime/TypeFeedback.h @@ -2,14 +2,81 @@ #define RIR_RUNTIME_FEEDBACK #include "R/r.h" +#include "Rinternals.h" #include "common.h" +#include "runtime/RirRuntimeObject.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/hash/hashRootOld.h" +#include "serializeHash/serializeUni.h" #include +#include #include #include +#include +#include +#include +#include namespace rir { struct Code; +struct Function; +class TypeFeedback; + +enum class FeedbackKind : uint8_t { + Call, + Test, + Type, +}; + +class FeedbackIndex { + private: + static constexpr unsigned IdxBits = 24; + static constexpr unsigned Undefined = (1 << IdxBits) - 1; + + FeedbackIndex(FeedbackKind kind_, uint32_t idx_) : kind(kind_), idx(idx_) {} + friend struct std::hash; + + public: + FeedbackKind kind; + uint32_t idx : IdxBits; + + FeedbackIndex() : kind(FeedbackKind::Call), idx(Undefined) {} + + static FeedbackIndex call(uint32_t idx) { + return FeedbackIndex(FeedbackKind::Call, idx); + } + static FeedbackIndex test(uint32_t idx) { + return FeedbackIndex(FeedbackKind::Test, idx); + } + static FeedbackIndex type(uint32_t idx) { + return FeedbackIndex(FeedbackKind::Type, idx); + } + + bool isUndefined() const { return idx == Undefined; } + + const char* name() const; + + uint32_t asInteger() const { return *((uint32_t*)this); } + + bool operator==(const FeedbackIndex& other) const { + return idx == other.idx && kind == other.kind; + } + + friend std::ostream& operator<<(std::ostream& out, + const FeedbackIndex& index) { + out << index.name() << "#"; + if (index.isUndefined()) { + out << "unknown"; + } else { + out << index.idx; + } + return out; + } +}; + +static_assert(sizeof(FeedbackIndex) == sizeof(uint32_t), + "Size needs to fit inside in integer for the llvm transition"); #pragma pack(push) #pragma pack(1) @@ -28,12 +95,14 @@ struct ObservedCallees { uint32_t numTargets : TargetBits; uint32_t taken : CounterBits; uint32_t invalid : 1; - - void record(Code* caller, SEXP callee, bool invalidateWhenFull = false); - SEXP getTarget(const Code* code, size_t pos) const; - std::array targets; + + void record(Function* function, SEXP callee, + bool invalidateWhenFull = false); + SEXP getTarget(const Function* function, size_t pos) const; + void print(std::ostream& out, const Function* function) const; }; + static_assert(sizeof(ObservedCallees) == 4 * sizeof(uint32_t), "Size needs to fit inside a record_ bc immediate args"); @@ -50,7 +119,7 @@ struct ObservedTest { ObservedTest() : seen(0), unused(0) {} - inline void record(SEXP e) { + inline void record(const SEXP e) { if (e == R_TrueValue) { if (seen == None) seen = OnlyTrue; @@ -67,6 +136,8 @@ struct ObservedTest { } seen = Both; } + + void print(std::ostream& out) const; }; static_assert(sizeof(ObservedTest) == sizeof(uint32_t), "Size needs to fit inside a record_ bc immediate args"); @@ -97,34 +168,9 @@ struct ObservedValues { void reset() { *this = ObservedValues(); } - void print(std::ostream& out) const { - if (numTypes) { - for (size_t i = 0; i < numTypes; ++i) { - out << Rf_type2char(seen[i]); - if (i != (unsigned)numTypes - 1) - out << ", "; - } - out << " (" << (object ? "o" : "") << (attribs ? "a" : "") - << (notFastVecelt ? "v" : "") << (!notScalar ? "s" : "") << ")"; - if (stateBeforeLastForce != - ObservedValues::StateBeforeLastForce::unknown) { - out << " | " - << ((stateBeforeLastForce == - ObservedValues::StateBeforeLastForce::value) - ? "value" - : (stateBeforeLastForce == - ObservedValues::StateBeforeLastForce:: - evaluatedPromise) - ? "evaluatedPromise" - : "promise"); - } - } else { - out << ""; - } - } + void print(std::ostream& out) const; inline void record(SEXP e) { - // Set attribs flag for every object even if the SEXP does not // have attributes. The assumption used to be that e having no // attributes implies that it is not an object, but this is not @@ -134,7 +180,7 @@ struct ObservedValues { // > .Internal(inspect(mf[["x"]])) // @56546cb06390 14 REALSXP g0c3 [OBJ,NAM(2)] (len=3, tl=0) 41,42,43 - notScalar = notScalar || XLENGTH(e) != 1; + notScalar = notScalar || RAW_LENGTH(e) != 1; object = object || Rf_isObject(e); attribs = attribs || object || ATTRIB(e) != R_NilValue; notFastVecelt = notFastVecelt || !fastVeceltOk(e); @@ -156,26 +202,28 @@ static_assert(sizeof(ObservedValues) == sizeof(uint32_t), enum class Opcode : uint8_t; -struct FeedbackOrigin { - private: - uint32_t offset_ = 0; - Code* srcCode_ = nullptr; +class FeedbackPosition { + FeedbackIndex index_; + Function* function_ = nullptr; public: - FeedbackOrigin() {} - FeedbackOrigin(rir::Code* src, Opcode* pc); + FeedbackPosition() {} + FeedbackPosition(rir::Function* fun, FeedbackIndex idx); - Opcode* pc() const { - if (offset_ == 0) - return nullptr; - return (Opcode*)((uintptr_t)srcCode() + offset_); + bool hasSlot() const; + FeedbackIndex index() const { return index_; } + uint32_t idx() const { return index_.idx; } + Function* function() const { return function_; } + void function(Function* fun); + + bool operator==(const FeedbackPosition& other) const { + return index_ == other.index_ && function_ == other.function_; } - uint32_t offset() const { return offset_; } - Code* srcCode() const { return srcCode_; } - void srcCode(Code* src) { srcCode_ = src; } - bool operator==(const FeedbackOrigin& other) const { - return offset_ == other.offset_ && srcCode_ == other.srcCode_; + friend std::ostream& operator<<(std::ostream& out, + const FeedbackPosition& origin) { + out << (void*)origin.function_ << "[" << origin.index_ << "]"; + return out; } }; @@ -192,12 +240,9 @@ struct DeoptReason { }; DeoptReason::Reason reason; - FeedbackOrigin origin; - - DeoptReason(const FeedbackOrigin& origin, DeoptReason::Reason reason); + FeedbackPosition origin; - Code* srcCode() const { return origin.srcCode(); } - Opcode* pc() const { return origin.pc(); } + DeoptReason(const FeedbackPosition& origin, DeoptReason::Reason reason); bool operator==(const DeoptReason& other) const { return reason == other.reason && origin == other.origin; @@ -228,13 +273,11 @@ struct DeoptReason { out << "Unknown"; break; } - out << "@" << (void*)reason.pc(); + out << "@" << reason.origin; return out; } - static DeoptReason unknown() { - return DeoptReason(FeedbackOrigin(0, 0), Unknown); - } + static DeoptReason unknown() { return DeoptReason({}, Unknown); } void record(SEXP val) const; @@ -246,15 +289,90 @@ struct DeoptReason { static_assert(sizeof(DeoptReason) == 4 * sizeof(uint32_t), "Size needs to fit inside a record_deopt_ bc immediate args"); +#define TYPEFEEDBACK_MAGIC (unsigned)0xfeedbac0 + +class TypeFeedback : public RirRuntimeObject { + private: + friend Function; + + Function* owner_; + size_t callees_size_; + size_t tests_size_; + size_t types_size_; + ObservedCallees* callees_; + ObservedTest* tests_; + ObservedValues* types_; + // All the data are stored in this array: callees, tests and types in this + // order. The constructors sets the above pointers to point at the + // appropriate locations. + uint8_t slots_[]; + + explicit TypeFeedback(const std::vector& callees, + const std::vector& tests, + const std::vector& types); + + public: + static TypeFeedback* create(const std::vector& callees, + const std::vector& tests, + const std::vector& types); + + static TypeFeedback* empty(); + + class Builder { + unsigned ncallees_ = 0; + unsigned ntests_ = 0; + unsigned ntypes_ = 0; + + public: + uint32_t addCallee(); + uint32_t addTest(); + uint32_t addType(); + TypeFeedback* build(); + }; + + // TODO: Bug where, when we only send the compiler server the client source + // and feedback, we get record_call instructions with corrupt indices + unsigned numCallees() const; + ObservedCallees& callees(uint32_t idx); + ObservedTest& test(uint32_t idx); + ObservedValues& types(uint32_t idx); + + void print(std::ostream& out) const; + + static TypeFeedback* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& serializer) const; + + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; + + bool isValid(const FeedbackIndex& index) const; + + Function* owner() const { return owner_; } +}; + #pragma pack(pop) } // namespace rir namespace std { +template <> +struct hash { + std::size_t operator()(const rir::FeedbackIndex& v) const { + return hash_combine(hash_combine(0, v.kind), v.idx); + } +}; + +template <> +struct hash { + std::size_t operator()(const rir::FeedbackPosition& v) const { + return hash_combine(hash_combine(0, v.index()), v.function()); + } +}; + template <> struct hash { std::size_t operator()(const rir::DeoptReason& v) const { - return hash_combine(hash_combine(0, v.pc()), v.reason); + return hash_combine(hash_combine(0, v.origin), v.reason); } }; } // namespace std diff --git a/rir/src/runtime/log/RirObjectPrintStyle.cpp b/rir/src/runtime/log/RirObjectPrintStyle.cpp new file mode 100644 index 000000000..132ddb069 --- /dev/null +++ b/rir/src/runtime/log/RirObjectPrintStyle.cpp @@ -0,0 +1,35 @@ +// +// Created by Jakob Hain on 7/26/23. +// + +#include "RirObjectPrintStyle.h" +#include +#include +#include +#include + +namespace rir { + +static RirObjectPrintStyle getDefaultDebugStyle() { + const char* env = getenv("RIR_DEBUG_STYLE"); + if (env && strlen(env) > 0) { +#define V(name) \ + if (strcmp(env, #name) == 0) { \ + return RirObjectPrintStyle::name; \ + } + LIST_OF_RIR_PRINT_STYLES(V) +#undef V + std::cerr << "Unknown RIR_DEBUG_STYLE: " << env << "\n" + << "Supported options are: (unset)"; +#define V(name) std::cerr << ", '" << #name << "'"; + LIST_OF_RIR_PRINT_STYLES(V) +#undef V + assert(false); + } else { + return RirObjectPrintStyle::Default; + } +} + +RirObjectPrintStyle RIR_DEBUG_STYLE = getDefaultDebugStyle(); + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/log/RirObjectPrintStyle.h b/rir/src/runtime/log/RirObjectPrintStyle.h new file mode 100644 index 000000000..d93e8e112 --- /dev/null +++ b/rir/src/runtime/log/RirObjectPrintStyle.h @@ -0,0 +1,24 @@ +// +// Created by Jakob Hain on 7/26/23. +// + +#pragma once + +namespace rir { + +#define LIST_OF_RIR_PRINT_STYLES(V) \ + V(Default) \ + V(Detailed) \ + V(PrettyGraph) \ + +/// Style to print RIR objects via `RirRuntimeObject::print` +/// (`compiler/log/DebugStyle` is for PIR objects). +enum class RirObjectPrintStyle { +#define V(name) name, + LIST_OF_RIR_PRINT_STYLES(V) +#undef V +}; + +extern RirObjectPrintStyle RIR_DEBUG_STYLE; + +} // namespace rir diff --git a/rir/src/runtime/log/printPrettyGraph.cpp b/rir/src/runtime/log/printPrettyGraph.cpp new file mode 100644 index 000000000..5e2c8bbdb --- /dev/null +++ b/rir/src/runtime/log/printPrettyGraph.cpp @@ -0,0 +1,112 @@ +// +// Created by Jakob Hain on 7/28/23. +// + +#include "printPrettyGraph.h" +#include "R/r.h" +#include "runtime/rirObjectMagic.h" +#include "utils/HTMLBuilder/HTML.h" +#include "utils/HTMLBuilder/escapeHtml.h" +#include +#include + +namespace rir { + +static inline HTML::Text makeText(PrettyGraphContentPrinter content, bool escape = true) { + std::stringstream s; + content(s); + if (escape) { + return HTML::Text(escapeHtml(s.str())); + } else { + return HTML::Text(s.str()); + } +} + +static inline std::string sexpId(SEXP sexp) { + std::stringstream s; + s << "0x" << std::hex << (uintptr_t)sexp; + return s.str(); +} + +void +PrettyGraphInnerPrinter::printUsingImpl(SEXP root, + std::ostream& out, + std::function printImpl) { + std::unordered_set seen; + std::queue worklist; + seen.insert(root); + worklist.push(root); + + auto printItem = [&](SEXP sexp){ + auto nodeType = TYPEOF(sexp) == EXTERNALSXP ? rirObjectClassName(sexp) : "other"; + auto node = + HTML::Div("node") + .id(sexpId(sexp)) + .cls(std::string("node-") + nodeType); + PrettyGraphInnerPrinter print{ + [&](auto name){ + node << (HTML::Div("name") << makeText(name)); + }, + [&](auto body) { + node << (HTML::Div("body") << makeText(body, false)); + }, + [&](auto connected, auto isChild, auto type, auto description, auto isFarArway) { + // Add item to worklist to be printed, unless it was already + // printed, and add to seen + // Also, cppcheck can't parse this + // cppcheck-suppress internalAstError + if (seen.insert(connected).second) { + worklist.push(connected); + } + + // Print edge to node (buffered) + auto arrow = + HTML::Div("arrow") + .cls(std::string("arrow-") + nodeType + "-" + type) + .addAttribute("data-connected", sexpId(connected)) + << makeText(description); + if (isChild) { + arrow.addAttribute("data-is-child", "true"); + } + if (isFarArway) { + arrow.cls("arrow-far-away"); + } + node << std::move(arrow); + } + }; + printImpl(sexp, print); + // We've already printed connected objects' HTML nodes, this is the + // current object's HTML node + out << node; + }; + + // Print header + out << "\n" + "RIR\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "

Needs the rirPrettyGraph folder (located in tools) to be in the same location

\n" + "
\n"; + + // Print items + while (!worklist.empty()) { + printItem(worklist.front()); + worklist.pop(); + } + + // Print footer + out << "
\n" + "\n" + "\n" + "\n" + "\n" + ""; +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/log/printPrettyGraph.h b/rir/src/runtime/log/printPrettyGraph.h new file mode 100644 index 000000000..113985fb9 --- /dev/null +++ b/rir/src/runtime/log/printPrettyGraph.h @@ -0,0 +1,56 @@ +// +// Created by Jakob Hain on 7/28/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "runtime/log/RirObjectPrintStyle.h" +#include +#include + +namespace rir { + +typedef const std::function& PrettyGraphContentPrinter; + +class PrettyGraphInnerPrinter { + std::function addName_; + std::function addBody_; + std::function addEdgeTo_; + PrettyGraphInnerPrinter( + const std::function&& addName, + const std::function&& addBody, + const std::function&& addEdgeTo) + : addName_(addName), addBody_(addBody), addEdgeTo_(addEdgeTo) {} + + public: + /// Given a function which prints an SEXP's node content and adds connected + /// nodes via the inner printer, this function prints an HTML graph + /// containing the node and all its connected nodes. i.e. this function + /// maintains the SEXP worklist and prints the header and footer, as well + /// as constructing an PrettyGraphInnerPrinter which lets you print the + /// content. + /// + /// This should generally not be called. It's used by + /// `printRirObject(,,RirObjectPrintStyle::PrettyGraph)` which you probably + /// want to use instead. + static void + printUsingImpl(SEXP root, std::ostream& out, + std::function printImpl); + + void addName(PrettyGraphContentPrinter name) const { + addName_(name); + } + + void addBody(PrettyGraphContentPrinter body) const { + addBody_(body); + } + + void addEdgeTo(SEXP connected, bool isChild, const char* type, + PrettyGraphContentPrinter description = [](std::ostream& s){}, + bool isFarAway = false) const { + addEdgeTo_(connected, isChild, type, description, isFarAway); + } +}; + +} // namespace rir diff --git a/rir/src/runtime/log/printPrettyGraphFromEnv.cpp b/rir/src/runtime/log/printPrettyGraphFromEnv.cpp new file mode 100644 index 000000000..a5544b00e --- /dev/null +++ b/rir/src/runtime/log/printPrettyGraphFromEnv.cpp @@ -0,0 +1,103 @@ +// +// Created by Jakob Hain on 10/10/23. +// + +#include "printPrettyGraphFromEnv.h" +#include "R/r.h" +#include "compiler/parameter.h" +#include "runtime/log/printRirObject.h" +#include +#include +#include +#include +#include + +namespace rir { + +// TODO: Properly abstract these instead of writing the same code twice +bool pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS = + getenv("PIR_GRAPH_PRINT_RIR_OBJECTS") != nullptr && + strcmp(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS"), "") != 0 && + strcmp(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS"), "0") != 0 && + strcmp(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS"), "false") != 0; +const char* pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS_PATH = + getenv("PIR_GRAPH_PRINT_RIR_OBJECTS") != nullptr && + strcmp(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS"), "") != 0 && + strcmp(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS"), "0") != 0 && + strcmp(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS"), "false") != 0 && + strcmp(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS"), "1") != 0 && + strcmp(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS"), "true") != 0 ? + getenv("PIR_GRAPH_PRINT_RIR_OBJECTS") : nullptr; +unsigned pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS_FREQUENCY = + getenv("PIR_GRAPH_PRINT_RIR_OBJECTS_FREQUENCY") != nullptr + ? strtol(getenv("PIR_GRAPH_PRINT_RIR_OBJECTS_FREQUENCY"), nullptr, 10) + : 10; + +void initializePrintPrettyGraphFromEnv() { + if (pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS_PATH) { + // Create folder (not recursively) if it doesn't exist + auto code = mkdir(pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS_PATH, 0777); + if (code != 0 && errno != EEXIST) { + std::cerr << "Could not create folder for PIR_GRAPH_PRINT_RIR_OBJECTS: " + << strerror(errno) << std::endl; + std::abort(); + } + // Also softlink rirPrettyGraph (HTML dependency) in the folder. + // We do this even if the folder already exists, because the user may + // have corrupted it. + auto linkSource = getenv("PIR_PRETTY_GRAPH_DEPENDENCY_LOCATION"); + assert(linkSource && "PIR_PRETTY_GRAPH_DEPENDENCY_LOCATION should be set by the R executable, we need it to softlink rirPrettyGraph for the HTML prints"); + std::stringstream linkTarget; + linkTarget << pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS_PATH << "/rirPrettyGraph"; + code = symlink(linkSource, linkTarget.str().c_str()); + if (code != 0 && errno != EEXIST) { + std::cerr << "Could not symlink associated common styles/scripts for PIR_GRAPH_PRINT_RIR_OBJECTS: " + << strerror(errno) << std::endl; + std::abort(); + } + } +} + +static void printPrettyGraph(SEXP sexp, const std::string& associated) { + if (pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS_PATH) { + // Create new file which is denoted by the current date and hash + std::stringstream filePath; + filePath << pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS_PATH << "/" + << time(nullptr) << "-" << associated << ".html"; + std::ofstream file(filePath.str()); + if (!file.is_open()) { + std::cerr << "Could not open file for PIR_GRAPH_PRINT_RIR_OBJECTS: " + << strerror(errno) << std::endl; + std::abort(); + } + // Print HTML pretty graph to file + printRirObject(sexp, file, RirObjectPrintStyle::PrettyGraph); + // File closes automatically (RAII) + } else { + // Just print HTML pretty graph to stdout + printRirObject(sexp, std::cout, RirObjectPrintStyle::PrettyGraph); + } +} + +void printPrettyGraphIfNecessary(SEXP sexp, const std::string& associated) { + // cppcheck-suppress variableScope + static unsigned graphPrintCounter = 0; + if (pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS) { + graphPrintCounter++; + if (graphPrintCounter == pir::Parameter::PIR_GRAPH_PRINT_RIR_OBJECTS_FREQUENCY) { + printPrettyGraph(sexp, associated); + graphPrintCounter = 0; + } + } +} + +void printPrettyGraphOfInternedIfNecessary(SEXP sexp, const UUID& hash) { + auto associated = hash.str(); + printPrettyGraphIfNecessary(sexp, associated); +} + +void printPrettyGraphOfCompiledIfNecessary(SEXP sexp, const std::string& name) { + printPrettyGraphIfNecessary(sexp, name); +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/log/printPrettyGraphFromEnv.h b/rir/src/runtime/log/printPrettyGraphFromEnv.h new file mode 100644 index 000000000..e248b681b --- /dev/null +++ b/rir/src/runtime/log/printPrettyGraphFromEnv.h @@ -0,0 +1,18 @@ +// +// Created by Jakob Hain on 10/10/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "serializeHash/hash/UUID.h" + + + +namespace rir { + +void initializePrintPrettyGraphFromEnv(); +void printPrettyGraphOfInternedIfNecessary(SEXP sexp, const UUID& uuid); +void printPrettyGraphOfCompiledIfNecessary(SEXP sexp, const std::string& name); + +} // namespace rir diff --git a/rir/src/runtime/log/printRirObject.cpp b/rir/src/runtime/log/printRirObject.cpp new file mode 100644 index 000000000..8e20195d8 --- /dev/null +++ b/rir/src/runtime/log/printRirObject.cpp @@ -0,0 +1,100 @@ +// +// Created by Jakob Hain on 7/26/23. +// + +#include "printRirObject.h" +#include "R/Printing.h" +#include "interpreter/interp_incl.h" +#include "printPrettyGraph.h" +#include "runtime/Code.h" +#include "runtime/DispatchTable.h" +#include "runtime/Function.h" +#include "utils/HTMLBuilder/escapeHtml.h" +#include + +namespace rir { + +static void defaultPrintRirObject(SEXP sexp, std::ostream& s, bool isDetailed) { + if (isDetailed) { + s << Print::dumpSexp(sexp, SIZE_MAX) << "\n"; + } else { + s << Print::dumpSexp(sexp) << "\n"; + } +} + +static void +defaultPrintRirObjectPrettyGraphContent(SEXP sexp, + const PrettyGraphInnerPrinter& print) { + print.addName([&](std::ostream& s){ + s << Rf_type2char(TYPEOF(sexp)); + }); + print.addBody([&](std::ostream& s){ + s << "
" << escapeHtml(Print::dumpSexp(sexp, SIZE_MAX)) << "
"; + }); + if (isValidClosureSEXP(sexp)) { + print.addEdgeTo(BODY(sexp), true, "body"); + } else if (TYPEOF(sexp) == VECSXP) { + for (R_xlen_t i = 0; i < XLENGTH(sexp); i++) { + print.addEdgeTo(VECTOR_ELT(sexp, i), true, "elem", [&](std::ostream& s){ + s << i; + }); + } + } else if (TYPEOF(sexp) == LISTSXP) { + for (unsigned i = 0; sexp != R_NilValue; i++, sexp = CDR(sexp)) { + print.addEdgeTo(CAR(sexp), true, "elem", [&](std::ostream& s){ + s << i; + }); + } + } +} + +static void printRirObject(SEXP sexp, std::ostream& s, bool isDetailed) { + if (auto d = DispatchTable::check(sexp)) { + d->print(s, isDetailed); + } else if (auto f = Function::check(sexp)) { + f->print(s, isDetailed); + } else if (auto c = Code::check(sexp)) { + c->print(s, isDetailed); + } else { + defaultPrintRirObject(sexp, s, isDetailed); + } +} + +static void printRirObjectPrettyGraphContent(SEXP sexp, + const PrettyGraphInnerPrinter& print) { + if (auto d = DispatchTable::check(sexp)) { + d->printPrettyGraphContent(print); + } else if (auto f = Function::check(sexp)) { + f->printPrettyGraphContent(print); + } else if (auto c = Code::check(sexp)) { + c->printPrettyGraphContent(print); + } else { + defaultPrintRirObjectPrettyGraphContent(sexp, print); + } +} + +void prettyGraphPrintRirObject(SEXP sexp, std::ostream& s) { + PrettyGraphInnerPrinter::printUsingImpl(sexp, s, printRirObjectPrettyGraphContent); +} + +void printRirObject(SEXP sexp, std::ostream& s, RirObjectPrintStyle style) { + switch (style) { + case RirObjectPrintStyle::Default: + printRirObject(sexp, s, false); + break; + case RirObjectPrintStyle::Detailed: + printRirObject(sexp, s, true); + break; + case RirObjectPrintStyle::PrettyGraph: + prettyGraphPrintRirObject(sexp, s); + break; + } +} + +std::string printRirObject(SEXP sexp, RirObjectPrintStyle style) { + std::stringstream s; + printRirObject(sexp, s, style); + return s.str(); +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/log/printRirObject.h b/rir/src/runtime/log/printRirObject.h new file mode 100644 index 000000000..0a2f162e3 --- /dev/null +++ b/rir/src/runtime/log/printRirObject.h @@ -0,0 +1,20 @@ +// +// Created by Jakob Hain on 7/26/23. +// + +#pragma once + +#include "runtime/log/RirObjectPrintStyle.h" +#include "R/r_incl.h" +#include +#include + +namespace rir { + +/// Print an SEXP, printing it detailed if it's RIR +void printRirObject(SEXP sexp, std::ostream& s = std::cout, + RirObjectPrintStyle style = RIR_DEBUG_STYLE); +/// Print an SEXP, printing it detailed if it's RIR +std::string printRirObject(SEXP sexp, RirObjectPrintStyle style); + +} // namespace rir diff --git a/rir/src/runtime/rirObjectMagic.cpp b/rir/src/runtime/rirObjectMagic.cpp new file mode 100644 index 000000000..449111596 --- /dev/null +++ b/rir/src/runtime/rirObjectMagic.cpp @@ -0,0 +1,59 @@ +// +// Created by Jakob Hain on 7/26/23. +// + +#include "rirObjectMagic.h" +#include "Code.h" +#include "DispatchTable.h" +#include "GenericDispatchTable.h" +#include "LazyArglist.h" +#include "LazyEnvironment.h" +#include "PoolStub.h" +#include "ProxyEnv.h" +#include "RirRuntimeObject.h" + +namespace rir { + +const char* rirObjectClassName(unsigned magic) { + switch (magic) { + case CODE_MAGIC: + return "Code"; + case DISPATCH_TABLE_MAGIC: + return "DispatchTable"; + case FUNCTION_MAGIC: + return "Function"; + case ARGLIST_ORDER_MAGIC: + return "ArglistOrder"; + case LAZY_ARGS_MAGIC: + return "LazyArglist"; + case LAZY_ENVIRONMENT_MAGIC: + return "LazyEnvironment"; + case PIR_TYPE_FEEDBACK_MAGIC: + return "PirTypeFeedback"; + case TYPEFEEDBACK_MAGIC: + return "TypeFeedback"; + case SERIAL_MODULE_MAGIC: + return "SerialModule"; + case POOL_STUB_MAGIC: + return "PoolStub"; + case PROXY_ENV_MAGIC: + return "ProxyEnv"; + case GENERIC_DISPATCH_TABLE_MAGIC: + return "GenericDispatchTable"; + default: + std::cerr << "unhandled RIR object magic: 0x" << std::hex << magic + << "\n"; + assert(false); + } +} + +unsigned rirObjectMagic(SEXP rirObject) { + assert(TYPEOF(rirObject) == EXTERNALSXP && "Not a RIR object"); + return ((rir_header*)STDVEC_DATAPTR(rirObject))->magic; +} + +const char* rirObjectClassName(SEXP rirObject) { + return rirObjectClassName(rirObjectMagic(rirObject)); +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/runtime/rirObjectMagic.h b/rir/src/runtime/rirObjectMagic.h new file mode 100644 index 000000000..42858a74f --- /dev/null +++ b/rir/src/runtime/rirObjectMagic.h @@ -0,0 +1,21 @@ +// +// Created by Jakob Hain on 7/26/23. +// + +#pragma once + +#include "R/r_incl.h" + +namespace rir { + +/// Throws an error if the object isn't an EXTERNALSXP (RIR object) +unsigned rirObjectMagic(SEXP rirObject); + +/// Class name of rir object with the given magic +const char* rirObjectClassName(unsigned magic); + +/// Class name of rir object. +/// Throws an error if the object isn't an EXTERNALSXP (RIR object) +const char* rirObjectClassName(SEXP rirObject); + +} // namespace rir diff --git a/rir/src/serializeHash/globals.cpp b/rir/src/serializeHash/globals.cpp new file mode 100644 index 000000000..8d4ae1b01 --- /dev/null +++ b/rir/src/serializeHash/globals.cpp @@ -0,0 +1,54 @@ +// +// Created by Jakob Hain on 8/13/23. +// + +#include "globals.h" +#include "R/Symbols.h" + +namespace rir { + +std::vector* globals_; +std::unordered_set* globalsSet_; +std::unordered_map* global2Index_; +std::unordered_map* cppId2Global_; +std::unordered_map* global2CppId_; +const std::vector& globals = *globals_; +const std::unordered_set& globalsSet = *globalsSet_; +const std::unordered_map& global2Index = *global2Index_; +const std::unordered_map& cppId2Global = *cppId2Global_; +const std::unordered_map& global2CppId = *global2CppId_; + +void initGlobals() { + cppId2Global_ = new std::unordered_map(); + cppId2Global_->emplace("R_GlobalEnv", R_GlobalEnv); + cppId2Global_->emplace("R_BaseEnv", R_BaseEnv); + cppId2Global_->emplace("R_BaseNamespace", R_BaseNamespace); + cppId2Global_->emplace("R_TrueValue", R_TrueValue); + cppId2Global_->emplace("R_NilValue", R_NilValue); + cppId2Global_->emplace("R_FalseValue", R_FalseValue); + cppId2Global_->emplace("R_UnboundValue", R_UnboundValue); + cppId2Global_->emplace("R_MissingArg", R_MissingArg); + cppId2Global_->emplace("R_RestartToken", R_RestartToken); + cppId2Global_->emplace("R_LogicalNAValue", R_LogicalNAValue); + cppId2Global_->emplace("R_EmptyEnv", R_EmptyEnv); + cppId2Global_->emplace("R_DimSymbol", R_DimSymbol); + cppId2Global_->emplace("R_DotsSymbol", R_DotsSymbol); + cppId2Global_->emplace("R_NamesSymbol", R_NamesSymbol); + cppId2Global_->emplace("expandDotsTrigger", symbol::expandDotsTrigger); + + globals_ = new std::vector(); + globalsSet_ = new std::unordered_set(); + global2CppId_ = new std::unordered_map(); + for (auto& e : *cppId2Global_) { + globals_->push_back(e.second); + globalsSet_->insert(e.second); + global2CppId_->emplace(e.second, e.first); + } + global2Index_ = new std::unordered_map(); + for (unsigned i = 0; i < globals_->size(); ++i) { + global2Index_->emplace((*globals_)[i], i); + } +} + + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/globals.h b/rir/src/serializeHash/globals.h new file mode 100644 index 000000000..2f7b7fcea --- /dev/null +++ b/rir/src/serializeHash/globals.h @@ -0,0 +1,26 @@ +// +// Created by Jakob Hain on 8/13/23. +// + +#pragma once + +#include "R/r.h" +#include +#include +#include +#include + +namespace rir { + +// Globals aren't considered connected and references to them don't have +// recursive connected references +extern const std::vector& globals; +extern const std::unordered_set& globalsSet; +extern const std::unordered_map& global2Index; +extern const std::unordered_map& cppId2Global; +extern const std::unordered_map& global2CppId; + +/// Initialize globals. Needs to run after symbols are initialized +void initGlobals(); + +} // namespace rir diff --git a/rir/src/serializeHash/hash/UUID.cpp b/rir/src/serializeHash/hash/UUID.cpp new file mode 100644 index 000000000..a84d827a3 --- /dev/null +++ b/rir/src/serializeHash/hash/UUID.cpp @@ -0,0 +1,94 @@ +#include "UUID.h" +#include "R/Serialize.h" + +#include "xxhash.h" +#include +#include + +namespace rir { + +UUID UUID::hash(const void* data, size_t size) { + UUID::Hasher hasher; + hasher.hashBytes(data, size); + return hasher.finalize(); +} + +UUID UUID::deserialize(__attribute__((unused)) SEXP _refTable, R_inpstream_t inp) { + UUID uuid; + InBytes(inp, &uuid.high, sizeof(uuid.high)); + InBytes(inp, &uuid.low, sizeof(uuid.low)); + return uuid; +} + +void UUID::serialize(__attribute__((unused)) SEXP _refTable, R_outpstream_t out) const { + OutBytes(out, &high, sizeof(high)); + OutBytes(out, &low, sizeof(low)); +} + +std::string UUID::str() const { + std::ostringstream str; + str << std::setfill('0') << std::setw(sizeof(high) + sizeof(low)) + << std::right << std::hex << high << low << std::dec; + return str.str(); +} + +std::ostream& operator<<(std::ostream& stream, const UUID& uuid) { + stream << "0x" << uuid.str(); + return stream; +} + +UUID::operator bool() const { + return high || low; +} + +bool UUID::operator==(const UUID& other) const { + return high == other.high && low == other.low; +} + +bool UUID::operator!=(const UUID& other) const { + return high != other.high || low != other.low; +} + +UUID::Hasher::Hasher() : state(XXH3_createState()), finalized(false) { + assert(state && "Failed to create hash state"); + + if (XXH3_128bits_reset(state) == XXH_ERROR) { + XXH3_freeState(state); + assert(false && "Failed to initialize hash state as 128 bits"); + } +} + +UUID::Hasher::~Hasher() { + assert(finalized && "UUID::HasherOld was not finalized"); +} + +void UUID::Hasher::hashBytesOfCString(const char* c) { + hashBytes(c, strlen(c)); +} + +void UUID::Hasher::hashBytes(const void* data, size_t size) { + assert(!finalized && "UUID::HasherOld was already finalized"); + + if (XXH3_128bits_update(state, data, size) == XXH_ERROR) { + XXH3_freeState(state); + assert(false && "Failed to update hash state"); + } +} + +UUID UUID::Hasher::finalize() { + assert(!finalized && "UUID::HasherOld was already finalized"); + finalized = true; + + auto digest = XXH3_128bits_digest(state); + UUID uuid{digest.high64, digest.low64}; + XXH3_freeState(state); + return uuid; +} + +} // namespace rir + +namespace std { +std::size_t hash::operator()(const rir::UUID& v) const { + return v.high ^ v.low; +} +} // namespace std diff --git a/rir/src/serializeHash/hash/UUID.h b/rir/src/serializeHash/hash/UUID.h new file mode 100644 index 000000000..a2601908a --- /dev/null +++ b/rir/src/serializeHash/hash/UUID.h @@ -0,0 +1,66 @@ +#pragma once + +#include "R/r.h" + +#include + +typedef struct XXH3_state_s XXH3_state_t; + +namespace rir { + +/// A 128-bit UUID +#pragma pack(push, 1) +class UUID { + uint64_t high; + uint64_t low; + + UUID(uint64_t a, uint64_t low) : high(a), low(low) {} + + public: + class Hasher; + /// The null UUID (0x0) + UUID() : high(0), low(0) {} + /// Generates a UUID for the data + static UUID hash(const void* data, size_t size); + /// Deserialize a UUID from the R stream + static UUID deserialize(__attribute__((unused)) SEXP refTable, R_inpstream_t inp); + /// Serialize a UUID to the R stream + void serialize(SEXP refTable, R_outpstream_t out) const; + /// Print the UUID as a hexadecimal string + std::string str() const; + + friend std::ostream& operator<<(std::ostream&, const UUID&); + /// `false` iff this is the null UUID (0x0) + explicit operator bool() const; + bool operator==(const UUID& other) const; + bool operator!=(const UUID& other) const; + friend struct std::hash; +}; +#pragma pack(pop) + +/// Create a UUID for a stream of data +class UUID::Hasher { + XXH3_state_t* state; + bool finalized; + + public: + Hasher(); + ~Hasher(); + /// Hash the data-structure, which should not contain any references + template void hashBytesOf(T c) { hashBytes(&c, sizeof(T)); } + /// Hash the C-string + void hashBytesOfCString(const char* c); + /// Hash the data, which should not contain any references + void hashBytes(const void* data, size_t size); + /// Get the UUID. After calling this, you can't call hashBytes anymore. + UUID finalize(); +}; + +} // namespace rir + +namespace std { +template <> +struct hash { + std::size_t operator()(const rir::UUID& v) const; +}; +} // namespace std diff --git a/rir/src/serializeHash/hash/UUIDPool.cpp b/rir/src/serializeHash/hash/UUIDPool.cpp new file mode 100644 index 000000000..c663c2ad2 --- /dev/null +++ b/rir/src/serializeHash/hash/UUIDPool.cpp @@ -0,0 +1,466 @@ +// +// Created by Jakob Hain on 6/1/23. +// + +#include "UUIDPool.h" +#include "R/Printing.h" +#include "R/Protect.h" +#include "R/Serialize.h" +#include "R/disableGc.h" +#include "compiler/parameter.h" +#include "compilerClientServer/CompilerClient.h" +#include "compilerClientServer/CompilerServer.h" +#include "runtime/PoolStub.h" +#include "runtime/ProxyEnv.h" +#include "runtime/log/printPrettyGraphFromEnv.h" +#include "runtime/log/printRirObject.h" +#include "runtime/rirObjectMagic.h" +#include "serializeHash/hash/getConnected.h" +#include "serializeHash/hash/hashRoot.h" +#include "serializeHash/serialize/serialize.h" +#include "utils/measuring.h" + +// Can change this to log interned and uninterned hashes and pointers +#define LOG(stmt) if (pir::Parameter::PIR_LOG_INTERNING) stmt +#define LOG_WARN(stmt) if (pir::Parameter::PIR_LOG_INTERNING || pir::Parameter::PIR_WARN_INTERNING) stmt + +namespace rir { + +bool pir::Parameter::PIR_LOG_INTERNING = + getenv("PIR_LOG_INTERNING") != nullptr && + strcmp(getenv("PIR_LOG_INTERNING"), "") != 0 && + strcmp(getenv("PIR_LOG_INTERNING"), "0") != 0 && + strcmp(getenv("PIR_LOG_INTERNING"), "false") != 0; + +bool pir::Parameter::PIR_WARN_INTERNING = + getenv("PIR_WARN_INTERNING") != nullptr && + strcmp(getenv("PIR_WARN_INTERNING"), "") != 0 && + strcmp(getenv("PIR_WARN_INTERNING"), "0") != 0 && + strcmp(getenv("PIR_WARN_INTERNING"), "false") != 0; + +bool pir::Parameter::PIR_MEASURE_INTERNING = + getenv("PIR_MEASURE_INTERNING") != nullptr && + strtol(getenv("PIR_MEASURE_INTERNING"), nullptr, 10); + + +bool UUIDPool::isInitialized = false; +std::unordered_map UUIDPool::interned; +std::unordered_map UUIDPool::hashes; +std::unordered_map UUIDPool::nextToIntern; +std::unordered_map UUIDPool::prevToIntern; +std::unordered_set UUIDPool::preserved; + +#ifdef DEBUG_DISASSEMBLY +static std::unordered_map disassembly; +#endif + +bool UUIDPool::internable(SEXP sexp) { + // TypeFeedback and ArglistOrder aren't interned, they're serialized inline + // like non-RIR SEXPs because we never need to refer to them alone, identity + // doesn't matter (only equivalence), and they usually aren't big. Plus, + // TypeFeedback changes frequently, so it would need to be re-interned + // frequently + return TYPEOF(sexp) == EXTERNALSXP && + !TypeFeedback::check(sexp) && + !ArglistOrder::check(sexp) && + !PoolStub::check(sexp) && + !ProxyEnv::check(sexp); +} + +#ifdef DO_INTERN +static void registerFinalizerIfPossible(SEXP e, R_CFinalizer_t finalizer) { + switch (TYPEOF(e)) { + case NILSXP: + case ENVSXP: + case EXTPTRSXP: + case BCODESXP: + case EXTERNALSXP: + R_RegisterCFinalizerEx(e, finalizer, (Rboolean) true); + break; + default: + // can't register finalizer, assume these don't get gcd + break; + } +} + +void UUIDPool::initialize() { + assert(!isInitialized); + isInitialized = true; +} + +void UUIDPool::unintern(SEXP e, bool isGettingGcd) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_INTERNING, "UUIDPool.cpp: unintern", e, !isGettingGcd, [&] { + Protect p(e); + assert(hashes.count(e) && "SEXP not interned"); + + // Remove hash + auto hash = hashes.at(e); + hashes.erase(e); + if (!interned.count(hash)) { + LOG_WARN(std::cerr << "WARNING: SEXP was interned, but the " + "corresponding UUID is empty:\n" + << Print::dumpSexp(e) << "\n"); + // Don't return + } + + // Remove from the intern list for this UUID. If this is the first entry, + // update the interned UUID to point to the next SEXP. If there is no next, + // erase the interned UUID since there are no live SEXPs with that hash + // anymore. + if (prevToIntern.count(e)) { + // This isn't the first entry in the list with this UUID + + // Linked list intermediate removal algorithm + auto prev = prevToIntern.at(e); + prevToIntern.erase(e); + assert(nextToIntern.count(prev) && nextToIntern.at(prev) == e); + if (nextToIntern.count(e)) { + auto next = nextToIntern.at(e); + nextToIntern.erase(e); + assert(prevToIntern.count(next) && prevToIntern.at(next) == e); + nextToIntern.at(prev) = next; + prevToIntern.at(next) = prev; + } else { + nextToIntern.erase(prev); + } + LOG(std::cout << "GC intern: " << hash << " -> " << e << "\n"); + } else if (nextToIntern.count(e)) { + // This is the first entry in the list with this UUID, and there is + // another entry + + // Linked list head removal algorithm + auto next = nextToIntern.at(e); + nextToIntern.erase(e); + assert(prevToIntern.count(next) && prevToIntern.at(next) == e); + prevToIntern.erase(next); + + // Replace interned at UUID with the next SEXP + interned.at(hash) = next; + LOG(std::cout << "Switch intern: " << hash << " -> was " << e << " now " << next << "\n"); + } else { + // This is the first and only entry in the list with this UUID + + // Erase interned at UUID + interned.erase(hash); + LOG(std::cout << "Remove intern: " << hash << " -> " << e << "\n"); + } + }); +} + +void UUIDPool::uninternGcd(SEXP e) { + // There seems to be a bug somewhere where R is calls finalizer on the wrong + // object, or calls it twice. Or maybe it's in our code... + if (preserved.count(e)) { + LOG_WARN(std::cerr << "WARNING: preserved SEXP is supposedly getting gcd" + << std::endl); + return; + } + if (!hashes.count(e)) { + // Can happen if we manually unintern, since we can't remove the finalizer + return; + } + + unintern(e, true); +} +#endif + +SEXP UUIDPool::intern(SEXP e, const UUID& hash, bool preserve, bool isSexpComplete) { + return Measuring::timeEventIf2(pir::Parameter::PIR_MEASURE_INTERNING, "UUIDPool.cpp: intern specific", e, isSexpComplete, [&] { + Protect p(e); + assert(internable(e)); + (void)isSexpComplete; + +#ifdef DO_INTERN + if (interned.count(hash)) { + // Reuse interned SEXP + auto existing = interned.at(hash); + assert(TYPEOF(e) == TYPEOF(existing) && "obvious hash collision (different types)"); + assert((TYPEOF(e) != EXTERNALSXP || rirObjectMagic(e) == rirObjectMagic(existing) || !isSexpComplete) && + "obvious hash collision (different RIR types)"); + if (!hashes.count(e)) { + // This SEXP is structurally-equivalent to the interned SEXP but not + // the same (different pointers), so we must still record it + LOG(std::cout << "Reuse intern: " << hash << " -> " << e << (isSexpComplete ? "\n" : " (recursive)\n")); + hashes[e] = hash; + + // Add to intern list for this UUID + auto oldLast = existing; + while (nextToIntern.count(oldLast)) { + oldLast = nextToIntern.at(oldLast); + } + nextToIntern[oldLast] = e; + prevToIntern[e] = oldLast; + + // And register finalizer + if (!preserve) { + registerFinalizerIfPossible(e, uninternGcd); + } + } + // If preserve = true, we want to preserve both this SEXP and the + // interned one, because we could later fetch either. In the future, + // we can probably switch e to be existing so we don't need to + // preserve redundant SEXPs like this. + if (preserve && !preserved.count(e)) { + // Preserve this SEXP + R_PreserveObject(e); + preserved.insert(e); + } + e = existing; + if (preserve && !preserved.count(e)) { + // Preserve the interned SEXP + R_PreserveObject(e); + preserved.insert(e); + } + return e; + } + + // Intern new SEXP +#ifdef DEBUG_DISASSEMBLY + disassembly[hash] = + isSexpComplete + ? printRirObject(e, RirObjectPrintStyle::Detailed) + : "(couldn't be computed at the time it was interned)"; +#endif + + // Sanity check in case the UUID changed + if (hashes.count(e)) { + LOG_WARN(std::cerr << "SEXP UUID changed from " << hashes.at(e) + << " to " << hash << ": " << e << "\n" + << Print::dumpSexp(e) << "\n"); + +#ifdef DEBUG_DISASSEMBLY + auto oldDisassembly = disassembly[hashes.at(e)]; + auto newDisassembly = disassembly[hash]; + if (oldDisassembly != newDisassembly) { + LOG_WARN(std::cerr << "note: disassembly changed from:\n" + << oldDisassembly << "\nto:\n" + << newDisassembly << "\n"); + } else { + LOG_WARN(std::cerr << "note: disassembly:\n" << oldDisassembly + << "\n"); + } +#endif + + // assert(false); + LOG_WARN(std::cerr << "WARNING: SEXP UUID changed. Unsound, and " + "semantic errors may occur if we rely on " + "outdated behavior\n"); + // DON'T unintern because we or the compiler peer may request it + // from the old hash. + } + + // Do intern + LOG(std::cout << "New intern: " << hash << " -> " << e << "\n"); +#ifdef DEBUG_DISASSEMBLY + LOG(std::cout << "Disassembly:\n" << disassembly[hash] << "\n"); +#endif + if (isSexpComplete) { + printPrettyGraphOfInternedIfNecessary(e, hash); + } + interned[hash] = e; + hashes[e] = hash; + + // Preserve or register finalizer + if (preserve) { + R_PreserveObject(e); + preserved.insert(e); + } else { + registerFinalizerIfPossible(e, uninternGcd); + } +#endif + + return e; + }); +} + +SEXP UUIDPool::intern(SEXP e, bool recursive, bool preserve) { +#ifdef DO_INTERN + return disableGc2([&]{ + return Measuring::timeEventIf2(pir::Parameter::PIR_MEASURE_INTERNING, recursive ? "UUIDPool.cpp: intern recursive" : "UUIDPool.cpp: intern", e, [&] { + if (hashes.count(e) && !recursive) { + // Already interned, don't compute hash + if (preserve && !preserved.count(e)) { + R_PreserveObject(e); + preserved.insert(e); + } + return e; + } + auto ret = internable(e) ? intern(e, hashRoot(e), preserve) : e; + if (recursive) { + ConnectedSet connected = getConnected(e); + for (auto sexp : connected) { + if (hashes.count(sexp) || !internable(sexp)) { + continue; + } + + intern(sexp, hashRoot(sexp), preserve); + } + } + return ret; + }); + }); +#else + return e; +#endif +} + +SEXP UUIDPool::reintern(SEXP e) { +#ifdef DO_INTERN + // This is called before everything is initialized, so we need to ensure + // that isInitialized is set before we check hashes or we will crash + if (isInitialized && hashes.count(e)) { + unintern(e); + return Measuring::timeEventIf2(pir::Parameter::PIR_MEASURE_INTERNING, "UUIDPool.cpp: reintern", e, [&] { + return intern(e, false, false); + }); + } +#endif + return e; +} + +SEXP UUIDPool::get(const UUID& hash) { +#ifdef DO_INTERN + if (interned.count(hash)) { + return interned.at(hash); + } +#endif + return nullptr; +} + +const UUID& UUIDPool::getHash(SEXP sexp) { +#ifdef DO_INTERN + if (hashes.count(sexp)) { + return hashes.at(sexp); + } +#endif + static UUID empty; + return empty; +} + +SEXP UUIDPool::retrieve(const UUID& hash) { + if (interned.count(hash)) { + LOG(std::cout << "Retrieved by hash locally: " << hash << " -> " + << interned.at(hash) << "\n"); + return interned.at(hash); + } + if (CompilerClient::isRunning()) { + LOG(std::cout << "Retrieving by hash from server: " << hash << "\n"); + auto sexp = CompilerClient::retrieve(hash); + LOG(std::cout << "Retrieved by hash from server: " << hash << " -> " + << sexp << "\n"); + if (sexp) { +#ifdef DEBUG_DISASSEMBLY + disassembly[hash] = printRirObject(sexp, RirObjectPrintStyle::Detailed); + LOG(std::cout << "Disassembly:\n" << disassembly[hash] << "\n"); +#endif + intern(sexp, hash, false, true); + return sexp; + } + Rf_error("SEXP deserialized from hash which we don't have, and server also doesn't have it"); + } else if (CompilerServer::isRunning()) { + LOG(std::cout << "Retrieving by hash from client: " << hash << "\n"); + auto sexp = CompilerServer::retrieve(hash); + LOG(std::cout << "Retrieved by hash from client: " << hash << " -> " + << sexp << "\n"); + if (sexp) { +#ifdef DEBUG_DISASSEMBLY + disassembly[hash] = printRirObject(sexp, RirObjectPrintStyle::Detailed); + LOG(std::cout << "Disassembly:\n" << disassembly[hash] << "\n"); +#endif + intern(sexp, hash, true, true); + return sexp; + } + LOG(std::cout << "SEXP deserialized from hash which we don't have, and client also doesn't have it"); + // TODO: Should we be returning this, or returning an explicit "not + // found" token, or still erroring and instead handling explicitly + // via a separate method, maybe renaming the old tryReadHash to + // readHashIfNecessary and the new one to tryReadHashIfNecessary? + return R_NilValue; + } + Rf_error("SEXP deserialized from hash which we don't have, and no server"); +} + +SEXP UUIDPool::tryReadHash(R_inpstream_t inp) { + auto readHashInstead = InBool(inp); + if (readHashInstead) { + // Read hash instead of regular data, + // then retrieve by hash from interned or peer + UUID hash; + InBytes(inp, &hash, sizeof(hash)); + return retrieve(hash); + } + return nullptr; +} + +bool UUIDPool::tryWriteHash(SEXP sexp, R_outpstream_t out) { + auto writeHash = internable(sexp); + // Write whether we are serializing hash + OutBool(out, writeHash); + if (writeHash) { + // Write hash instead of regular data + if (!hashes.count(sexp)) { + LOG(std::cout << "Interning new SEXP at write: " << sexp << "\n"); + intern(sexp, hashRoot(sexp), false); + } + // cppcheck is wrong, this is read + // cppcheck-suppress unreadVariable + auto hash = hashes.at(sexp); + OutBytes(out, &hash, sizeof(hash)); + } + return writeHash; +} + +SEXP UUIDPool::tryReadHash(const ByteBuffer& buf) { + auto readHashInstead = buf.getBool(); + if (readHashInstead) { + // Read hash instead of regular data, + // then retrieve by hash from interned or peer + UUID hash; + buf.getBytes((uint8_t*)&hash, sizeof(hash)); + return retrieve(hash); + } + return nullptr; +} + +bool UUIDPool::tryWriteHash(SEXP sexp, ByteBuffer& buf) { + auto writeHash = internable(sexp); + // Write whether we are serializing hash + buf.putBool(writeHash); + if (writeHash) { + // Write hash instead of regular data + if (!hashes.count(sexp)) { + LOG(std::cout << "Interning new SEXP at write: " << sexp << "\n"); + intern(sexp, hashRoot(sexp), false); + } + // cppcheck is wrong, this is read + // cppcheck-suppress unreadVariable + auto hash = hashes.at(sexp); + buf.putBytes((uint8_t*)&hash, sizeof(hash)); + } + return writeHash; +} + +SEXP UUIDPool::readItem(const ByteBuffer& buf, bool useHashes) { + if (useHashes) { + if (auto result = tryReadHash(buf)) { + return result; + } + } + + // Read regular data + return deserialize(buf, SerialOptions{useHashes, useHashes, false, nullptr, SerialOptions::SourcePools()}); +} + +void UUIDPool::writeItem(SEXP sexp, __attribute__((unused)) bool isChild, + ByteBuffer& buf, bool useHashes) { + if (useHashes) { + if (tryWriteHash(sexp, buf)) { + return; + } + } + + // Write regular data + serialize(sexp, buf, SerialOptions{useHashes, useHashes, false, nullptr, SerialOptions::SourcePools()}); +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/UUIDPool.h b/rir/src/serializeHash/hash/UUIDPool.h new file mode 100644 index 000000000..8d837d93d --- /dev/null +++ b/rir/src/serializeHash/hash/UUIDPool.h @@ -0,0 +1,121 @@ +// +// Created by Jakob Hain on 6/1/23. +// + +#pragma once + +#include "R/r.h" +#include "UUID.h" +#include "bc/BC_inc.h" +#include "interpreter/instance.h" +#include "utils/ByteBuffer.h" + +#include +#include +#include + +#define DO_INTERN + +namespace rir { + +/// A global set of SEXPs identified by a unique UUID computed by hash. +/// Structurally equivalent SEXPs will have the same UUID, and structurally +/// different SEXPs will, with extremely high probability, have different UUIDs. +/// "Structurally equivalent" means that an SEXP's UUID is independent of its +/// address in memory, and even different R sessions can identify structurally- +/// equivalent SEXPs by the same UUID. +/// +/// The UUID is computed by hashing the SEXP's serialized form. When serializing +/// an SEXP, we only serialize hashes to connected RIR objects, to avoid +/// serializing copies of SEXPs we already have and then effectively duplicating +/// them by deserializing. However, when we serialize an SEXP to compute its +/// hash, we always serialize the connected objects, because some of those +/// connections may be cyclic and we a) need to handle this via refs (we use R's +/// ref-table) and b) want the refs to be deterministic (which requires the +/// "hash" of the connected object to be different than what we get from hashing +/// object directly, because the numbers and expansion of the refs differ). +/// +/// Each SEXP in the set has a WeakRef finalizer which will remove the SEXP when +/// it's garbage collected, so the pool won't continually increase in size. +/// Sometimes SEXPs need to be remembered (by the compiler server), in which +/// case `UUIDPool::intern(,,true)` will preserve them so they never get freed. +class UUIDPool { + static bool isInitialized; + static std::unordered_map interned; + static std::unordered_map hashes; + /// This and `prevToIntern` effectively form multiple double-linked lists of + /// SEXPs with the same UUID hash (one list for each hash) in the order we + /// would assign them to be the "interned" SEXP for the UUID; when the + /// "interned" SEXP gets gcd, we replace it with the next SEXP in the list, + /// otherwise we remove the UUID because there is no longer a corresponding + /// live SEXP. + static std::unordered_map nextToIntern; + /// See `nextToIntern` doc + static std::unordered_map prevToIntern; + static std::unordered_set preserved; + +#ifdef DO_INTERN + static void unintern(SEXP e, bool isGettingGcd = false); + static void uninternGcd(SEXP e); +#endif + + public: + static void initialize(); + + /// Whether the SEXP can be interned, and is serialized as a hash when + /// interned SEXPs are. + static bool internable(SEXP sexp); + + /// Intern the SEXP when we already know its hash, not recursively. + /// + /// \see UUIDPool::intern(SEXP, bool, bool) + static SEXP intern(SEXP e, const UUID& hash, bool preserve, + bool isSexpComplete = true); + /// Will hash the SEXP and: + /// - If not in the pool, will add it *and* if `recursive` is set, + /// recursively intern connected SEXPs. Then returns the original SEXP + /// - If already in the pool, returns the existing SEXP + static SEXP intern(SEXP e, bool recursive, bool preserve); + /// If SEXP is in the intern pool, re-compute its hash and remove/re-add it. + /// Returns a different SEXP if there already exists an interned SEXP with + /// the recomputed hash. + static SEXP reintern(SEXP e); + + /// Gets the interned SEXP by hash, or nullptr if not interned + static SEXP get(const UUID& hash); + /// Gets the SEXP if interned locally, otherwise sends a request to the + /// compiler peer. If the compiler peer doesn't have it, calls `Rf_error` on + /// the client and returns `R_NilValue` on the server. + static SEXP retrieve(const UUID& hash); + /// Gets the SEXP's memoized hash, or the null hash if the SEXP was never + /// interned + static const UUID& getHash(SEXP sexp); + + /// \see tryReadHash(const ByteBuffer&) + static SEXP tryReadHash(R_inpstream_t in); + /// \see tryWriteHash(SEXP, ByteBuffer&) + static bool tryWriteHash(SEXP sexp, R_outpstream_t out); + /// Reads a boolean. If `true`, reads a hash and returns the interned SEXP, + /// fetching from the compiler peer if necessary. If `false`, returns + /// `nullptr`. + /// + /// If this is the compiler server and the compiler client doesn't have the + /// hash, it will return `R_NilValue`. If this is the client and the server + /// doesn't have the hash, it will `Rf_error`. This is the same behavior of + /// `UUIDPool::readItem`. + static SEXP tryReadHash(const ByteBuffer& buf); + + /// If the SEXP is internable, writes `true`, writes its hash, then returns + /// `true`. Otherwise, writes `false`, then returns `false`. + /// + /// This will intern the SEXP if it's not already interned, unlike + /// `writeItem` which will error. + static bool tryWriteHash(SEXP sexp, ByteBuffer& buf); + /// Calls `tryReadHash`, otherwise reads normally. + static SEXP readItem(const ByteBuffer& buf, bool useHashes); + /// Calls `tryWriteHash`, otherwise writes normally. `isChild` is unused, + /// but may be an optimization in the future. + static void writeItem(SEXP sexp, bool isChild, ByteBuffer& buf, bool useHashes); +}; + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/getConnected.cpp b/rir/src/serializeHash/hash/getConnected.cpp new file mode 100644 index 000000000..63f97fd4a --- /dev/null +++ b/rir/src/serializeHash/hash/getConnected.cpp @@ -0,0 +1,51 @@ +// +// Created by Jakob Hain on 8/15/23. +// + +// Connected objects are currently different, not worth making them identical +// though as long as they both work the same +#define DEBUG_CONNECTED_DIFFERENCES 0 + +#include "getConnected.h" +#include "getConnectedOld.h" +#include "getConnectedUni.h" +#if DEBUG_CONNECTED_DIFFERENCES +#include "R/Printing.h" +#include +#include +#endif + +namespace rir { + +ConnectedSet getConnected(SEXP root) { +#if defined(ENABLE_SLOWASSERT) || DEBUG_CONNECTED_DIFFERENCES + auto set1 = getConnectedUni(root); +#endif + auto set2 = getConnectedOld(root); +#if DEBUG_CONNECTED_DIFFERENCES + std::unordered_set set1MinusSet2; + std::unordered_set set2MinusSet1; + std::set_difference(set1.begin(), set1.end(), set2.begin(), set2.end(), + std::inserter(set1MinusSet2, set1MinusSet2.begin())); + std::set_difference(set2.begin(), set2.end(), set1.begin(), set1.end(), + std::inserter(set2MinusSet1, set2MinusSet1.begin())); + if (!set1MinusSet2.empty()) { + std::cerr << "getConnectedUni has " << set1MinusSet2.size() << " elements not in getConnectedOld:\n"; + for (auto e : set1MinusSet2) { + std::cerr << " " << Print::dumpSexp(e, 75) << "\n"; + } + } + if (!set2MinusSet1.empty()) { + std::cerr << "getConnectedOld has " << set2MinusSet1.size() << " elements not in getConnectedUni:\n"; + for (auto e : set2MinusSet1) { + std::cerr << " " << Print::dumpSexp(e, 75) << "\n"; + } + } +#elif defined(ENABLE_SLOWASSERT) + (void)set1; +#endif + + return set2; +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/getConnected.h b/rir/src/serializeHash/hash/getConnected.h new file mode 100644 index 000000000..f84e55d69 --- /dev/null +++ b/rir/src/serializeHash/hash/getConnected.h @@ -0,0 +1,33 @@ +// +// Created by Jakob Hain on 8/15/23. +// + +#pragma once + +#include "R/r_incl.h" +#include + +namespace rir { + +/// Set of RIR SEXPs connected to another SEXP +class ConnectedSet { + std::unordered_set seen; + + friend ConnectedSet getConnectedOld(SEXP root); + friend ConnectedSet getConnectedUni(SEXP root); + friend ConnectedSet getConnected(SEXP root); + friend class ConnectedCollectorOld; + friend class ConnectedCollectorUni; + ConnectedSet() : seen() {} + bool insert(SEXP e) { return seen.insert(e).second; } + + public: + using const_iterator = std::unordered_set::const_iterator; + const_iterator begin() const { return seen.begin(); } + const_iterator end() const { return seen.end(); } +}; + +/// Get RIR SEXPs connected to this SEXP. Used during recursive interning. +ConnectedSet getConnected(SEXP root); + +} // namespace rir diff --git a/rir/src/serializeHash/hash/getConnectedOld.cpp b/rir/src/serializeHash/hash/getConnectedOld.cpp new file mode 100644 index 000000000..ef2b82182 --- /dev/null +++ b/rir/src/serializeHash/hash/getConnectedOld.cpp @@ -0,0 +1,205 @@ +// +// Created by Jakob Hain on 7/23/23. +// + +#include "getConnectedOld.h" +#include "R/r.h" +#include "compiler/parameter.h" +#include "runtime/Code.h" +#include "runtime/DispatchTable.h" +#include "runtime/Function.h" +#include "runtime/LazyArglist.h" +#include "runtime/LazyEnvironment.h" +#include "runtime/PoolStub.h" +#include "runtime/ProxyEnv.h" +#include "serializeHash/globals.h" +#include "serializeHash/hash/hashRoot_getConnected_common.h" +#include "utils/Pool.h" +#include "utils/measuring.h" + +namespace rir { + +// Will hash sexp if it's an instance of CLS +template +static inline bool tryAddConnected(SEXP sexp, + ConnectedCollectorOld& collector) { + if (CLS* b = CLS::check(sexp)) { + b->addConnected(collector); + return true; + } else { + return false; + } +} + +static inline void addConnectedRir(SEXP sexp, + ConnectedCollectorOld& collector) { + if (!tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector) && + !tryAddConnected(sexp, collector)) { + std::cerr << "couldn't add connected in EXTERNALSXP: "; + Rf_PrintValue(sexp); + assert(false); + } +} + +static void addConnectedBc1(SEXP sexp, ConnectedCollectorOld& collector, + std::queue& bcWorklist) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_INTERNING, "getConnected.cpp: addConnectedBc1", sexp, [&] { + auto consts = BCODE_CONSTS(sexp); + auto n = LENGTH(consts); + for (auto i = 0; i < n; i++) { + auto c = VECTOR_ELT(consts, i); + // Adds to collector either way, but bcWorklist may (?) be faster + // (this weird function structure is what R does with serialization) + if (TYPEOF(c) == BCODESXP) { + bcWorklist.push(c); + } else { + collector.add(c); + } + } + }); +} + +static void addConnectedBc(SEXP sexp, ConnectedCollectorOld& collector) { + std::queue bcWorklist; + bcWorklist.push(sexp); + while (!bcWorklist.empty()) { + sexp = bcWorklist.front(); + bcWorklist.pop(); + + addConnectedBc1(sexp, collector, bcWorklist); + } +} + +static void addConnected(SEXP sexp, ConnectedCollectorOld& collector) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_INTERNING, "getConnected.cpp: addConnected", sexp, [&] { + auto type = TYPEOF(sexp); + if (ALTREP(sexp)) { + auto info = ALTREP_SERIALIZED_CLASS(sexp); + auto state = ALTREP_SERIALIZED_STATE(sexp); + auto attrib = ATTRIB(sexp); + if (info != nullptr && state != nullptr) { + collector.add(info); + collector.add(state); + collector.add(attrib); + return; + } + /* else fall through to standard processing */ + } else if (globalsSet.count(sexp)) { + return; + } + + // With the CHARSXP cache chains maintained through the ATTRIB + // field the content of that field must not be serialized, so + // we treat it as not there. + auto hasAttr = (type != CHARSXP && ATTRIB(sexp) != R_NilValue); + if (hasAttr) { + collector.add(ATTRIB(sexp)); + } + + switch (type) { + case NILSXP: + case SYMSXP: + break; + case LISTSXP: + // LANGSXP can contain RIR objects (perhaps in its tag) + case LANGSXP: + case PROMSXP: + case DOTSXP: + if (hasTag(sexp)) { + collector.add(TAG(sexp)); + } + if (BNDCELL_TAG(sexp)) { + assert(false && "TODO R_expand_binding_value isn't public"); + } + collector.add(CAR(sexp)); + // ???: use goto tailcall like R for perf boost? + collector.add(CDR(sexp)); + break; + case CLOSXP: + collector.add(CLOENV(sexp)); + collector.add(FORMALS(sexp)); + // ???: use goto tailcall like R for perf boost? + collector.add(BODY(sexp)); + break; + case EXTPTRSXP: + collector.add(EXTPTR_PROT(sexp)); + collector.add(EXTPTR_TAG(sexp)); + break; + case WEAKREFSXP: + break; + case ENVSXP: + if (!R_IsPackageEnv(sexp) && !R_IsNamespaceEnv(sexp)) { + collector.add(ENCLOS(sexp)); + collector.add(FRAME(sexp)); + collector.add(HASHTAB(sexp)); + collector.add(ATTRIB(sexp)); + } + break; + case SPECIALSXP: + case BUILTINSXP: + case CHARSXP: + case LGLSXP: + case INTSXP: + case REALSXP: + case CPLXSXP: + case RAWSXP: + case STRSXP: + break; + case EXPRSXP: + case VECSXP: { + auto n = XLENGTH(sexp); + for (int i = 0; i < n; ++i) { + collector.add(VECTOR_ELT(sexp, i)); + } + break; + } + case S4SXP: + break; + case BCODESXP: { + addConnectedBc(sexp, collector); + break; + } + case EXTERNALSXP: + addConnectedRir(sexp, collector); + break; + default: + Rf_error("hashChild: unknown type %i", type); + } + }); +} + +void ConnectedCollectorOld::addConstant(unsigned idx) { + add(Pool::get(idx)); +} + +void ConnectedCollectorOld::addSrc(unsigned idx) { + add(src_pool_at(idx)); +} + +ConnectedSet getConnectedOld(SEXP root) { + ConnectedSet set; + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_INTERNING, "getConnected", root, [&] { + std::queue worklist; + worklist.push(root); + ConnectedCollectorOld collector(set, worklist); + + while (!worklist.empty()) { + auto sexp = worklist.front(); + worklist.pop(); + + addConnected(sexp, collector); + } + }); + return set; +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/getConnectedOld.h b/rir/src/serializeHash/hash/getConnectedOld.h new file mode 100644 index 000000000..c6e8333ff --- /dev/null +++ b/rir/src/serializeHash/hash/getConnectedOld.h @@ -0,0 +1,51 @@ +// +// Created by Jakob Hain on 7/23/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "getConnected.h" +#include +#include + +namespace rir { + +/// Facade to add connected RIR SEXPs which is exposed to RIR objects. +class ConnectedCollectorOld { + /// Underlying connected set + ConnectedSet& set; + /// Next SEXPs to process: instead of recursing, we add nested SEXPs to this + /// queue and then process them in a loop. + std::queue& worklist; + + ConnectedCollectorOld(ConnectedSet& set, std::queue& worklist) + : set(set), worklist(worklist) {} + + friend ConnectedSet getConnectedOld(SEXP root); + + public: + /// Add connected objects in SEXP, which may or may not be a RIR object + /// itself. isChild is currently unused but may be for an optimization + /// later. + void add(SEXP s, __attribute__((unused)) bool isChild = false) { + if (set.insert(s)) { + worklist.push(s); + } + } + /// Add connected objects in SEXP in constant pool ([Pool]) + void addConstant(unsigned idx); + /// Add connected objects in SEXP in source pool ([src_pool_at]) + void addSrc(unsigned idx); + /// Add connected objects in SEXP which could be nullptr + void addNullable(SEXP s, bool isChild = false) { + if (s) { + add(s, isChild); + } + } +}; + +/// Get RIR SEXPs connected to this SEXP. Used during recursive interning. +ConnectedSet getConnectedOld(SEXP root); + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/getConnectedUni.cpp b/rir/src/serializeHash/hash/getConnectedUni.cpp new file mode 100644 index 000000000..ac4d2fe34 --- /dev/null +++ b/rir/src/serializeHash/hash/getConnectedUni.cpp @@ -0,0 +1,62 @@ +// +// Created by Jakob Hain on 7/23/23. +// + +#include "getConnectedUni.h" +#include "R/r.h" +#include "compiler/parameter.h" +#include "runtime/LazyArglist.h" +#include "serializeHash/serialize/serialize.h" +#include "utils/measuring.h" + +namespace rir { + +SerialOptions& ConnectedCollectorUni::serialOptions() const { + // Doesn't matter what we return here, but we unfortunately need something + return SerialOptions::DeepCopy; +} + +bool ConnectedCollectorUni::willWrite(const rir::SerialFlags& flags) const { + // We only care about writing SEXPs, all other writes are no-ops. + // This also skips the assertion Code.cpp which requires the native code + // object to be ready for serialization (which it's not, but we're not + // actually serializing) + return flags.contains(SerialFlag::MaybeSexp); +} + +void ConnectedCollectorUni::write(SEXP s, const rir::SerialFlags& flags) { + assert(flags.contains(SerialFlag::MaybeSexp) && + "Hashing non SEXP with SEXP flag"); + + if (!willWrite(flags)) { + return; + } + + if (set.insert(s)) { + worklist.push(s); + } +} + +void ConnectedCollectorUni::doGetConnected(SEXP root) { + set.insert(root); + writeInline(root); + while (!worklist.empty()) { + auto elem = worklist.front(); + worklist.pop(); + + writeInline(elem); + } +} + +ConnectedSet getConnectedUni(SEXP root) { + ConnectedSet set; + disableInterpreter([&]{ + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_INTERNING, "getConnected", root, [&] { + ConnectedCollectorUni collector(set); + collector.doGetConnected(root); + }); + }); + return set; +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/getConnectedUni.h b/rir/src/serializeHash/hash/getConnectedUni.h new file mode 100644 index 000000000..3d669f57c --- /dev/null +++ b/rir/src/serializeHash/hash/getConnectedUni.h @@ -0,0 +1,41 @@ +// +// Created by Jakob Hain on 7/23/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "getConnected.h" +#include "serializeHash/serializeUni.h" +#include +#include + +namespace rir { + +/// Facade to add connected RIR SEXPs which is exposed to RIR objects. +class ConnectedCollectorUni : AbstractSerializer { + /// Underlying connected set + ConnectedSet& set; + /// Next SEXPs to process: instead of recursing, we add nested SEXPs to this + /// queue and then process them in a loop. + std::queue worklist; + + explicit ConnectedCollectorUni(ConnectedSet& set) + : set(set), worklist() {} + + SerializedRefs* refs() override { return nullptr; } + void doGetConnected(SEXP root); + friend ConnectedSet getConnectedUni(SEXP root); + public: + SerialOptions& serialOptions() const override; + bool willWrite(const SerialFlags& flags) const override; + void writeBytes(const void *data, size_t size, + const SerialFlags& flags) override {} + void writeInt(int data, const SerialFlags& flags) override {} + void write(SEXP s, const SerialFlags& flags) override; +}; + +/// Get RIR SEXPs connected to this SEXP. Used during recursive interning. +ConnectedSet getConnectedUni(SEXP root); + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/hashAst.cpp b/rir/src/serializeHash/hash/hashAst.cpp new file mode 100644 index 000000000..68897b3b0 --- /dev/null +++ b/rir/src/serializeHash/hash/hashAst.cpp @@ -0,0 +1,302 @@ +#include "hashAst.h" +#include "R/Funtab.h" +#include "R/Symbols.h" +#include "compiler/parameter.h" +#include "interpreter/instance.h" +#include "runtime/DispatchTable.h" +#include "utils/measuring.h" +#include +#include +#include + +namespace rir { + +// Assumes all symbols are never freed (currently yes because they're in a pool, +// and it makes sense since they're all AST nodes that they're persistent) +static std::unordered_map* hashCache; + +void initAstHashCache() { + hashCache = new std::unordered_map(); +} + +inline static void +serializeAstVector(SEXP s, const std::function& serializeElem) { + // These haven't caused problems yet, but maybe need to be handled? + // assert(ATTRIB(s) == R_NilValue && "unexpected attributes in AST"); + // assert(!OBJECT(s) && "unexpected object in AST"); + // assert(!IS_S4_OBJECT(s) && "unexpected S4 object in AST"); + // assert(!ALTREP(s) && "unexpected altrep in AST"); + size_t length = LENGTH(s); + for (size_t i = 0; i < length; ++i) { + serializeElem(i); + } +} + +/// Manual tagged union simulating a stack frame of a function which takes an +/// SEXP and creates a UUID from hashing. +struct Frame { + bool started = false; + Frame* parent; + unsigned parentIdx; + SEXP sexp; + UUID::Hasher hasher; + std::vector children; + + explicit Frame(Frame* parent, SEXP sexp) + : started(false), parent(parent), + parentIdx(parent ? parent->children.size() : 0), sexp(sexp), hasher(), + children() {} + + UUID finalize() { + for (auto child : children) { + hasher.hashBytesOf(child); + } + return hasher.finalize(); + } +}; +using Stack = std::stack; + +static void hashNewAst(SEXP s, UUID::Hasher& hasher, + std::function recurse) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashAst.cpp: hashNewAst", s, [&]{ + // 2 fastcases below mean that every SEXP on the stack is not yet hashed, + // unless the symbol is a shared global, in which case it's trivial. So we + // don't bother checking if it's in hashCache + + hasher.hashBytesOf(TYPEOF(s)); + switch (TYPEOF(s)) { + case NILSXP: { + break; + } + + case SYMSXP: { + if (s == R_UnboundValue) { + hasher.hashBytesOf(0); + } else if (s == R_MissingArg) { + hasher.hashBytesOf(1); + } else if (s == R_RestartToken) { + hasher.hashBytesOf(2); + } else if (s == symbol::expandDotsTrigger) { + hasher.hashBytesOf(3); + } else { + hasher.hashBytesOf(4); + const char* name = CHAR(PRINTNAME(s)); + hasher.hashBytesOf(strlen(name)); + hasher.hashBytes((const void*)name, strlen(name)); + } + break; + } + + case LISTSXP: + case LANGSXP: { + hasher.hashBytesOf(Rf_length(s)); + for (SEXP cur = s; cur != R_NilValue; cur = CDR(cur)) { + recurse(CAR(cur)); + auto tag = TAG(cur); + hasher.hashBytesOf(tag != R_NilValue); + if (tag) { + recurse(tag); + } + } + break; + } + + case PROMSXP: { + assert(false && "unexpected PROMSXP in AST"); + } + + case DOTSXP: { + assert(false && "unexpected DOTSXP in AST"); + } + + case ENVSXP: { + assert(false && "unexpected ENVSXP in AST"); + } + + // Not sure if this should actually happen or if it's a bug in RIR, but + // this is encountered in regression_intern_reg_s4.R + case CLOSXP: { + auto body = BODY(s); + SEXP src; + switch (TYPEOF(body)) { + case EXTERNALSXP: + src = src_pool_at(DispatchTable::unpack(body)->baseline()->body()->src); + break; + case BCODESXP: + src = VECTOR_ELT(CDR(body), 0); + break; + case LANGSXP: + // These cases should maybe be part of default + case SYMSXP: + case NILSXP: + case LISTSXP: + case SPECIALSXP: + case BUILTINSXP: + case CHARSXP: + case LGLSXP: + case INTSXP: + case REALSXP: + case CPLXSXP: + case STRSXP: + case VECSXP: + case RAWSXP: + src = body; + break; + default: + assert(false && "unexpected body type in AST closure"); + } + recurse(src); + break; + } + + case SPECIALSXP: + case BUILTINSXP: { + hasher.hashBytesOf(getBuiltinNr(s)); + break; + } + + case CHARSXP: { + if (s == NA_STRING) { + hasher.hashBytesOf(0); + } else { + hasher.hashBytesOf(1); + const char* chr = CHAR(s); + hasher.hashBytesOf(strlen(chr)); + hasher.hashBytes((const void*)chr, strlen(chr)); + } + break; + } + + case LGLSXP: { + serializeAstVector(s, [&](int i) { + hasher.hashBytesOf(LOGICAL(s)[i]); + }); + break; + } + + case INTSXP: { + serializeAstVector(s, [&](int i) { + hasher.hashBytesOf(INTEGER(s)[i]); + }); + break; + } + + case REALSXP: { + serializeAstVector(s, [&](int i) { + hasher.hashBytesOf(REAL(s)[i]); + }); + break; + } + + case CPLXSXP: { + serializeAstVector(s, [&](int i) { + hasher.hashBytesOf(COMPLEX(s)[i]); + }); + break; + } + + case STRSXP: { + serializeAstVector(s, [&](int i) { + const char* chr = CHAR(STRING_ELT(s, i)); + hasher.hashBytesOf(strlen(chr)); + hasher.hashBytes((const void*)chr, strlen(chr)); + }); + break; + } + + case VECSXP: { + serializeAstVector(s, [&](int i) { + recurse(VECTOR_ELT(s, i)); + }); + break; + } + + case RAWSXP: { + serializeAstVector(s, [&](int i) { + hasher.hashBytesOf(RAW(s)[i]); + }); + break; + } + + case EXTERNALSXP: { + assert(false && "unexpected RIR object in AST"); + } + + case ANYSXP: + case EXPRSXP: + case BCODESXP: + case WEAKREFSXP: + case EXTPTRSXP: + case S4SXP: + case NEWSXP: + case FREESXP: + default: { + assert(false && "unexpected type in AST"); + } + } + }); +} + +UUID hashAst(SEXP root) { + UUID result; + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashAst", root, [&]{ + // Fastcase + if (hashCache->count(root)) { + result = hashCache->at(root); + return; + } + + // Simulate a recursive call chain. Is this better or even as good letting + // the compiler do it? (There are a few differences in semantics from + // regular recursion which don't affect hash quality, like putting all SEXPs + // at the end) + Stack stack; + stack.emplace(nullptr, root); + while (true) { + auto& top = stack.top(); + // Hash this SEXP, changing the hasher and pushing not-started recursive + // calls onto the stack + top.started = true; + hashNewAst(top.sexp, top.hasher, [&](SEXP next){ + if (hashCache->count(next)) { + // Fastcase + top.children.push_back(hashCache->at(next)); + } else { + stack.emplace(&top, next); + // Push null UUID to be filled in later. Need to push after + // emplace because the emplaced Frame uses the vector's size + // as its parent index + top.children.emplace_back(); + } + }); + + // If this SEXP pushed not-started recursive calls we have to + // process them. If not, we can finish this call, and then finish + // outer calls which also have no more not-started recursive calls. + while (stack.top().started) { + auto parent = stack.top().parent; + auto parentIdx = stack.top().parentIdx; + auto sexp = stack.top().sexp; + auto hash = stack.top().finalize(); + (*hashCache)[sexp] = hash; + stack.pop(); + if (parent) { + // The SEXP's hash is part of the parent's hash. + parent->children[parentIdx] = hash; + } else { + // Done + assert(parentIdx == 0); + result = hash; + return; + } + } + } + }); + return result; +} + +UUID hashDecompiled(SEXP decompiledClosure) { + return hashAst(BODY(decompiledClosure)); +} + +} // namespace rir diff --git a/rir/src/serializeHash/hash/hashAst.h b/rir/src/serializeHash/hash/hashAst.h new file mode 100644 index 000000000..4ed0c1f42 --- /dev/null +++ b/rir/src/serializeHash/hash/hashAst.h @@ -0,0 +1,16 @@ +#pragma once + +#include "R/r.h" +#include "UUID.h" + +namespace rir { + +void initAstHashCache(); + +/// Create a UUID from only the AST part of a SEXP. +UUID hashAst(SEXP s); + +/// Create a UUID from the AST of a decompiled closure's body +UUID hashDecompiled(SEXP decompiledClosure); + +} // namespace rir diff --git a/rir/src/serializeHash/hash/hashRoot.cpp b/rir/src/serializeHash/hash/hashRoot.cpp new file mode 100644 index 000000000..2f981b5a5 --- /dev/null +++ b/rir/src/serializeHash/hash/hashRoot.cpp @@ -0,0 +1,38 @@ +// +// Created by Jakob Hain on 8/15/23. +// + +// Hashes are currently different, not worth making them identical though as +// long as they both work the same +#define DEBUG_HASH_DIFFERENCES 0 + +#include "hashRoot.h" +#include "hashRootOld.h" +#include "hashRootUni.h" +#if DEBUG_HASH_DIFFERENCES +#include "runtime/log/printRirObject.h" +#include +#endif + +namespace rir { + +UUID hashRoot(SEXP root) { +#if defined(ENABLE_SLOWASSERT) || DEBUG_HASH_DIFFERENCES + auto uuid1 = hashRootUni(root); +#endif + auto uuid2 = hashRootOld(root); +#if DEBUG_HASH_DIFFERENCES + if (uuid1 != uuid2) { + std::cerr << "hashRootOld and hashRootUni disagree:\n"; + std::cerr << " "; + printRirObject(root, std::cerr); + std::cerr << "\n"; + } +#elif defined(ENABLE_SLOWASSERT) + (void)uuid1; +#endif + + return uuid2; +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/hashRoot.h b/rir/src/serializeHash/hash/hashRoot.h new file mode 100644 index 000000000..bb1cf7870 --- /dev/null +++ b/rir/src/serializeHash/hash/hashRoot.h @@ -0,0 +1,14 @@ +// +// Created by Jakob Hain on 8/15/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "UUID.h" + +namespace rir { + +UUID hashRoot(SEXP root); + +} // namespace rir diff --git a/rir/src/serializeHash/hash/hashRootOld.cpp b/rir/src/serializeHash/hash/hashRootOld.cpp new file mode 100644 index 000000000..ac99fa015 --- /dev/null +++ b/rir/src/serializeHash/hash/hashRootOld.cpp @@ -0,0 +1,430 @@ +// +// Created by Jakob Hain on 7/21/23. +// + +#include "hashRootOld.h" +#include "R/Funtab.h" +#include "R/disableGc.h" +#include "compiler/parameter.h" +#include "runtime/Code.h" +#include "runtime/DispatchTable.h" +#include "runtime/Function.h" +#include "runtime/LazyArglist.h" +#include "runtime/LazyEnvironment.h" +#include "runtime/PoolStub.h" +#include "runtime/ProxyEnv.h" +#include "serializeHash/globals.h" +#include "serializeHash/hash/hashAst.h" +#include "serializeHash/hash/hashRoot_getConnected_common.h" +#include "utils/Pool.h" +#include "utils/measuring.h" +#include + +namespace rir { + +using HashRefTable = std::unordered_map; + +/// "TYPEOF" for special cases, different than any normal SEXP TYPEOF, to ensure +/// they are hashed differently. This is similar to what serialize.c does. +/// +/// This has the same size as TYPEOF (unsigned) +enum class SpecialType : SEXPTYPE { + Global = 0x10000000, + Ref = 0x10000001, + Altrep = 0x10000002, + AttrLangSexp = 0x10000003, + AttrListSexp = 0x10000004, + BcRef = 0x10000005, +}; + +static bool canSelfReference(SEXPTYPE type) { + switch (type) { + case SYMSXP: + case ENVSXP: + case EXTPTRSXP: + case WEAKREFSXP: + case BCODESXP: + case EXTERNALSXP: + return true; + case NILSXP: + case LISTSXP: + case CLOSXP: + case PROMSXP: + case LANGSXP: + case SPECIALSXP: + case BUILTINSXP: + case CHARSXP: + case LGLSXP: + case INTSXP: + case REALSXP: + case CPLXSXP: + case STRSXP: + case DOTSXP: + case ANYSXP: + case VECSXP: + case EXPRSXP: + case RAWSXP: + case S4SXP: + return false; + default: + assert(false && "canSelfReference: unhandled type"); + } +} + +/* + * From serialize.c + * Type/Flag Packing and Unpacking + * + * To reduce space consumption for serializing code (lots of list + * structure) the type (at most 8 bits), several single bit flags, + * and the sxpinfo gp field (LEVELS, 16 bits) are packed into a single + * integer. The integer is signed, so this shouldn't be pushed too + * far. It assumes at least 28 bits, but that should be no problem. + */ + +#define IS_OBJECT_BIT_MASK (1 << 8) +#define HAS_ATTR_BIT_MASK (1 << 9) +#define HAS_TAG_BIT_MASK (1 << 10) +#define ENCODE_LEVELS(v) ((v) << 12) + +static unsigned packFlags(SEXPTYPE type, int levs, int isobj, int hasattr, int hastag) +{ + unsigned val; + val = type | ENCODE_LEVELS(levs); + if (isobj) val |= IS_OBJECT_BIT_MASK; + if (hasattr) val |= HAS_ATTR_BIT_MASK; + if (hastag) val |= HAS_TAG_BIT_MASK; + return val; +} + +// Will hash sexp if it's an instance of CLS +template +static inline bool tryHash(SEXP sexp, HasherOld& hasher) { + if (CLS* b = CLS::check(sexp)) { + hasher.hashBytesOf(b->info.magic); + b->hash(hasher); + return true; + } else { + return false; + } +} + +static inline void hashRir(SEXP sexp, HasherOld& hasher) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashRir", sexp, [&]{ + if (!tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher) && + !tryHash(sexp, hasher)) { + std::cerr << "couldn't hash EXTERNALSXP: "; + Rf_PrintValue(sexp); + assert(false); + } + }); +} + +static void hashBcLang1(SEXP sexp, HasherOld& hasher, HashRefTable& bcRefs, + std::queue& bcLangWorklist) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashBcLang1", sexp, [&]{ + int type = TYPEOF(sexp); + if (type == LANGSXP || type == LISTSXP) { + if (bcRefs.count(sexp)) { + hasher.hashBytesOf(SpecialType::BcRef); + hasher.hashBytesOf(bcRefs.at(sexp)); + return; + } else { + bcRefs[sexp] = bcRefs.size(); + } + + auto attr = ATTRIB(sexp); + if (attr != R_NilValue) { + switch (type) { + case LANGSXP: + type = (SEXPTYPE)SpecialType::AttrLangSexp; + break; + case LISTSXP: + type = (SEXPTYPE)SpecialType::AttrListSexp; + break; + default: + assert(false); + } + hasher.hashBytesOf(type); + hasher.hash(attr); + } + hasher.hash(TAG(sexp)); + bcLangWorklist.push(CAR(sexp)); + bcLangWorklist.push(CDR(sexp)); + } else { + hasher.hash(sexp); + } + }); +} + +static void hashBcLang(SEXP sexp, HasherOld& hasher, HashRefTable& bcRefs) { + std::queue bcLangWorklist; + bcLangWorklist.push(sexp); + while (!bcLangWorklist.empty()) { + sexp = bcLangWorklist.front(); + bcLangWorklist.pop(); + + hashBcLang1(sexp, hasher, bcRefs, bcLangWorklist); + } +} + +static void hashBc1(SEXP sexp, HasherOld& hasher, HashRefTable& bcRefs, std::queue& bcWorklist) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashBc1", sexp, [&]{ + SEXP code = R_bcDecode(BCODE_CODE(sexp)); + hasher.hash(code); + auto consts = BCODE_CONSTS(sexp); + auto n = LENGTH(consts); + hasher.hashBytesOf(n); + for (auto i = 0; i < n; i++) { + auto c = VECTOR_ELT(consts, i); + auto type = TYPEOF(c); + switch (type) { + case BCODESXP: + hasher.hashBytesOf(type); + bcWorklist.push(c); + break; + case LANGSXP: + case LISTSXP: + hashBcLang(c, hasher, bcRefs); + break; + default: + hasher.hashBytesOf(type); + hasher.hash(c); + break; + } + } + }); +} + +static void hashBc(SEXP sexp, HasherOld& hasher, HashRefTable& bcRefs) { + std::queue bcWorklist; + bcWorklist.push(sexp); + while (!bcWorklist.empty()) { + sexp = bcWorklist.front(); + bcWorklist.pop(); + + hashBc1(sexp, hasher, bcRefs, bcWorklist); + } +} + +static void hashChild(SEXP sexp, HasherOld& hasher, HashRefTable& refs) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild", sexp, [&]{ + auto type = TYPEOF(sexp); + + if (ALTREP(sexp)) { + auto info = ALTREP_SERIALIZED_CLASS(sexp); + auto state = ALTREP_SERIALIZED_STATE(sexp); + auto attrib = ATTRIB(sexp); + if (info != nullptr && state != nullptr) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild altrep", sexp, [&]{ + auto flags = packFlags((SEXPTYPE)SpecialType::Altrep, + LEVELS(sexp), OBJECT(sexp), 0, 0); + PROTECT(state); + PROTECT(info); + hasher.hashBytesOf(flags); + hasher.hash(info); + hasher.hash(state); + hasher.hash(attrib); + UNPROTECT(2); /* state, info */ + return; + }); + } + /* else fall through to standard processing */ + } else if (global2Index.count(sexp)) { + hasher.hashBytesOf(SpecialType::Global); + hasher.hashBytesOf(global2Index.at(sexp)); + return; + } else if (canSelfReference(type)) { + if (refs.count(sexp)) { + hasher.hashBytesOf(SpecialType::Ref); + hasher.hashBytesOf(refs[sexp]); + return; + } else { + refs[sexp] = refs.size(); + } + } + hasher.hashBytesOf(type); + + bool hasTag_ = hasTag(sexp); + // With the CHARSXP cache chains maintained through the ATTRIB + // field the content of that field must not be serialized, so + // we treat it as not there. + auto hasAttr = (type != CHARSXP && ATTRIB(sexp) != R_NilValue); + auto flags = packFlags(type, LEVELS(sexp), OBJECT(sexp), hasAttr, hasTag_); + hasher.hashBytesOf(flags); + hasher.hashBytesOf(hasAttr); + if (hasAttr) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild attrib", sexp, [&]{ + hasher.hash(ATTRIB(sexp)); + }); + } + + switch (type) { + case NILSXP: + break; + case SYMSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild symbol", sexp, [&]{ + hasher.hash(PRINTNAME(sexp)); + }); + break; + case LISTSXP: + case LANGSXP: + case PROMSXP: + case DOTSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild tag", sexp, [&]{ + if (hasTag_) { + hasher.hash(TAG(sexp)); + } + }); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild list elem", sexp, [&]{ + if (BNDCELL_TAG(sexp)) { + assert(false && "TODO R_expand_binding_value isn't public"); + } + hasher.hash(CAR(sexp)); + }); + // ???: use goto tailcall like R for perf boost? + hasher.hash(CDR(sexp)); + break; + case CLOSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild closure sans body", sexp, [&]{ + hasher.hash(CLOENV(sexp)); + hasher.hash(FORMALS(sexp)); + }); + // ???: use goto tailcall like R for perf boost? + hasher.hash(BODY(sexp)); + break; + case EXTPTRSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild external pointer", sexp, [&]{ + hasher.hash(EXTPTR_PROT(sexp)); + hasher.hash(EXTPTR_TAG(sexp)); + }); + break; + case WEAKREFSXP: + // Currently we don't hash environment data because it's mutable + case ENVSXP: + break; + case SPECIALSXP: + case BUILTINSXP: + hasher.hashBytesOf(getBuiltinNr(sexp)); + break; + case CHARSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild char vector", sexp, [&]{ + auto n = LENGTH(sexp); + hasher.hashBytesOf(n); + hasher.hashBytes(CHAR(sexp), n * sizeof(char)); + }); + break; + case LGLSXP: + case INTSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild int vector", sexp, [&]{ + auto n = XLENGTH(sexp); + hasher.hashBytesOf(n); + hasher.hashBytes(INTEGER(sexp), n * sizeof(int)); + }); + break; + case REALSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild real vector", sexp, [&]{ + auto n = XLENGTH(sexp); + hasher.hashBytesOf(n); + hasher.hashBytes(REAL(sexp), n * sizeof(double)); + }); + break; + case CPLXSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild complex number vector", sexp, [&]{ + auto n = XLENGTH(sexp); + hasher.hashBytesOf(n); + hasher.hashBytes(COMPLEX(sexp), n * sizeof(Rcomplex)); + }); + break; + case RAWSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild byte vector", sexp, [&]{ + auto n = XLENGTH(sexp); + hasher.hashBytesOf(n); + hasher.hashBytes(RAW(sexp), n * sizeof(Rbyte)); + }); + break; + case STRSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild string vector", sexp, [&]{ + auto n = XLENGTH(sexp); + hasher.hashBytesOf(n); + for (int i = 0; i < n; ++i) { + hasher.hash(STRING_ELT(sexp, i)); + } + }); + break; + case VECSXP: + case EXPRSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot.cpp: hashChild expression or vector", sexp, [&]{ + auto n = XLENGTH(sexp); + hasher.hashBytesOf(n); + for (int i = 0; i < n; ++i) { + hasher.hash(VECTOR_ELT(sexp, i)); + } + }); + break; + case S4SXP: + // Only attributes (i.e., slots) count + break; + case BCODESXP: { + HashRefTable bcRefs; + hashBc(sexp, hasher, bcRefs); + break; + } + case EXTERNALSXP: + hashRir(sexp, hasher); + break; + default: + Rf_error("hashChild: unknown type %i", type); + } + }); +} + +void HasherOld::hashConstant(unsigned idx) { + hash(Pool::get(idx)); +} + +void HasherOld::hashSrc(unsigned idx) { + hash(src_pool_at(idx), true); +} + +UUID hashRootOld(SEXP root) { + UUID result; + disableInterpreter([&]{ + disableGc([&]{ + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRoot", root, [&]{ + UUID::Hasher uuidHasher; + HasherOld::Worklist worklist; + HashRefTable refs; + worklist.push({root, false}); + HasherOld hasher(uuidHasher, worklist); + + while (!worklist.empty()) { + auto& elem = worklist.front(); + auto sexp = elem.sexp; + auto isAst = elem.isAst; + worklist.pop(); + + if (isAst) { + auto uuid = hashAst(sexp); + hasher.hashBytesOf(uuid); + } else { + hashChild(sexp, hasher, refs); + } + } + result = uuidHasher.finalize(); + }); + }); + }); + return result; +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/hashRootOld.h b/rir/src/serializeHash/hash/hashRootOld.h new file mode 100644 index 000000000..cf0cb95be --- /dev/null +++ b/rir/src/serializeHash/hash/hashRootOld.h @@ -0,0 +1,74 @@ +// +// Created by Jakob Hain on 7/21/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "UUID.h" +#include +#include + +namespace rir { + +/// SEXP->UUID hasher which is exposed to RIR objects so that they can hash +/// themselves +class HasherOld { + struct Elem { + SEXP sexp; + bool isAst; + }; + using Worklist = std::queue; + + /// Underlying UUID hasher + UUID::Hasher& hasher; + /// Next SEXPs to process: instead of recursing, we add nested SEXPs to this + /// queue and then process them in a loop. This is different semantics than + /// actually recursing, but it doesn't matter because hashes are still the + /// same quality and consistent. + Worklist& worklist; + + HasherOld(UUID::Hasher& hasher, Worklist& worklist) + : hasher(hasher), worklist(worklist) {} + + friend UUID hashRootOld(SEXP root); + public: + /// Hash raw data, can't contain any references + template void hashBytesOf(T c) { + hasher.hashBytesOf(c); + } + /// Hash raw data, can't contain any references + void hashBytes(const void* data, size_t size) { + hasher.hashBytes(data, size); + } + /// Hash SEXP. ASTs hash differently and faster + void hash(SEXP s, bool isAst = false) { + worklist.push({s, isAst}); + } + /// Hash SEXP in constant pool ([Pool]) + void hashConstant(unsigned idx); + /// Hash SEXP in source pool ([src_pool_at]) + void hashSrc(unsigned idx); + /// Hash SEXP which could be nullptr + void hashNullable(SEXP s, bool isAst = false) { + hashBytesOf(s != nullptr); + if (s) { + hash(s, isAst); + } + } +}; + +/// Hash an SEXP (doesn't have to be RIR) into a UUID, by serializing it but +/// EVP-MD hashing ("fancy XOR"-ing) the bits instead of collecting them. +///

+/// This is called `hashRoot` to signify that we hash other SEXPs after this +/// one, which is relevant when we hash cyclic references: later occurrences of +/// the same SEXP are replaced by refs, but the location of these refs differ +/// depending on which SEXP is the root. You can think of the SEXP and all its +/// connected SEXPs as a graph, and hashRoot` creates a view of the graph with +/// this one at the center; if we call `hashRoot` with a different SEXP in the +/// connected graph, even though we have the same graph, we get a different view +/// and thus a different hash. +UUID hashRootOld(SEXP root); + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/hashRootUni.cpp b/rir/src/serializeHash/hash/hashRootUni.cpp new file mode 100644 index 000000000..239fb47b4 --- /dev/null +++ b/rir/src/serializeHash/hash/hashRootUni.cpp @@ -0,0 +1,81 @@ +// +// Created by Jakob Hain on 7/21/23. +// + +#include "hashRootUni.h" +#include "R/disableGc.h" +#include "compiler/parameter.h" +#include "hashAst.h" +#include "hashRoot_getConnected_common.h" +#include "runtime/LazyArglist.h" +#include "serializeHash/serialize/serialize.h" +#include "utils/measuring.h" + +namespace rir { + +SerialOptions& HasherUni::serialOptions() const { + // Doesn't matter what we return here, but we unfortunately need something + return SerialOptions::DeepCopy; +} + +bool HasherUni::willWrite(const rir::SerialFlags& flags) const { + return flags.contains(SerialFlag::Hashed); +} + +void HasherUni::writeBytes(const void* data, size_t size, + const SerialFlags& flags) { + if (!willWrite(flags)) { + return; + } + + hasher.hashBytes((uint8_t*)data, size); +} + +void HasherUni::writeInt(int data, const SerialFlags& flags) { + if (!willWrite(flags)) { + return; + } + + hasher.hashBytesOf(data); +} + +void HasherUni::write(SEXP s, const SerialFlags& flags) { + assert(flags.contains(SerialFlag::MaybeSexp) && + "Hashing non SEXP with SEXP flag"); + + if (!willWrite(flags)) { + return; + } + + if (flags.contains(SerialFlag::MaybeNotAst)) { + worklist.push(s); + } else { + hasher.hashBytesOf(hashAst(s)); + } +} + +void HasherUni::doHashRoot(SEXP root) { + writeInline(root); + while (!worklist.empty()) { + auto sexp = worklist.front(); + worklist.pop(); + writeInline(sexp); + } +} + +UUID hashRootUni(SEXP root) { + UUID result; + disableInterpreter([&]{ + disableGc([&]{ + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "hashRootUni", root, [&]{ + UUID::Hasher uuidHasher; + HasherUni hasher(uuidHasher); + hasher.doHashRoot(root); + result = uuidHasher.finalize(); + }); + }); + }); + return result; +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/hashRootUni.h b/rir/src/serializeHash/hash/hashRootUni.h new file mode 100644 index 000000000..7935c2315 --- /dev/null +++ b/rir/src/serializeHash/hash/hashRootUni.h @@ -0,0 +1,58 @@ +// +// Created by Jakob Hain on 7/21/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "UUID.h" +#include "serializeHash/serializeUni.h" +#include +#include + +namespace rir { + +/// SEXP->UUID hasher which is exposed to RIR objects so that they can hash +/// themselves +class HasherUni : AbstractSerializer { + using Worklist = std::queue; + + /// Underlying UUID hasher + UUID::Hasher& hasher; + // SEXPs already processed; we serialize these as refs instead of recursing. + SerializedRefs refs_; + /// Next SEXPs to process: instead of recursing, we add nested SEXPs to this + /// queue and then process them in a loop. This is different semantics than + /// actually recursing, but it doesn't matter because hashes are still the + /// same quality and consistent. We still hash ASTs immediately since those + /// are hashed with a different function. + Worklist worklist; + + explicit HasherUni(UUID::Hasher& hasher) + : hasher(hasher), refs_(), worklist() {} + SerializedRefs* refs() override { return &refs_; } + + void doHashRoot(SEXP root); + friend UUID hashRootUni(SEXP root); + public: + SerialOptions& serialOptions() const override; + bool willWrite(const SerialFlags& flags) const override; + void writeBytes(const void *data, size_t size, const SerialFlags& flags) override; + void writeInt(int data, const SerialFlags& flags) override; + void write(SEXP s, const SerialFlags& flags) override; +}; + +/// Hash an SEXP (doesn't have to be RIR) into a UUID, by serializing it but +/// EVP-MD hashing ("fancy XOR"-ing) the bits instead of collecting them. +///

+/// This is called `hashRoot` to signify that we hash other SEXPs after this +/// one, which is relevant when we hash cyclic references: later occurrences of +/// the same SEXP are replaced by refs, but the location of these refs differ +/// depending on which SEXP is the root. You can think of the SEXP and all its +/// connected SEXPs as a graph, and hashRoot` creates a view of the graph with +/// this one at the center; if we call `hashRoot` with a different SEXP in the +/// connected graph, even though we have the same graph, we get a different view +/// and thus a different hash. +UUID hashRootUni(SEXP root); + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/hash/hashRoot_getConnected_common.h b/rir/src/serializeHash/hash/hashRoot_getConnected_common.h new file mode 100644 index 000000000..77896d3a9 --- /dev/null +++ b/rir/src/serializeHash/hash/hashRoot_getConnected_common.h @@ -0,0 +1,22 @@ +// TODO: Merge more in hashRoot.cpp and getConnected.cpp if it's not noticeably +// slower or too complicated. +#pragma once + +#include +#include "R/r.h" + +__attribute__((unused)) static bool hasTag(SEXP sexp) { + switch (TYPEOF(sexp)) { + case LISTSXP: + case LANGSXP: + case PROMSXP: + case DOTSXP: + return TAG(sexp) != R_NilValue; + case CLOSXP: + return true; + // External pointers have tags but they are handled differently. + // Some other SEXPs have tags in bytecodes, also handled differently. + default: + return false; + } +} \ No newline at end of file diff --git a/rir/src/serializeHash/serialize/native/SerialModule.cpp b/rir/src/serializeHash/serialize/native/SerialModule.cpp new file mode 100644 index 000000000..53ff14905 --- /dev/null +++ b/rir/src/serializeHash/serialize/native/SerialModule.cpp @@ -0,0 +1,120 @@ +// +// Created by Jakob Hain on 6/6/23. +// + +#include "SerialModule.h" +#include "R/Serialize.h" +#include "SerialRepr.h" +#include "compiler/native/pir_jit_llvm.h" +#include +#include +#include +#include + +namespace rir { + +static llvm::ExitOnError ExitOnErr; + +SerialModule::SerialModule(size_t bitcodeSize, const SerialOptions& serialOpts) // NOLINT(*-pass-by-value) + : RirRuntimeObject(0, 0), serialOpts(serialOpts), + bitcodeSize(bitcodeSize) {} + +SerialModule::SerialModule(std::string&& bitcode, const SerialOptions& serialOpts) // NOLINT(*-pass-by-value) + : SerialModule(bitcode.size(), serialOpts) { + std::copy(bitcode.begin(), bitcode.end(), this->bitcode); +} + +SEXP SerialModule::create(std::string&& bitcode, const SerialOptions& serialOpts) { + auto store = Rf_allocVector(EXTERNALSXP, (R_xlen_t)size(bitcode.size())); + new (DATAPTR(store)) SerialModule(std::move(bitcode), serialOpts); + return store; +} + +size_t SerialModule::size(size_t bitcodeSize) { + return sizeof(SerialModule) + bitcodeSize; +} + +SEXP SerialModule::create(const llvm::Module& module, + const SerialOptions& serialOpts) { + std::string bitcode; + llvm::raw_string_ostream os(bitcode); + // In the future, if we want deterministic and hashable modules (e.g. want + // to share between compiler servers), we will set + // ShouldPreserveUseListOrder and GenerateHash to true + llvm::WriteBitcodeToFile(module, os); + os.flush(); + + return create(std::move(bitcode), serialOpts); +} + +std::unique_ptr +SerialModule::decode(Code* outer, + const SerialOptions& overrideSerialOpts) const { + assert(serialOpts.areCompatibleWith(overrideSerialOpts) && + "serial options module is decoded with must be compatible with " + "those it was encoded with"); + llvm::StringRef data(bitcode, bitcodeSize); + llvm::MemoryBufferRef buffer(data, "rir::SerialModule"); + auto mod = ExitOnErr(llvm::parseBitcodeFile(buffer, pir::PirJitLLVM::getContext())); + pir::SerialRepr::patch(*mod, outer, overrideSerialOpts); + return mod; +} + +std::unique_ptr SerialModule::decode(Code* outer) const { + return decode(outer, serialOpts); +} + +size_t SerialModule::size() const { + return size(bitcodeSize); +} + +uint64_t SerialModule::firstBitcodeBytes() const { + uint64_t result = 0; + for (size_t i = 0; i < std::min((size_t)8, bitcodeSize); ++i) { + result |= ((uint64_t)bitcode[i]) << (i * 8); + } + return result; +} + +void SerialModule::print(std::ostream& out) const { + auto mod = decode(nullptr); + llvm::raw_os_ostream ro(out); + mod->print(ro, nullptr, true, true); +} + +SerialModule* SerialModule::deserialize(AbstractDeserializer& deserializer) { + auto serialOpts = SerialOptions::deserializeCompatible(deserializer, SerialFlags::CodeNative); + + auto bitcodeSize = deserializer.readBytesOf(SerialFlags::CodeNative); + auto store = Rf_allocVector(EXTERNALSXP, size(bitcodeSize)); + auto module = new (DATAPTR(store)) SerialModule(bitcodeSize, serialOpts); + // Magic is already set. Also, SerialModule isn't actually recursive, we + // just use refs because we don't want copies. + deserializer.addRef(store); + + deserializer.readBytes((void*)module->bitcode, bitcodeSize, SerialFlags::CodeNative); + + return unpack(store); +} + +void SerialModule::serialize(AbstractSerializer& serializer) const { + serialOpts.serializeCompatible(serializer, SerialFlags::CodeNative); + + serializer.writeBytesOf(bitcodeSize, SerialFlags::CodeNative); + serializer.writeBytes((const void*)bitcode, bitcodeSize, + SerialFlags::CodeNative); +} + +void SerialModule::hash(HasherOld& hasher) const { + serialOpts.hashCompatible(hasher); + + hasher.hashBytesOf(bitcodeSize); + hasher.hashBytes(bitcode, bitcodeSize); +} + +void SerialModule::addConnected(__attribute__((unused)) + ConnectedCollectorOld& collector) const { + // No connected UUIDs +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/serialize/native/SerialModule.h b/rir/src/serializeHash/serialize/native/SerialModule.h new file mode 100644 index 000000000..5ffbe0d37 --- /dev/null +++ b/rir/src/serializeHash/serialize/native/SerialModule.h @@ -0,0 +1,67 @@ +// +// Created by Jakob Hain on 6/6/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "runtime/RirRuntimeObject.h" +#include "serializeHash/hash/getConnectedOld.h" +#include "serializeHash/serialize/serialize.h" +#include "serializeHash/serializeUni.h" +#include +#include +#include + +namespace llvm { + +class Module; + +} // namespace llvm + +namespace rir { + +struct Code; +struct SerialOptions; + +namespace pir { +class PirJitLLVM; +} + +/// "SMOD" ASCII -> hex +#define SERIAL_MODULE_MAGIC 0x534d4f44 + +/// Serialized module bitcode +class SerialModule + : public RirRuntimeObject { + SerialOptions serialOpts; + size_t bitcodeSize; + char bitcode[]; + + SerialModule(size_t bitcodeSize, const SerialOptions& serialOpts); // NOLINT(*-pass-by-value) + SerialModule(std::string&& bitcode, const SerialOptions& serialOpts); // NOLINT(*-pass-by-value) + static SEXP create(std::string&& bitcode, const SerialOptions& serialOpts); + + /// Size of the `SerialModule` structure from its `bitcodeSize` + static size_t size(size_t bitcodeSize); + + // These methods WOULD be public, except we don't want to accidentally call + // them without PirJitLLVM because the modules won't actually be added to + // LLJit and currently we always want to add them to LLJIT. + friend class pir::PirJitLLVM; + static SEXP create(const llvm::Module& module, const SerialOptions& serialOpts); + std::unique_ptr decode( + Code* outer, const SerialOptions& overrideSerialOpts) const; + std::unique_ptr decode(Code* outer) const; + public: + size_t size() const; + uint64_t firstBitcodeBytes() const; + + void print(std::ostream&) const; + static SerialModule* deserialize(AbstractDeserializer& deserializer); + void serialize(AbstractSerializer& serializer) const; + void hash(HasherOld& hasher) const; + void addConnected(ConnectedCollectorOld& collector) const; +}; + +} // namespace rir diff --git a/rir/src/serializeHash/serialize/native/SerialRepr.cpp b/rir/src/serializeHash/serialize/native/SerialRepr.cpp new file mode 100644 index 000000000..4d91978d3 --- /dev/null +++ b/rir/src/serializeHash/serialize/native/SerialRepr.cpp @@ -0,0 +1,498 @@ +// +// Created by Jakob Hain on 6/24/23. +// + +#include "SerialRepr.h" +#include "R/Funtab.h" +#include "compiler/native/lower_function_llvm.h" +#include "compiler/native/types_llvm.h" +#include "runtime/ProxyEnv.h" +#include "serializeHash/globals.h" +#include "serializeHash/serialize/serialize.h" +#include "utils/ByteBuffer.h" +#include +#include +#include + +namespace rir { +namespace pir { + + +llvm::MDNode* +SerialRepr::SEXP::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + // Some of these would serialize fine regardless, thanks to + // serialize.c:SaveSpecialHook + // Also, hashing handles all globals and builtins already, and serialization + // will once we migrate from R's serializer to RIR's + if (global2CppId.count(what)) { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "Global"), + llvm::MDString::get(ctx, global2CppId.at(what))}); + } else if (TYPEOF(what) == BUILTINSXP || TYPEOF(what) == SPECIALSXP) { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "Builtin"), + llvm::MDString::get(ctx, getBuiltinName(what))}); + } + ByteBuffer buf; + serialize(what, buf, serialOpts); + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "SEXP"), + llvm::MDString::get( + ctx, + llvm::StringRef((const char*)buf.data(), buf.size()))}); +} + +llvm::MDNode* +SerialRepr::String::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "String"), + llvm::MDString::get(ctx, str)}); +} + +llvm::MDNode* +SerialRepr::Function::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + ByteBuffer buf; + auto sexp = function->container(); + serialize(sexp, buf, serialOpts); + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "Function"), + llvm::MDString::get( + ctx, + llvm::StringRef((const char*)buf.data(), buf.size()))}); +} + +llvm::MDNode* +SerialRepr::TypeFeedback::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + ByteBuffer buf; + auto sexp = typeFeedback->container(); + serialize(sexp, buf, serialOpts); + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "TypeFeedback"), + llvm::MDString::get( + ctx, + llvm::StringRef((const char*)buf.data(), buf.size()))}); +} + +llvm::MDNode* +SerialRepr::DeoptMetadata::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + ByteBuffer buf; + m->serialize(buf, serialOpts); + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "DeoptMetadata"), + llvm::MDString::get( + ctx, + llvm::StringRef((const char*)buf.data(), buf.size()))}); +} + +llvm::MDNode* +SerialRepr::OpaqueTrue::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "OpaqueTrue")}); +} + +llvm::MDNode* +SerialRepr::R_Visible::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "R_Visible")}); +} + +llvm::MDNode* +SerialRepr::R_BCNodeStackTop::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "R_BCNodeStackTop")}); +} + +llvm::MDNode* +SerialRepr::R_GlobalContext::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "R_GlobalContext")}); +} + +llvm::MDNode* +SerialRepr::R_ReturnedValue::metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "R_ReturnedValue")}); +} + +llvm::MDNode* SerialRepr::functionMetadata(llvm::LLVMContext& ctx, + const char* llvmValueName, + int builtinId) { + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, llvmValueName), + llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( + llvm::Type::getInt32Ty(ctx), builtinId))}); +} + +llvm::MDNode* SerialRepr::srcIdxMetadata(llvm::LLVMContext& ctx, Immediate i, + const SerialOptions& serialOpts) { + // Source pool should never have global SEXPs, except R_NilValue which is + // trivial to serialize (specifically, we care about having no global envs) + auto what = src_pool_at(i); + ByteBuffer buf; + serialize(what, buf, serialOpts); + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get( + ctx, + llvm::StringRef((const char*)buf.data(), buf.size()))}); +} + +llvm::MDNode* SerialRepr::poolIdxMetadata(llvm::LLVMContext& ctx, BC::PoolIdx i, + const SerialOptions& serialOpts) { + // We assume the constant pool as used here has no global environments or + // other tricky exprs, if it does we need to abstract SEXP::metadata... + auto what = Pool::get(i); + ByteBuffer buf; + serialize(what, buf, serialOpts); + return llvm::MDTuple::get( + ctx, + {llvm::MDString::get( + ctx, + llvm::StringRef((const char*)buf.data(), buf.size()))}); +} + +llvm::MDNode* SerialRepr::namesMetadata(llvm::LLVMContext& ctx, + const std::vector& names) { + std::vector args; + args.reserve(names.size()); + for (auto i : names) { + auto sexp = Pool::get(i); + if (global2CppId.count(sexp)) { + args.push_back( + llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "Global"), + llvm::MDString::get(ctx, global2CppId.at(sexp))})); + } else { + ByteBuffer buf; + // Custom serialOpts isn't necessary because these are all ASTs + serialize(sexp, buf, SerialOptions::DeepCopy); + args.push_back( + llvm::MDTuple::get( + ctx, + {llvm::MDString::get(ctx, "SEXP"), + llvm::MDString::get(ctx, + llvm::StringRef((const char*)buf.data(), buf.size()))})); + } + } + return llvm::MDTuple::get(ctx, args); +} + +static void* +getMetadataPtr_Global(const llvm::MDNode& meta, + __attribute__((unused)) rir::Code* outer, + __attribute__((unused)) const SerialOptions& serialOpts) { + auto name = ((llvm::MDString*)meta.getOperand(1).get())->getString(); + return (void*)cppId2Global.at(name.str()); +} + +static void* +getMetadataPtr_Builtin(const llvm::MDNode& meta, + __attribute__((unused)) rir::Code* outer, + __attribute__((unused)) const SerialOptions& serialOpts) { + auto name = ((llvm::MDString*)meta.getOperand(1).get())->getString(); + return (void*)getBuiltinOrSpecialFun(name.str().c_str()); +} + +static void* getMetadataPtr_SEXP(const llvm::MDNode& meta, + rir::Code* outer, + const SerialOptions& serialOpts) { + auto data = ((llvm::MDString*)meta.getOperand(1).get())->getString(); + ByteBuffer buffer((uint8_t*)data.data(), (uint32_t)data.size()); + auto sexp = deserialize(buffer, serialOpts); + if (outer) { + // TODO: why is gcAttach not enough? + R_PreserveObject(sexp); + outer->addExtraPoolEntry(sexp); + } + return (void*)sexp; +} + +static void* +getMetadataPtr_String(const llvm::MDNode& meta, + __attribute__((unused)) rir::Code* outer, + __attribute__((unused)) const SerialOptions& serialOpts) { + auto data = ((llvm::MDString*)meta.getOperand(1).get())->getString(); + auto dataSexp = Rf_install(data.str().c_str()); + // Rf_install makes it permanent, so no need to gc-attach + return (void*)CHAR(PRINTNAME(dataSexp)); +} + +static void* getMetadataPtr_Function(const llvm::MDNode& meta, + rir::Code* outer, + const SerialOptions& serialOpts) { + auto data = ((llvm::MDString*)meta.getOperand(1).get())->getString(); + ByteBuffer buffer((uint8_t*)data.data(), (uint32_t)data.size()); + auto sexp = deserialize(buffer, serialOpts); + if (outer) { + // TODO: why is gcAttach not enough? + R_PreserveObject(sexp); + outer->addExtraPoolEntry(sexp); + } + assert(TYPEOF(sexp) == EXTERNALSXP && + "deserialized Function SEXP is not actually an EXTERNALSXP"); + assert(rir::Function::check(sexp) && + "deserialized Function SEXP is not actually a Function"); + return (void*)rir::Function::unpack(sexp); +} + +static void* getMetadataPtr_TypeFeedback(const llvm::MDNode& meta, + rir::Code* outer, + const SerialOptions& serialOpts) { + auto data = ((llvm::MDString*)meta.getOperand(1).get())->getString(); + ByteBuffer buffer((uint8_t*)data.data(), (uint32_t)data.size()); + auto sexp = deserialize(buffer, serialOpts); + if (outer) { + // TODO: why is gcAttach not enough? + R_PreserveObject(sexp); + outer->addExtraPoolEntry(sexp); + } + assert(TYPEOF(sexp) == EXTERNALSXP && + "deserialized TypeFeedback SEXP is not actually an EXTERNALSXP"); + assert(rir::TypeFeedback::check(sexp) && + "deserialized TypeFeedback SEXP is not actually a TypeFeedback"); + return (void*)rir::TypeFeedback::unpack(sexp); +} + +static void* getMetadataPtr_DeoptMetadata(const llvm::MDNode& meta, + rir::Code* outer, + const SerialOptions& serialOpts) { + auto data = ((llvm::MDString*)meta.getOperand(1).get())->getString(); + ByteBuffer buffer((uint8_t*)data.data(), (uint32_t)data.size()); + auto m = DeoptMetadata::deserialize(buffer, serialOpts); + assert(m->numFrames < 65536 && + "deserialized obviously corrupt DeoptMetadata"); + if (outer) { + // TODO: why is gcAttach not enough? + R_PreserveObject(m->container()); + for (int i = 0; i < (int)m->numFrames; i++) { + R_PreserveObject(m->frames[i].code->container()); + } + m->gcAttach(outer); + } + return (void*)m; +} + +static void* +getMetadataPtr_OpaqueTrue(__attribute__((unused)) const llvm::MDNode& meta, + __attribute__((unused)) rir::Code* outer, + __attribute__((unused)) const SerialOptions& serialOpts) { + return (void*)OpaqueTrue::instance(); +} + +static void* +getMetadataPtr_R_Visible(__attribute__((unused)) const llvm::MDNode& meta, + __attribute__((unused)) rir::Code* outer, + __attribute__((unused)) const SerialOptions& serialOpts) { + return (void*)&R_Visible; +} + +static void* +getMetadataPtr_R_BCNodeStackTop(__attribute__((unused)) const llvm::MDNode& meta, + __attribute__((unused)) rir::Code* outer, + __attribute__((unused)) const SerialOptions& serialOpts) { + return (void*)&R_BCNodeStackTop; +} + +static void* +getMetadataPtr_R_GlobalContext(__attribute__((unused)) const llvm::MDNode& meta, + __attribute__((unused)) rir::Code* outer, + __attribute__((unused)) const SerialOptions& serialOpts) { + return (void*)&R_GlobalContext; +} + +static void* +getMetadataPtr_R_ReturnedValue(__attribute__((unused)) const llvm::MDNode& meta, + __attribute__((unused)) rir::Code* outer, + __attribute__((unused)) const SerialOptions& serialOpts) { + return (void*)&R_ReturnedValue; +} + +typedef void* (*GetMetadataPtr)(const llvm::MDNode& meta, rir::Code* outer, + const SerialOptions& serialOpts); +static std::unordered_map getMetadataPtr{ + {"Global", getMetadataPtr_Global}, + {"Builtin", getMetadataPtr_Builtin}, + {"SEXP", getMetadataPtr_SEXP}, + {"String", getMetadataPtr_String}, + {"Function", getMetadataPtr_Function}, + {"TypeFeedback", getMetadataPtr_TypeFeedback}, + {"DeoptMetadata", getMetadataPtr_DeoptMetadata}, + {"OpaqueTrue", getMetadataPtr_OpaqueTrue}, + {"R_Visible", getMetadataPtr_R_Visible}, + {"R_BCNodeStackTop", getMetadataPtr_R_BCNodeStackTop}, + {"R_GlobalContext", getMetadataPtr_R_GlobalContext}, + {"R_ReturnedValue", getMetadataPtr_R_ReturnedValue} +}; + +static void patchPointerMetadata(llvm::GlobalVariable& inst, + llvm::MDNode* ptrMeta, rir::Code* outer, + const SerialOptions& serialOpts) { + auto type = ((llvm::MDString&)*ptrMeta->getOperand(0)).getString(); + auto ptr = getMetadataPtr[type.str()](*ptrMeta, outer, serialOpts); + + char name[21]; + sprintf(name, "ept_%lx", (uintptr_t)ptr); + inst.setName(name); +} + +static void patchSrcIdxMetadata(llvm::GlobalVariable& inst, + llvm::MDNode* srcIdxMeta, + const SerialOptions& serialOpts) { + auto data = ((llvm::MDString*)srcIdxMeta->getOperand(0).get())->getString(); + ByteBuffer buffer((uint8_t*)data.data(), (uint32_t)data.size()); + auto sexp = deserialize(buffer, serialOpts); + + // TODO: Reuse index if it's already in the source pool + // (and maybe merge and refactor pools) + char name[13]; + sprintf(name, "src_%08x", (uint32_t)src_pool_add(sexp)); + inst.setName(name); +} + +static void patchPoolIdxMetadata(llvm::GlobalVariable& inst, + llvm::MDNode* poolIdxMeta, + const SerialOptions& serialOpts) { + auto data = ((llvm::MDString*)poolIdxMeta->getOperand(0).get())->getString(); + ByteBuffer buffer((uint8_t*)data.data(), (uint32_t)data.size()); + auto sexp = deserialize(buffer, serialOpts); + + // TODO: Reuse index if it's already in the constant pool + // (and maybe merge and refactor pools) + char name[12]; + sprintf(name, "cp_%08x", (uint32_t)Pool::insert(sexp)); + inst.setName(name); +} + +static void patchNamesMetadata(llvm::GlobalVariable& inst, + llvm::MDNode* namesMeta) { + std::stringstream llvmName; + llvmName << "names"; + if (namesMeta->getNumOperands() == 0) { + // Special case so that the empty vector name still starts with "names_" + llvmName << "_"; + } else for (auto& nameOperand : namesMeta->operands()) { + auto nameMetadata = (llvm::MDTuple*)nameOperand.get(); + auto type = ((llvm::MDString*)(nameMetadata->getOperand(0)).get())->getString(); + auto data = ((llvm::MDString*)(nameMetadata->getOperand(1)).get())->getString(); + SEXP sexp; + if (type.equals("Global")) { + assert(cppId2Global.count(data.str()) && "Invalid global"); + sexp = cppId2Global.at(data.str()); + } else if (type.equals("SEXP")) { + ByteBuffer buffer((uint8_t*)data.data(), (uint32_t)data.size()); + // Custom serialOpts isn't necessary because these are all ASTs + sexp = deserialize(buffer, SerialOptions::DeepCopy); + } else { + assert(false && "Invalid name type (not \"Global\" or \"SEXP\")"); + } + // TODO: Reuse index if it's already in the constant pool + // (and maybe merge and refactor pools) + BC::PoolIdx nextName = Pool::insert(sexp); + llvmName << "_" << std::hex << nextName; + } + + inst.setName(llvmName.str()); +} + +static void patchGlobalMetadatas(llvm::Module& mod, rir::Code* outer, + const SerialOptions& serialOpts) { + // Need to store globals first, because otherwise we'll replace already- + // added values and cause an infinite loop. We also defer replacements + // although that probably isn't necessary + for (auto& global : mod.globals()) { + auto ptrMeta = global.getMetadata(SerialRepr::POINTER_METADATA_NAME); + auto srcIdxMeta = global.getMetadata(SerialRepr::SRC_IDX_METADATA_NAME); + auto poolIdxMeta = global.getMetadata(SerialRepr::POOL_IDX_METADATA_NAME); + auto namesMeta = global.getMetadata(SerialRepr::NAMES_METADATA_NAME); + + bool replaced = false; + if (ptrMeta) { + patchPointerMetadata(global, ptrMeta, outer, serialOpts); + replaced = true; + } + if (srcIdxMeta) { + assert(!replaced); + patchSrcIdxMetadata(global, srcIdxMeta, serialOpts); + replaced = true; + } + if (poolIdxMeta) { + assert(!replaced); + patchPoolIdxMetadata(global, poolIdxMeta, serialOpts); + replaced = true; + } + if (namesMeta) { + assert(!replaced); + patchNamesMetadata(global, namesMeta); + // replaced = true; + } + } +} + +static llvm::MDNode* patchFunctionMetadata(llvm::Module& mod, + const llvm::MDNode* operand) { + auto& meta = *(const llvm::MDTuple*)operand; + auto llvmValueName = ((llvm::MDString*)meta.getOperand(0).get())->getString(); + auto llvmValue = mod.getNamedValue(llvmValueName); + auto builtinId = (int)((llvm::ConstantInt*)((llvm::ConstantAsMetadata*)meta.getOperand(1).get())->getValue())->getZExtValue(); + auto builtin = getBuiltin(getBuiltinFun(builtinId)); + if (!llvmValue) { + return nullptr; + } + + char name[21]; + sprintf(name, "efn_%lx", (uintptr_t)builtin); + llvmValue->setName(name); + + return SerialRepr::functionMetadata(llvmValue->getContext(), name, builtinId); +} + +static void patchFunctionMetadatas(llvm::Module& mod) { + auto meta = mod.getNamedMetadata(pir::SerialRepr::FUNCTION_METADATA_NAME); + if (!meta) { + return; + } + std::vector newOperands; + for (auto operand : meta->operands()) { + auto newOperand = patchFunctionMetadata(mod, operand); + if (newOperand) { + newOperands.push_back(newOperand); + } + } + meta->clearOperands(); + for (auto newOperand : newOperands) { + meta->addOperand(newOperand); + } +} + +void SerialRepr::patch(llvm::Module& mod, rir::Code* outer, + const SerialOptions& serialOpts) { + patchGlobalMetadatas(mod, outer, serialOpts); + patchFunctionMetadatas(mod); +} + +} // namespace pir +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/serialize/native/SerialRepr.h b/rir/src/serializeHash/serialize/native/SerialRepr.h new file mode 100644 index 000000000..ba935e3f9 --- /dev/null +++ b/rir/src/serializeHash/serialize/native/SerialRepr.h @@ -0,0 +1,153 @@ +// +// Created by Jakob Hain on 6/24/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "bc/BC.h" +#include "runtime/Deoptimization.h" + +namespace llvm { +class Module; +class LLVMContext; +class MDNode; +} + +namespace rir { +namespace pir { + +class SerialRepr { + protected: + explicit SerialRepr() {} + + public: + static constexpr const char* POINTER_METADATA_NAME = "rir.serial.pointer"; + static constexpr const char* FUNCTION_METADATA_NAME = "rir.serial.function"; + static constexpr const char* SRC_IDX_METADATA_NAME = "rir.serial.srcIdx"; + static constexpr const char* POOL_IDX_METADATA_NAME = "rir.serial.poolIdx"; + static constexpr const char* NAMES_METADATA_NAME = "rir.serial.names"; + + class SEXP; + class String; + class Function; + class TypeFeedback; + class DeoptMetadata; + class OpaqueTrue; + class R_Visible; + class R_BCNodeStackTop; + class R_GlobalContext; + class R_ReturnedValue; + + virtual llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const = 0; + static llvm::MDNode* functionMetadata(llvm::LLVMContext& ctx, + const char* llvmValueName, + int builtinId); + static llvm::MDNode* srcIdxMetadata(llvm::LLVMContext& ctx, + Immediate srcIdx, + const SerialOptions& serialOpts); + static llvm::MDNode* poolIdxMetadata(llvm::LLVMContext& ctx, + BC::PoolIdx poolIdx, + const SerialOptions& serialOpts); + static llvm::MDNode* namesMetadata(llvm::LLVMContext& ctx, + const std::vector& names); + + /// Replace pointers with the serialized encodings, fetching from the + /// compiler server if necessary. See lower_function_llvm.cpp for where + /// exactly we store the metadata. + /// + /// `outer` is the code which the module resides in. It's needed because we + /// add stuff to its extra pool. It can be nullptr if we only create the + /// objects for a short period of time (when printing). + /// + /// `serialOpts` contains options which affect deserialization. These must + /// be compatible with the `serialOpts` passed to the metadata constructors. + static void patch(llvm::Module& mod, rir::Code* outer, + const SerialOptions& serialOpts); +}; + +class SerialRepr::SEXP : public SerialRepr { + ::SEXP what; + + public: + explicit SEXP(::SEXP what) : SerialRepr(), what(what) {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::String : public SerialRepr { + const char* str; + + public: + explicit String(const char* str) : SerialRepr(), str(str) {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::Function : public SerialRepr { + rir::Function* function; + + public: + explicit Function(rir::Function* function) : SerialRepr(), function(function) {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::TypeFeedback : public SerialRepr { + rir::TypeFeedback* typeFeedback; + + public: + explicit TypeFeedback(rir::TypeFeedback* typeFeedback) + : SerialRepr(), typeFeedback(typeFeedback) {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::DeoptMetadata : public SerialRepr { + rir::DeoptMetadata* m; + + public: + explicit DeoptMetadata(rir::DeoptMetadata* m) : SerialRepr(), m(m) {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::OpaqueTrue : public SerialRepr { + public: + OpaqueTrue() : SerialRepr() {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::R_Visible : public SerialRepr { + public: + R_Visible() : SerialRepr() {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::R_BCNodeStackTop : public SerialRepr { + public: + R_BCNodeStackTop() : SerialRepr() {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::R_GlobalContext : public SerialRepr { + public: + R_GlobalContext() : SerialRepr() {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; +class SerialRepr::R_ReturnedValue : public SerialRepr { + public: + R_ReturnedValue() : SerialRepr() {} + + llvm::MDNode* metadata(llvm::LLVMContext& ctx, + const SerialOptions& serialOpts) const override; +}; + +} // namespace pir +} // namespace rir diff --git a/rir/src/serializeHash/serialize/rPackFlags.cpp b/rir/src/serializeHash/serialize/rPackFlags.cpp new file mode 100644 index 000000000..49ed1ba87 --- /dev/null +++ b/rir/src/serializeHash/serialize/rPackFlags.cpp @@ -0,0 +1,51 @@ +// +// Created by Jakob Hain on 10/22/23. +// + +#include "rPackFlags.h" +#include "R/r.h" + +namespace rir { + +/* + * From serialize.c + * Type/Flag Packing and Unpacking + * + * To reduce space consumption for serializing code (lots of list + * structure) the type (at most 8 bits), several single bit flags, + * and the sxpinfo gp field (LEVELS, 16 bits) are packed into a single + * integer. The integer is signed, so this shouldn't be pushed too + * far. It assumes at least 28 bits, but that should be no problem. + */ + +#define IS_OBJECT_BIT_MASK (1 << 8) +#define HAS_ATTR_BIT_MASK (1 << 9) +#define HAS_TAG_BIT_MASK (1 << 10) +#define ENCODE_LEVELS(v) ((v) << 12) +#define DECODE_LEVELS(v) ((v) >> 12) +#define DECODE_TYPE(v) ((v) & 255) +#define CACHED_MASK (1<<5) +#define HASHASH_MASK 1 + +unsigned packFlags(SEXPTYPE type, int levs, bool isobj, bool hasattr, + bool hastag) { + unsigned val; + if (type == CHARSXP) levs &= (~(CACHED_MASK | HASHASH_MASK)); + val = type | ENCODE_LEVELS(levs); + if (isobj) val |= IS_OBJECT_BIT_MASK; + if (hasattr) val |= HAS_ATTR_BIT_MASK; + if (hastag) val |= HAS_TAG_BIT_MASK; + return val; +} + + +void unpackFlags(unsigned flags, SEXPTYPE& ptype, int& plevs, bool& pisobj, + bool& phasattr, bool& phastag) { + ptype = DECODE_TYPE(flags); + plevs = DECODE_LEVELS(flags); + pisobj = !!(flags & IS_OBJECT_BIT_MASK); + phasattr = !!(flags & HAS_ATTR_BIT_MASK); + phastag = !!(flags & HAS_TAG_BIT_MASK); +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/serialize/rPackFlags.h b/rir/src/serializeHash/serialize/rPackFlags.h new file mode 100644 index 000000000..0bc186f05 --- /dev/null +++ b/rir/src/serializeHash/serialize/rPackFlags.h @@ -0,0 +1,36 @@ +// +// Created by Jakob Hain on 10/22/23. +// + +#pragma once + +#include "R/r_incl.h" + +namespace rir { + +/// "TYPEOF" for special cases, different than any normal SEXP TYPEOF, to ensure +/// they are hashed differently. This is similar to what serialize.c does. +/// +/// This has the same size as TYPEOF (unsigned) +enum class SpecialType : SEXPTYPE { + // Starts at 128, assuming regular SEXPTYPEs only go up to 127, and we + // remove bytes after 255 + Global = 128, + Ref = 129, + Altrep = 130, + // Only used in writeBc and readBc (when reading and writing bytecode) + BcRef = 131 +}; + +enum class EnvType { + Package, + Namespace, + Regular +}; + +unsigned packFlags(SEXPTYPE type, int levs, bool isobj, bool hasattr, + bool hastag); +void unpackFlags(unsigned flags, SEXPTYPE& ptype, int& plevs, bool& pisobj, + bool& phasattr, bool& phastag); + +} // namespace rir diff --git a/rir/src/serializeHash/serialize/serialize.cpp b/rir/src/serializeHash/serialize/serialize.cpp new file mode 100644 index 000000000..30acf125c --- /dev/null +++ b/rir/src/serializeHash/serialize/serialize.cpp @@ -0,0 +1,425 @@ +#include "serialize.h" +#include "R/Printing.h" +#include "R/Protect.h" +#include "R/disableGc.h" +#include "compiler/parameter.h" +#include "compilerClientServer/CompilerServer.h" +#include "runtime/PoolStub.h" +#include "runtime/ProxyEnv.h" +#include "serializeHash/hash/UUIDPool.h" +#include "serializeHash/hash/hashAst.h" +#include "serializeHash/hash/hashRootOld.h" +#include "traceSerialize.h" +#include "utils/measuring.h" +#include + +/// This adds padding to each serialize call, but immediately raises an +/// assertion failure when a deserialize call deserializes a region which was +/// not serialized with the same type/size and flags in a serialize call. +/// +/// Regardless of whether this is enabled, we always serialize and check options +/// because that is cheap (only a few bytes at the start of serializing the +/// root, whereas this adds padding to each child and even non-SEXP fields) +#define DEBUG_SERIALIZE_CONSISTENCY 1 + +namespace rir { + +#if DEBUG_SERIALIZE_CONSISTENCY +static const uint64_t sexpBound = 0x123456789abcdef0; +static const uint64_t sexpEndBound = 0x123456789abcdef1; +static const uint64_t dataBound = 0xfedcba9876543210; +static const uint64_t intBound = 0xfedcba9876543211; +#endif + +SerialOptions SerialOptions::DeepCopy{false, false, false, nullptr, SerialOptions::SourcePools()}; + +SerialOptions SerialOptions::CompilerServer(bool intern) { + return SerialOptions{intern, intern, false, nullptr, SerialOptions::SourcePools()}; +} + +SerialOptions SerialOptions::CompilerClient(bool intern, Function* function, + SEXP decompiledClosure) { + return SerialOptions{intern, intern, false, CLOENV(decompiledClosure), SerialOptions::SourcePools(function, decompiledClosure)}; +} + +SerialOptions SerialOptions::CompilerClientRetrieve{false, true, false, nullptr, SerialOptions::SourcePools()}; +SerialOptions SerialOptions::SourceAndFeedback{false, true, true, nullptr, SerialOptions::SourcePools()}; + +unsigned pir::Parameter::RIR_SERIALIZE_CHAOS = + getenv("RIR_SERIALIZE_CHAOS") ? strtol(getenv("RIR_SERIALIZE_CHAOS"), nullptr, 10) : 0; +bool pir::Parameter::PIR_MEASURE_SERIALIZATION = + getenv("PIR_MEASURE_SERIALIZATION") != nullptr && + strtol(getenv("PIR_MEASURE_SERIALIZATION"), nullptr, 10); + +SerialOptions::SourcePools::SourcePools(Function* function, + SEXP decompiledClosure) + : sourceHash(hashDecompiled(decompiledClosure)), poolSeparatorIndices(), + map() { + auto body = function->body(); + for (unsigned i = 0; i < body->extraPoolSize; i++) { + map.push_back(body->getExtraPoolEntry(i)); + } + for (unsigned defaultArgIdx = 0; defaultArgIdx < function->nargs(); + defaultArgIdx++) { + poolSeparatorIndices.push_back(map.size()); + if (auto defaultArg = function->defaultArg(defaultArgIdx)) { + for (unsigned i = 0; i < defaultArg->extraPoolSize; i++) { + map.push_back(defaultArg->getExtraPoolEntry(i)); + } + } + } +} + +bool SerialOptions::SourcePools::isStub(SEXP stub) const { + auto rirStub = PoolStub::check(stub); + return rirStub && rirStub->sourceHash == sourceHash; +} + +bool SerialOptions::SourcePools::isEntry(SEXP entry) const { + return map.count(entry); +} + +SEXP SerialOptions::SourcePools::entry(SEXP stub) const { + assert(isStub(stub) && "not a stub for this extra pool"); + auto index = PoolStub::unpack(stub)->index; + auto defaultArgIdx = PoolStub::unpack(stub)->defaultArgIdx; + auto absoluteIndex = defaultArgIdx == UINT32_MAX ? index : (index + poolSeparatorIndices[defaultArgIdx]); + return map.at(absoluteIndex); +} + +SEXP SerialOptions::SourcePools::stub(SEXP entry) const { + assert(isEntry(entry) && "not an entry in this extra pool"); + auto absoluteIndex = (unsigned)map.at(entry); + auto poolSeparator = std::upper_bound(poolSeparatorIndices.begin(), + poolSeparatorIndices.end(), absoluteIndex); + auto index = poolSeparator == poolSeparatorIndices.begin() + ? absoluteIndex + : absoluteIndex - *(poolSeparator - 1); + // The `- 1` may wrap around, we want body to have index `UINT32_MAX` + auto defaultArgIdx = std::distance(poolSeparatorIndices.begin(), + poolSeparator) - 1; + return PoolStub::create(sourceHash, defaultArgIdx, index); +} + +SerialOptions +SerialOptions::deserializeCompatible(AbstractDeserializer& deserializer, + const SerialFlags& flags) { + SerialOptions options; + options.useHashes = deserializer.readBytesOf(flags); + options.onlySourceAndFeedback = deserializer.readBytesOf(flags); + return options; +} + +void SerialOptions::serializeCompatible(AbstractSerializer& serializer, + const SerialFlags& flags) const { + serializer.writeBytesOf(useHashes, flags); + serializer.writeBytesOf(onlySourceAndFeedback, flags); +} + +void SerialOptions::hashCompatible(HasherOld& hasher) const { + hasher.hashBytesOf(useHashes); + hasher.hashBytesOf(onlySourceAndFeedback); +} + +bool SerialOptions::areCompatibleWith(const rir::SerialOptions& other) const { + return useHashes == other.useHashes && + onlySourceAndFeedback == other.onlySourceAndFeedback; +} + +bool SerialOptions::willReadOrWrite(const SerialFlags& flags) const { + return + (!onlySourceAndFeedback || + flags.contains(SerialFlag::InSource) || + flags.contains(SerialFlag::InFeedback)); +} + +bool Serializer::willWrite(const rir::SerialFlags& flags) const { + return options.willReadOrWrite(flags); +} + +void Serializer::writeBytes(const void* data, size_t size, + const SerialFlags& flags) { + if (!willWrite(flags)) { + return; + } + +#if DEBUG_SERIALIZE_CONSISTENCY + buffer.putLong(dataBound); + buffer.putLong(size); + buffer.putInt(flags.id()); +#endif + + buffer.putBytes((uint8_t*)data, size); +} + +void Serializer::writeInt(int data, const SerialFlags& flags) { + if (!willWrite(flags)) { + return; + } + +#if DEBUG_SERIALIZE_CONSISTENCY + buffer.putLong(intBound); + buffer.putInt(flags.id()); +#endif + + buffer.putInt(*reinterpret_cast(&data)); +} + +SEXP Serializer::stub(SEXP sexp) const { + if (options.sourcePools.isEntry(sexp)) { + return options.sourcePools.stub(sexp); + } else if (sexp == options.closureEnvAndIfSetWeTryToSerializeLocalEnvsAsProxies) { + return ProxyEnv::create(options.closureEnvAndIfSetWeTryToSerializeLocalEnvsAsProxies); + } else if (options.closureEnvAndIfSetWeTryToSerializeLocalEnvsAsProxies && + TYPEOF(sexp) == ENVSXP && !isGlobalEnv(sexp)) { + std::cerr << "WARNING: local envs aren't correctly handled, and " + << "we're serializing a local env: " << Print::dumpSexp(sexp) + << std::endl; + return sexp; + } else { + return sexp; + } +} + +void Serializer::write(SEXP sexp, const SerialFlags& flags) { + assert(flags.contains(SerialFlag::MaybeSexp) && + "Serializing non SEXP with SEXP flag"); + + if (!willWrite(flags)) { + return; + } + + sexp = stub(sexp); + +#if DEBUG_SERIALIZE_CONSISTENCY + buffer.putLong(sexpBound); + buffer.putInt(flags.id()); + auto type = TYPEOF(sexp); + buffer.putInt(type); +#endif + + // If `useHashes` or `useHashesForRecordedCalls` depending on flags, either + // serialize via hash or (if this can't be serialized via hash) serialize + // children via hash. Otherwise serialize children regularly. If this is a + // recorded call, `useHashesForRecordedCalls` is ture, and `useHashes` is + // false, we have to construct a different serializer where `useHashes` is + // true, but if `useHashes` is true we can use this one. Either way we must + // call `writeInline` if we didn't write the hash directly to not infinitely + // recurse. + if (options.useHashes) { + if (!UUIDPool::tryWriteHash(sexp, buffer)) { + writeInline(sexp); + } + } else if (options.useHashesForRecordedCalls && + !flags.contains(SerialFlag::MaybeNotRecordedCall)) { + if (!UUIDPool::tryWriteHash(sexp, buffer)) { + // Still serialize children via hashes + auto innerOptions = options; + innerOptions.useHashes = true; + Serializer innerSerializer(buffer, innerOptions); + innerSerializer.writeInline(sexp); + } + } else { + writeInline(sexp); + } + +#if DEBUG_SERIALIZE_CONSISTENCY + buffer.putLong(sexpEndBound); + assert(type == TYPEOF(sexp) && "sanity check failed, SEXP changed type after serialization?"); +#endif +} + +bool Deserializer::willRead(const rir::SerialFlags& flags) const { + return options.willReadOrWrite(flags); +} + +#ifdef DEBUG_SERIALIZE_CONSISTENCY +static void checkFlagConsistency(const char* deserializedType, + unsigned deserializedId, const SerialFlags& flags) { + if (deserializedId != flags.id()) { + std::cerr << "serialize/deserialize " << deserializedType + << " flags mismatch: " << deserializedId << "("; + if (deserializedId + 1 < SerialFlags::ById.size()) { + std::cerr << SerialFlags::ById[deserializedId]; + } else { + std::cerr << "???"; + } + std::cerr << ")" << " vs " << flags.id() << " (" << flags << ")" << std::endl; + assert(false && "serialize/deserialize flags mismatch"); + } +} +#endif + +void Deserializer::readBytes(void* data, size_t size, const SerialFlags& flags) { + if (!willRead(flags)) { + memset(data, 0, size); + return; + } + +#if DEBUG_SERIALIZE_CONSISTENCY + assert(buffer.getLong() == dataBound && "serialize/deserialize data boundary mismatch"); + assert(buffer.getLong() == size && "serialize/deserialize data size mismatch"); + checkFlagConsistency("data", buffer.getInt(), flags); +#endif + + buffer.getBytes((uint8_t*)data, size); +} + +int Deserializer::readInt(const SerialFlags& flags) { + if (!willRead(flags)) { + return 0; + } + +#if DEBUG_SERIALIZE_CONSISTENCY + assert(buffer.getLong() == intBound && "serialize/deserialize int boundary mismatch"); + checkFlagConsistency("int", buffer.getInt(), flags); +#endif + + auto result = buffer.getInt(); + return *reinterpret_cast(&result); +} + +SEXP Deserializer::destub(SEXP sexp) const { + if (options.closureEnvAndIfSetWeTryToSerializeLocalEnvsAsProxies && + ProxyEnv::check(sexp)) { + return ProxyEnv::unpack(sexp)->materialize( + options.closureEnvAndIfSetWeTryToSerializeLocalEnvsAsProxies); + } else if (options.sourcePools.isStub(sexp)) { + return options.sourcePools.entry(sexp); + } else { + return sexp; + } +} + +SEXP Deserializer::read(const SerialFlags& flags) { + assert(flags.contains(SerialFlag::MaybeSexp) && + "Deserializing non SEXP with SEXP flag"); + + if (!willRead(flags)) { + return nullptr; + } + + SEXP result; + +#if DEBUG_SERIALIZE_CONSISTENCY + assert(buffer.getLong() == sexpBound && + "serialize/deserialize sexp boundary mismatch"); + checkFlagConsistency("sexp", buffer.getInt(), flags); + auto expectedType = buffer.getInt(); +#endif + + // If `useHashes` or `useHashesForRecordedCalls` depending on flags, either + // deserialize via hash or (if this wasn't serialized via hash) deserialize + // children via hash. Otherwise deserialize children regularly. If this is a + // recorded call, `useHashesForRecordedCalls` is true, and `useHashes` is + // false, we have to construct a different deserializer where `useHashes` is + // true, but if `useHashes` is true we can use this one. Either way we must + // call `readInline` if we didn't read the hash directly to not infinitely + // recurse. + if (options.useHashes) { + result = UUIDPool::tryReadHash(buffer); + if (!result) { + result = readInline(); + } + } else if (options.useHashesForRecordedCalls && + !flags.contains(SerialFlag::MaybeNotRecordedCall)) { + result = UUIDPool::tryReadHash(buffer); + if (!result) { + // Still deserialize children via hashes + auto innerOptions = options; + innerOptions.useHashes = true; + Deserializer innerDeserializer(buffer, innerOptions); + result = innerDeserializer.readInline(); + } + } else { + result = readInline(); + } + +#if DEBUG_SERIALIZE_CONSISTENCY + assert(buffer.getLong() == sexpEndBound && + "serialize/deserialize sexp end boundary mismatch"); + assert(expectedType == TYPEOF(result) && + "serialize/deserialize sexp type mismatch"); +#endif + + result = destub(result); + + return result; +} + +void Deserializer::addRef(SEXP sexp) { + AbstractDeserializer::addRef(sexp); + if (retrieveHash && UUIDPool::internable(sexp)) { + // TODO: Hacky that we hardcode preserve to whether the compiler server + // is running + UUIDPool::intern(sexp, retrieveHash, CompilerServer::isRunning(), false); + retrieveHash = UUID(); + } +} + +void serialize(SEXP sexp, ByteBuffer& buffer, const SerialOptions& options) { + disableInterpreter([&]{ + disableGc([&] { + if (pir::Parameter::PIR_TRACE_SERIALIZATION) { + auto oldWritePos = buffer.getWritePos(); + auto sexpPrint = Print::dumpSexp(sexp, 120); + std::cerr << "+ serialize " << sexpPrint << std::endl; + TraceSerializer traceSerializer(buffer, options); + traceSerializer.writeInline(traceSerializer.stub(sexp)); + std::cerr << "+ serialized " + << buffer.getWritePos() - oldWritePos << " bytes, " + << sexpPrint << std::endl; + } else { + Serializer serializer(buffer, options); + serializer.writeInline(serializer.stub(sexp)); + } + }); + }); +} + +SEXP deserialize(const ByteBuffer& buffer, const SerialOptions& options) { + return deserialize(buffer, options, UUID()); +} + +SEXP deserialize(const ByteBuffer& buffer, const SerialOptions& options, + const UUID& retrieveHash) { + SEXP result; + disableInterpreter([&]{ + disableGc([&] { + if (pir::Parameter::PIR_TRACE_SERIALIZATION) { + auto oldReadPos = buffer.getReadPos(); + std::cerr << "- deserialize" << std::endl; + TraceDeserializer traceDeserializer(buffer, options, retrieveHash); + result = traceDeserializer.destub(traceDeserializer.readInline()); + std::cerr << "- deserialized " + << buffer.getReadPos() - oldReadPos << " bytes, " + << Print::dumpSexp(result, 120) << std::endl; + + assert(!traceDeserializer.retrieveHash && "retrieve hash not filled"); + } else { + Deserializer deserializer(buffer, options, retrieveHash); + result = deserializer.destub(deserializer.readInline()); + + assert(!deserializer.retrieveHash && "retrieve hash not filled"); + } + + assert((!retrieveHash || UUIDPool::getHash(result) == retrieveHash) && + "deserialized SEXP not given retrieve hash"); + }); + }); + return result; +} + +SEXP copyBySerial(SEXP x) { + if (!pir::Parameter::RIR_SERIALIZE_CHAOS) + return x; + + return Measuring::timeEventIf2(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serialize.cpp: copyBySerial", x, [&]{ + Protect p(x); + ByteBuffer buffer; + serialize(x, buffer, SerialOptions::DeepCopy); + return p(deserialize(buffer, SerialOptions::DeepCopy)); + }); +} + +} // namespace rir diff --git a/rir/src/serializeHash/serialize/serialize.h b/rir/src/serializeHash/serialize/serialize.h new file mode 100644 index 000000000..dea36ce1a --- /dev/null +++ b/rir/src/serializeHash/serialize/serialize.h @@ -0,0 +1,192 @@ +// +// Created by Jakob Hain on 6/27/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "serializeHash/hash/UUID.h" +#include "serializeHash/serializeUni.h" +#include "utils/BimapVector.h" +#include "utils/ByteBuffer.h" + +namespace rir { + +struct Function; +class HasherOld; + +/// Controls what data is serialized / deserialized and what format some of it +/// uses. The same options data is serialized with, it must also be deserialized +/// with. +struct SerialOptions { + class SourcePools { + UUID sourceHash; + std::vector poolSeparatorIndices; + BimapVector map; + + public: + SourcePools() : sourceHash(), poolSeparatorIndices(), map() {} + SourcePools(Function* function, SEXP decompiledClosure); + explicit operator bool() const { return (bool)sourceHash; } + + bool isEntry(SEXP entry) const; + bool isStub(SEXP stub) const; + SEXP entry(SEXP stub) const; + SEXP stub(SEXP entry) const; + }; + + /// Whether to serialize connected RIR objects as UUIDs instead of their + /// full content, besides recorded calls, which are serialized as UUIDs + /// depending on `useHashesForRecordedCalls`. + bool useHashes; + /// Whether to serialize recorded calls as UUIDs instead of their full + /// content. + bool useHashesForRecordedCalls; + /// Whether to only serialize source and feedback (no optimized code). + bool onlySourceAndFeedback; + /// If set, will serialize this as a closure environment stub, and warn when + /// other local environments are serialized + SEXP closureEnvAndIfSetWeTryToSerializeLocalEnvsAsProxies; + /// If nonempty, we serialize the corresponding SEXPs with stubs from these + /// pools + SourcePools sourcePools; + + /// Don't serialize the extra pool, since we are only serializing to check + /// compatibility and that isn't used + static SerialOptions deserializeCompatible( + AbstractDeserializer& deserializer, + const SerialFlags& flags = SerialFlags::Inherit); + /// Don't serialize the extra pool, since we are only serializing to check + /// compatibility and that isn't used + void serializeCompatible( + AbstractSerializer& serializer, + const SerialFlags& flags = SerialFlags::Inherit) const; + void hashCompatible(HasherOld& hasher) const; + /// Check equality of everything except the extra pool + bool areCompatibleWith(const SerialOptions& other) const; + + bool willReadOrWrite(const SerialFlags& flags) const; + + /// Serialize everything, no hashes, environment locks + static SerialOptions DeepCopy; + /// Serialize everything, no hashes, no environment locks + static SerialOptions CompilerServer(bool intern); + /// Serialize everything, no hashes, no environment locks. + /// Serialize and deserialize pool entries from stubs + static SerialOptions CompilerClient(bool intern, Function* function, + SEXP decompiledClosure); + // TODO: Remove both of the below + /// Serialize everything, hashes for recorded calls, no environment locks + static SerialOptions CompilerClientRetrieve; + /// Serialize only source and feedback, no hashes, no environment locks + static SerialOptions SourceAndFeedback; +}; + +// TODO: Serializer/Deserializer and serialize/deserialize are tightly coupled. +// Serializer/Deserializer expect GC to be disabled which only +// serialize/deserialize do, they can also only be created in +// serialize/deserialize. Lastly, serialize/deserialize will use +// TracingSerializer/TracingDeserializer (subclasses) if tracing is enabled. + +class Serializer : public AbstractSerializer { + /// Underlying byte buffer + ByteBuffer& buffer; + /// Ref table for recursively-serialized SEXPs + SerializedRefs refs_; + /// Controls what data is serialized and what format some of it uses. The + /// corresponding deserializer must have the same options. + SerialOptions options; + + SerializedRefs* refs() override { return &refs_; } + + /// If the SEXP is a stubbed pool entry or closure environment, return its + /// stub or proxy. Otherwise return it unchanged. + SEXP stub(SEXP sexp) const; + + protected: + Serializer(ByteBuffer& buffer, const SerialOptions& options) + : buffer(buffer), refs_(), options(options) { + options.serializeCompatible(*this); + } + friend void serialize(SEXP sexp, ByteBuffer& buffer, + const SerialOptions& options); + + unsigned getWritePos() const { return buffer.getWritePos(); } + public: + const SerialOptions& serialOptions() const override { return options; } + bool willWrite(const SerialFlags& flags) const override; + void writeBytes(const void *data, size_t size, const SerialFlags& flags) override; + void writeInt(int data, const SerialFlags& flags) override; + void write(SEXP sexp, const SerialFlags& flags) override; +}; + +class Deserializer : public AbstractDeserializer { + /// Underlying byte buffer + const ByteBuffer& buffer; + /// Ref table for recursively-(de)serialized SEXPs + DeserializedRefs refs_; + /// Controls what data is deserialized and what format some of it uses. The + /// corresponding serializer must have the same options. + SerialOptions options; + /// If set, the first rir object deserialized will use this hash + UUID retrieveHash; + + DeserializedRefs* refs() override { return &refs_; } + + /// If the SEXP is a stubbed pool entry or proxy environment, return its + /// materialized counterpart. Otherwise return it unchanged. + SEXP destub(SEXP sexp) const; + + protected: + Deserializer(const ByteBuffer& buffer, const SerialOptions& options, + const UUID& retrieveHash = UUID()) + : buffer(buffer), refs_(), options(options), + retrieveHash(retrieveHash) { + auto serializedOptions = SerialOptions::deserializeCompatible(*this); + assert(serializedOptions.areCompatibleWith(options) && + "serialize/deserialize options incompatible (not equal)"); + } + friend SEXP deserialize(const ByteBuffer& sexpBuffer, + const SerialOptions& options, + const UUID& retrieveHash); + + unsigned getReadPos() const { return buffer.getReadPos(); } + public: + const SerialOptions& serialOptions() const override { return options; } + bool willRead(const SerialFlags& flags) const override; + void readBytes(void *data, size_t size, const SerialFlags& flags) override; + int readInt(const SerialFlags& flags) override; + SEXP read(const SerialFlags& flags) override; + void addRef(SEXP sexp) override; +}; + +/// Serialize a SEXP (doesn't have to be RIR) into the buffer, using RIR's +/// custom serialization format. +/// +/// The corresponding call to deserialize MUST have the same options. +/// Additionally, if options.useHashes is true, connected RIR objects are +/// serialized as UUIDs instead of their full content, and these SEXP MUST be +/// interned and preserved because they must be retrievable when deserialized. +void serialize(SEXP sexp, ByteBuffer& buffer, const SerialOptions& options); +/// Deserialize an SEXP (doesn't have to be RIR) from the buffer, using RIR's +/// custom serialization format. +/// +/// The corresponding call to serialize MUST have had the same options. +/// Additionally, if options.useHashes is true, connected RIR objects MUST be +/// retrievable. +SEXP deserialize(const ByteBuffer& sexpBuffer, const SerialOptions& options); +/// Equivalent to +/// `deserialize(const ByteBuffer& sexpBuffer, const SerialOptions& options)`, except +/// if the hash is non-null, the first deserialized internable SEXP will be +/// interned with it before being fully deserialized. This function is +/// used/needed to support deserializing recursive hashed structures. +/// +/// \see deserialize(const ByteBuffer& sexpBuffer, const SerialOptions& options) +SEXP deserialize(const ByteBuffer& sexpBuffer, const SerialOptions& options, + const UUID& retrieveHash); + +/// Will serialize and deserialize the SEXP, returning a deep copy, using RIR's +/// custom serialization format. +SEXP copyBySerial(SEXP x); + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/serialize/serializeR.cpp b/rir/src/serializeHash/serialize/serializeR.cpp new file mode 100644 index 000000000..5b860a496 --- /dev/null +++ b/rir/src/serializeHash/serialize/serializeR.cpp @@ -0,0 +1,395 @@ +#include "serializeR.h" +#include "R/Protect.h" +#include "R/disableGc.h" +#include "api.h" +#include "compiler/parameter.h" +#include "interpreter/interp_incl.h" +#include "runtime/DispatchTable.h" +#include "runtime/LazyArglist.h" +#include "runtime/LazyEnvironment.h" +#include "runtime/PoolStub.h" +#include "runtime/ProxyEnv.h" +#include "serialize.h" +#include "serializeHash/hash/UUIDPool.h" +#include "utils/measuring.h" +#include + +namespace rir { + +bool pir::Parameter::RIR_PRESERVE = + getenv("RIR_PRESERVE") != nullptr && strtol(getenv("RIR_PRESERVE"), nullptr, 10); +bool pir::Parameter::SERIALIZE_LLVM = + RIR_PRESERVE || + (getenv("PIR_DEBUG_SERIALIZE_LLVM") != nullptr && strtol(getenv("PIR_DEBUG_SERIALIZE_LLVM"), nullptr, 10)); + +// This is a magic constant in custom-r/src/main/saveload.c:defaultSaveVersion +static const int R_STREAM_DEFAULT_VERSION = 3; +static const R_pstream_format_t R_STREAM_FORMAT = R_pstream_xdr_format; + +/// Controls what data is serialized and what format some of it uses. The SEXP +/// must be deserialized with the same options it was serialized with. +/// +/// Unfortunately, this is a global variable, because that is the easiest way to +/// thread these options through the GNU-R serialization API, and because we +/// already have a separate RIR serializer which stores these in the serializer +/// (the GNU-R serialization API is serializing children with only `out` and +/// `refTable`, so we can't just pass our serializer to children). +static SerialOptions* R_SERIAL_OPTIONS = nullptr; +/// Similar to R_SERIAL_OPTIONS, we store the retrieve hash for +/// deserialized RIR objects as a global. As a consequence, we can't deserialize +/// with before we consume the retrieve hash from a previous serialization. +static UUID R_SERIAL_RETRIEVE_HASH; + +struct RSerializer : AbstractSerializer { + /// Underlying R output stream + R_outpstream_t out; + /// Underlying R ref table + SEXP refTable; + + RSerializer(R_outpstream_t out, SEXP refTable) + : out(out), refTable(refTable) {} + + SerializedRefs* refs() override { return nullptr; } + + const SerialOptions& serialOptions() const override { + return *R_SERIAL_OPTIONS; + } + bool willWrite(const SerialFlags& flags) const override { + assert(R_SERIAL_OPTIONS && "not setup for serialization"); + return R_SERIAL_OPTIONS->willReadOrWrite(flags); + } + void writeBytes(const void *data, size_t size, + const SerialFlags& flags) override { + if (!willWrite(flags)) { + return; + } + + OutBytes(out, data, (int)size); + } + void writeInt(int data, const SerialFlags& flags) override { + if (!willWrite(flags)) { + return; + } + + OutInteger(out, data); + } + void write(SEXP s, const SerialFlags& flags) override { + if (!willWrite(flags)) { + return; + } + + if (R_SERIAL_OPTIONS->useHashes) { + if (!UUIDPool::tryWriteHash(s, out)) { + WriteItem(s, refTable, out); + } + } else if (R_SERIAL_OPTIONS->useHashesForRecordedCalls && + !flags.contains(SerialFlag::MaybeNotRecordedCall)) { + if (!UUIDPool::tryWriteHash(s, out)) { + // Still serialize children via hashes + R_SERIAL_OPTIONS->useHashes = true; + WriteItem(s, refTable, out); + } + } else { + WriteItem(s, refTable, out); + } + } +}; + +struct RDeserializer : AbstractDeserializer { + /// Underlying R input stream + R_inpstream_t inp = nullptr; + /// Underlying R read-ref table + SEXP refTable = nullptr; + + RDeserializer(R_inpstream_t inp, SEXP refTable) + : inp(inp), refTable(refTable) {} + + DeserializedRefs* refs() override { return nullptr; } + + const SerialOptions& serialOptions() const override { + return *R_SERIAL_OPTIONS; + } + + bool willRead(const SerialFlags& flags) const override { + assert(R_SERIAL_OPTIONS && "not setup for deserialization"); + return R_SERIAL_OPTIONS->willReadOrWrite(flags); + } + + void readBytes(void *data, size_t size, const SerialFlags& flags) override { + if (!willRead(flags)) { + return; + } + + InBytes(inp, data, (int)size); + } + + int readInt(const SerialFlags& flags) override { + if (!willRead(flags)) { + return 0; + } + + return InInteger(inp); + } + + SEXP read(const SerialFlags& flags) override { + if (!willRead(flags)) { + return nullptr; + } + + SEXP result; + if (R_SERIAL_OPTIONS->useHashes) { + result = UUIDPool::tryReadHash(inp); + if (!result) { + result = ReadItem(refTable, inp); + } + } else if (R_SERIAL_OPTIONS->useHashesForRecordedCalls && + !flags.contains(SerialFlag::MaybeNotRecordedCall)) { + result = UUIDPool::tryReadHash(inp); + if (!result) { + // Still deserialize children via hashes + R_SERIAL_OPTIONS->useHashes = true; + result = ReadItem(refTable, inp); + R_SERIAL_OPTIONS->useHashes = false; + } + } else { + result = ReadItem(refTable, inp); + } + + return result; + } + + void addRef(SEXP sexp) override { + AddReadRef(refTable, sexp); + if (R_SERIAL_RETRIEVE_HASH && TYPEOF(sexp) == EXTERNALSXP) { + UUIDPool::intern(sexp, R_SERIAL_RETRIEVE_HASH, false, false); + R_SERIAL_RETRIEVE_HASH = UUID(); + } + } +}; + +// Will serialize s if it's an instance of CLS +template +static bool trySerializeR(SEXP s, SEXP refTable, R_outpstream_t out) { + if (CLS* b = CLS::check(s)) { + if (canSelfReference(s)) { + HashAdd(s, refTable); + } + + OutInteger(out, b->info.magic); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeR.cpp: rirSerializeHook", s, [&]{ + RSerializer serializer(out, refTable); + b->serialize(serializer); + }); + return true; + } else { + return false; + } +} + +void rirSerializeHook(SEXP s, SEXP refTable, R_outpstream_t out) { + if (pir::Parameter::RIR_PRESERVE) { + OutInteger(out, EXTERNALSXP); + if (!trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out) && + !trySerializeR(s, refTable, out)) { + std::cerr << "couldn't serialize EXTERNALSXP: "; + Rf_PrintValue(s); + assert(false); + } + } else { + WriteItem(rirDecompile(s), refTable, out); + } +} + +SEXP rirDeserializeHook(SEXP refTable, R_inpstream_t inp) { + return Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeR.cpp: rirDeserializeHook", [&]{ + RDeserializer deserializer(inp, refTable); + unsigned magic = InInteger(inp); + switch (magic) { + case DISPATCH_TABLE_MAGIC: + return DispatchTable::deserialize(deserializer)->container(); + case CODE_MAGIC: + return Code::deserialize(deserializer)->container(); + case FUNCTION_MAGIC: + return Function::deserialize(deserializer)->container(); + case ARGLIST_ORDER_MAGIC: + return ArglistOrder::deserialize(deserializer)->container(); + case LAZY_ARGS_MAGIC: + return LazyArglist::deserialize(deserializer)->container(); + case LAZY_ENVIRONMENT_MAGIC: + return LazyEnvironment::deserialize(deserializer)->container(); + case PIR_TYPE_FEEDBACK_MAGIC: + return PirTypeFeedback::deserialize(deserializer)->container(); + case TYPEFEEDBACK_MAGIC: + return TypeFeedback::deserialize(deserializer)->container(); + case SERIAL_MODULE_MAGIC: + return SerialModule::deserialize(deserializer)->container(); + case POOL_STUB_MAGIC: + return PoolStub::deserialize(deserializer)->container(); + case PROXY_ENV_MAGIC: + return ProxyEnv::deserialize(deserializer)->container(); + default: + std::cerr << "unhandled RIR object magic: 0x" << std::hex << magic + << "\n"; + assert(false); + } + }, [&](SEXP s){ + // TODO: Find out why this doesn't work for some nested code objects, + // and fix if possible. + return false; + }); +} + +static void rStreamOutChar(R_outpstream_t stream, int data) { + auto buffer = (ByteBuffer*)stream->data; + auto data2 = (unsigned char)data; + buffer->putBytes(&data2, sizeof(unsigned char)); +} + +static void rStreamOutBytes(R_outpstream_t stream, void* data, int length) { + auto buffer = (ByteBuffer*)stream->data; + buffer->putBytes((uint8_t*)data, length); +} + +static int rStreamInChar(R_inpstream_t stream) { + auto buffer = (ByteBuffer*)stream->data; + unsigned char c; + buffer->getBytes(&c, sizeof(unsigned char)); + return c; +} + +static void rStreamInBytes(R_inpstream_t stream, void* data, int length) { + auto buffer = (ByteBuffer*)stream->data; + buffer->getBytes((uint8_t*)data, length); +} + +static SerialOptions* newRSerialOptions(bool useHashes) { + return new SerialOptions{useHashes, useHashes, false, nullptr, SerialOptions::SourcePools()}; +} + +void serializeR(SEXP sexp, ByteBuffer& buffer, bool useHashes) { + disableInterpreter([&]{ + disableGc([&] { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeR.cpp: serializeR", sexp, [&]{ + auto oldPreserve = pir::Parameter::RIR_PRESERVE; + auto oldSerialOptions = R_SERIAL_OPTIONS; + pir::Parameter::RIR_PRESERVE = true; + R_SERIAL_OPTIONS = newRSerialOptions(useHashes); + + struct R_outpstream_st out{}; + R_InitOutPStream(&out, (R_pstream_data_t)&buffer, R_STREAM_FORMAT, + R_STREAM_DEFAULT_VERSION, rStreamOutChar, + rStreamOutBytes, nullptr, nullptr); + R_Serialize(sexp, &out); + + delete R_SERIAL_OPTIONS; + R_SERIAL_OPTIONS = oldSerialOptions; + pir::Parameter::RIR_PRESERVE = oldPreserve; + }); + }); + }); +} + +SEXP deserializeR(const ByteBuffer& sexpBuffer, bool useHashes, const UUID& newRetrieveHash) { + assert(!R_SERIAL_RETRIEVE_HASH && + "bad state: deserializing a different SEXP before we set the retrieve hash from last deserialization"); + SEXP result; + disableInterpreter([&]{ + disableGc([&] { + result = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeR.cpp: deserializeR", [&]{ + auto oldPreserve = pir::Parameter::RIR_PRESERVE; + auto oldSerialOptions = R_SERIAL_OPTIONS; + pir::Parameter::RIR_PRESERVE = true; + R_SERIAL_OPTIONS = newRSerialOptions(useHashes); + R_SERIAL_RETRIEVE_HASH = newRetrieveHash; + + struct R_inpstream_st in{}; + R_InitInPStream(&in, (R_pstream_data_t)&sexpBuffer, R_STREAM_FORMAT, + rStreamInChar, rStreamInBytes, nullptr, nullptr); + SEXP sexp = R_Unserialize(&in); + + assert(!R_SERIAL_RETRIEVE_HASH && "retrieve hash not filled"); + assert((!newRetrieveHash || UUIDPool::getHash(sexp) == newRetrieveHash) && + "deserialized SEXP not given retrieve hash"); + + delete R_SERIAL_OPTIONS; + R_SERIAL_OPTIONS = oldSerialOptions; + pir::Parameter::RIR_PRESERVE = oldPreserve; + + return sexp; + }, [&](SEXP s){ + // TODO: Find out why this doesn't work for some nested code objects, + // and fix if possible. + return false; + }); + }); + }); + return result; +} + +SEXP deserializeR(const ByteBuffer& sexpBuffer, bool useHashes) { + return deserializeR(sexpBuffer, useHashes, UUID()); +} + +SEXP copyBySerialR(SEXP x) { + if (!pir::Parameter::RIR_SERIALIZE_CHAOS) + return x; + + return Measuring::timeEventIf2(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeR.cpp: copyBySerialR", x, [&]{ + Protect p(x); + + auto oldOptions = R_SERIAL_OPTIONS; + auto oldPreserve = pir::Parameter::RIR_PRESERVE; + pir::Parameter::RIR_PRESERVE = true; + R_SERIAL_OPTIONS = newRSerialOptions(false); + + SEXP copy; + disableInterpreter([&]{ + SEXP data = p(R_serialize(x, R_NilValue, R_NilValue, R_NilValue, R_NilValue)); + disableGc([&] { copy = p(R_unserialize(data, R_NilValue)); }); + }); + +#if defined(ENABLE_SLOWASSERT) && defined(CHECK_COPY_BY_SERIAL) + auto xHash = hashRoot(x); + auto copyHash = hashRoot(copy); + if (xHash != copyHash) { + std::stringstream ss; + ss << "hash mismatch after serializing: " << xHash + << " != " << copyHash; + Rf_warning(ss.str().c_str()); + Rf_PrintValue(x); + Rf_PrintValue(copy); + + SEXP copy2; + disableInterpreter([&]{ + SEXP data = p(R_serialize(copy, R_NilValue, R_NilValue, R_NilValue, R_NilValue)); + disableGc([&]{ copy = p(R_unserialize(data2, R_NilValue)); }); + }); + + auto copyHash2 = hashRoot(copy2); + if (copyHash != copyHash2) { + std::stringstream ss2; + ss2 << "copy hash is also different: " << copyHash2; + Rf_warning(ss2.str().c_str()); + Rf_PrintValue(copy2); + } + } +#endif + + delete R_SERIAL_OPTIONS; + R_SERIAL_OPTIONS = oldOptions; + pir::Parameter::RIR_PRESERVE = oldPreserve; + + return copy; + }); +} + +} // namespace rir diff --git a/rir/src/serializeHash/serialize/serializeR.h b/rir/src/serializeHash/serialize/serializeR.h new file mode 100644 index 000000000..3d9266a60 --- /dev/null +++ b/rir/src/serializeHash/serialize/serializeR.h @@ -0,0 +1,48 @@ +// +// Created by Jakob Hain on 6/27/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "serializeHash/hash/UUID.h" +#include "utils/ByteBuffer.h" + +namespace rir { + +/// Function passed to GNU-R, use `serialize` instead +void rirSerializeHook(SEXP s, SEXP refTable, R_outpstream_t out); +/// Function passed to GNU-R, use `deserialize` instead +SEXP rirDeserializeHook(SEXP refTable, R_inpstream_t inp); +/// Will serialize and deserialize the SEXP, returning a deep copy, using R's +/// serialization format. +SEXP copyBySerialR(SEXP x); + +/// Serialize a SEXP (doesn't have to be RIR) into the buffer, using R's +/// serialization format. +/// +/// If useHashes is true, connected RIR objects are serialized as UUIDs +/// instead of their full content. The corresponding call to deserialize MUST be +/// done with `useHashes=true` as well, AND the SEXP must have already been +/// recursively interned and preserved. +void serializeR(SEXP sexp, ByteBuffer& buffer, bool useHashes); +/// Deserialize an SEXP (doesn't have to be RIR) from the buffer, using R's +/// serialization format. +/// +/// If useHashes is true, connected RIR objects are deserialized from UUIDs +/// and retrieved from the UUIDPool. If the UUIDs aren't in the pool, this +/// sends a request to compiler peer, and fails if it isn't connected or we +/// can't get a response. The corresponding call to serialize MUST have been +/// done with `useHashes=true` as well. +SEXP deserializeR(const ByteBuffer& sexpBuffer, bool useHashes); +/// Equivalent to +/// `deserializeR(const ByteBuffer& sexpBuffer, bool useHashes)`, except the +/// first deserialized internable SEXP will also be interned with that hash +/// before being fully deserialized. This function is used/needed to support +/// deserializing recursive hashed structures. +/// +/// \see deserialize(const ByteBuffer& sexpBuffer, bool useHashes) +SEXP deserializeR(const ByteBuffer& sexpBuffer, bool useHashes, + const UUID& retrieveHash); + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/serialize/traceSerialize.cpp b/rir/src/serializeHash/serialize/traceSerialize.cpp new file mode 100644 index 000000000..bc159dbfd --- /dev/null +++ b/rir/src/serializeHash/serialize/traceSerialize.cpp @@ -0,0 +1,289 @@ +// +// Created by Jakob Hain on 10/22/23. +// + +#include "traceSerialize.h" +#include "R/Printing.h" +#include "compiler/parameter.h" +#include "rPackFlags.h" +#include "runtime/rirObjectMagic.h" +#include +#include +#include +#include + +namespace rir { + +bool pir::Parameter::PIR_TRACE_SERIALIZATION = + getenv("PIR_TRACE_SERIALIZATION") != nullptr && + strtol(getenv("PIR_TRACE_SERIALIZATION"), nullptr, 10); +unsigned pir::Parameter::PIR_TRACE_SERIALIZATION_MAX_RAW_PRINT_LENGTH = + getenv("PIR_TRACE_SERIALIZATION_MAX_RAW_PRINT_LENGTH") != nullptr ? + strtol(getenv("PIR_TRACE_SERIALIZATION_MAX_RAW_PRINT_LENGTH"), nullptr, 10) : + 64; +size_t pir::Parameter::PIR_TRACE_SERIALIZATION_MIN_SIZE = + getenv("PIR_TRACE_SERIALIZATION_MIN_SIZE") != nullptr ? + strtol(getenv("PIR_TRACE_SERIALIZATION_MIN_SIZE"), nullptr, 10) : + 0; +std::vector* pir::Parameter::PIR_TRACE_SERIALIZATION_EXCLUDE = nullptr; + +static std::vector* getPirTraceSerializationExcludeFlags() { + auto flags = new std::vector(); + if (getenv("PIR_TRACE_SERIALIZATION_EXCLUDE") != nullptr) { + std::string flagsStr = getenv("PIR_TRACE_SERIALIZATION_EXCLUDE"); + std::stringstream ss(flagsStr); + std::string flag; + while (std::getline(ss, flag, ',')) { + flags->push_back(SerialFlags::parse(flag).id()); + } + } + return flags; +} + +void initPirTraceSerializationExcludeFlags() { + pir::Parameter::PIR_TRACE_SERIALIZATION_EXCLUDE = getPirTraceSerializationExcludeFlags(); +} + +bool Tracer::shouldTrace(const SerialFlags& flags) { + return std::none_of(pir::Parameter::PIR_TRACE_SERIALIZATION_EXCLUDE->begin(), + pir::Parameter::PIR_TRACE_SERIALIZATION_EXCLUDE->end(), + [&flags](unsigned excludeFlagId) { + return flags.id() == excludeFlagId; + }); +} + +bool Tracer::shouldTrace(const SerialFlags& flags, size_t size) { + return shouldTrace(flags) && + size >= pir::Parameter::PIR_TRACE_SERIALIZATION_MIN_SIZE; +} + +void Tracer::tracePrefix(char prefixChar, const SerialFlags& flags) { + assert(shouldTrace(flags)); + + for (size_t i = 0; i < depth; i++) { + out << " "; + } + auto ioflags = out.flags(); + out << prefixChar << prefixChar << " (" << std::setfill(' ') + << std::setw(16) << std::left << flags << ") "; + out.flags(ioflags); +} + +bool Tracer::traceSpecial(const SerialFlags& flags, const void* data, + size_t size) { + assert(shouldTrace(flags, size)); + + if (flags.id() == SerialFlags::String.id() || + flags.id() == SerialFlags::SymbolName.id()) { + out << "str "; + + out << std::string((const char*)data, size); + } else if (flags.id() == SerialFlags::RFlags.id()) { + out << "type "; + + unsigned rFlags = *(const unsigned*)data; + SEXPTYPE type; + int levs; + bool isObj; + bool hasAttr; + bool hasTag; + unpackFlags(rFlags, type, levs, isObj, hasAttr, hasTag); + + switch (type) { + case (SEXPTYPE)SpecialType::Altrep: + out << "altrep"; + break; + case (SEXPTYPE)SpecialType::Global: + out << "global"; + break; + case (SEXPTYPE)SpecialType::Ref: + out << "ref"; + break; + default: + out << Rf_type2char(type); + break; + } + if (levs) { + out << " +levs=" << levs; + } + if (isObj) { + out << " +obj"; + } + if (hasAttr) { + out << " +attr"; + } + if (hasTag) { + out << " +tag"; + } + } else if (flags.id() == SerialFlags::RirMagic.id()) { + out << "rir "; + + out << rirObjectClassName(*(const unsigned*)data); + } else if (flags.id() == SerialFlags::BuiltinNr.id() || + flags.id() == SerialFlags::EnvType.id() || + flags.id() == SerialFlags::RefId.id() || + flags.id() == SerialFlags::GlobalId.id()) { + out << "int "; + + out << *(const unsigned*)data; + } else { + return false; + } + + // A bit confusing: we handle all other cases in the else branch, + // this saves LOC because we don't return true in any of the handled cases, + // we just fall through to this + return true; +} + +void Tracer::traceInt(char prefixChar, int data, const SerialFlags& flags) { + if (!shouldTrace(flags, sizeof(data))) { + return; + } + + tracePrefix(prefixChar, flags); + if (!traceSpecial(flags, &data, sizeof(data))) { + out << "int 0x"; + auto ioflags = out.flags(); + out << std::setfill('0') << std::setw(8) << std::right << std::hex; + out << data; + out.flags(ioflags); + out << " (" << data << ")"; + } + out << std::endl; +} + +void Tracer::traceBytes(char prefixChar, const void* data, size_t size, + const SerialFlags& flags) { + if (!shouldTrace(flags, size)) { + return; + } + + tracePrefix(prefixChar, flags); + if (!traceSpecial(flags, data, size)) { + out << "bytes "; + out << "0x"; + auto ioflags = out.flags(); + out << std::setfill('0') << std::setw(2) << std::right << std::hex; + for (size_t i = 0; i < size; ++i) { + out << (unsigned)((const uint8_t*)data)[i]; + if (i == maxRawPrintLength) { + out.flags(ioflags); + out << "... (" << size << ")"; + break; + } + } + out.flags(ioflags); + } + out << std::endl; +} + +void Tracer::traceSexp(char prefixChar, SEXP s, unsigned size, + const SerialFlags& flags) { + if (!shouldTrace(flags, size == UINT32_MAX ? 0 : size)) { + return; + } + + tracePrefix(prefixChar, flags); + out << "SEXP " << Print::dumpSexp(s, maxRawPrintLength); + auto len = Rf_xlength(s); + if (len != 0 && len != 1) { + out << " (" << Rf_type2char(TYPEOF(s)) << " len " << len << ")"; + } + if (size != UINT32_MAX) { + out << " (" << size << " bytes)"; + } + out << std::endl; +} + +void Tracer::traceSexp(char prefixChar, SEXP s, const SerialFlags& flags) { + traceSexp(prefixChar, s, UINT32_MAX, flags); +} + +void Tracer::traceSexpDone(char prefixChar, SEXP s, unsigned size, + const SerialFlags& flags) { + if (!shouldTrace(flags, size)) { + return; + } + + tracePrefix(prefixChar, flags); + out << "done " << Print::dumpSexp(s, maxRawPrintLength); + if (size != UINT32_MAX) { + out << " (" << size << " bytes)"; + } + out << std::endl; +} + +TraceSerializer::TraceSerializer(ByteBuffer& buffer, + const rir::SerialOptions& options, + std::ostream& out) + : TraceSerializer(buffer, options, out, + pir::Parameter::PIR_TRACE_SERIALIZATION_MAX_RAW_PRINT_LENGTH) {} + +void TraceSerializer::writeBytes(const void *data, size_t size, const SerialFlags& flags) { + if (willWrite(flags)) { + traceBytes('+', data, size, flags); + } + Serializer::writeBytes(data, size, flags); +} + +void TraceSerializer::writeInt(int data, const SerialFlags& flags) { + if (willWrite(flags)) { + traceInt('+', data, flags); + } + Serializer::writeInt(data, flags); +} + +void TraceSerializer::write(SEXP s, const SerialFlags& flags) { + if (willWrite(flags)) { + traceSexp('+', s, flags); + } + + depth++; + auto startPos = getWritePos(); + Serializer::write(s, flags); + auto size = getWritePos() - startPos; + depth--; + + if (startPos != UINT32_MAX && willWrite(flags)) { + traceSexpDone('+', s, size, flags); + } +} + +TraceDeserializer::TraceDeserializer(const ByteBuffer& buffer, + const rir::SerialOptions& options, + const rir::UUID& retrieveHash, + std::ostream& out) + : TraceDeserializer(buffer, options, retrieveHash, out, + pir::Parameter::PIR_TRACE_SERIALIZATION_MAX_RAW_PRINT_LENGTH) {} + + +void TraceDeserializer::readBytes(void *data, size_t size, const SerialFlags& flags) { + Deserializer::readBytes(data, size, flags); + if (willRead(flags)) { + traceBytes('-', data, size, flags); + } +} + +int TraceDeserializer::readInt(const SerialFlags& flags) { + int data = Deserializer::readInt(flags); + if (willRead(flags)) { + traceInt('-', data, flags); + } + return data; +} + +SEXP TraceDeserializer::read(const SerialFlags& flags) { + depth++; + auto startPos = getReadPos(); + SEXP s = Deserializer::read(flags); + auto size = getReadPos() - startPos; + depth--; + + if (willRead(flags)) { + traceSexp('-', s, startPos == UINT32_MAX ? UINT32_MAX : size, flags); + } + return s; +} + +} // namespace rir \ No newline at end of file diff --git a/rir/src/serializeHash/serialize/traceSerialize.h b/rir/src/serializeHash/serialize/traceSerialize.h new file mode 100644 index 000000000..40e1cb43f --- /dev/null +++ b/rir/src/serializeHash/serialize/traceSerialize.h @@ -0,0 +1,75 @@ +// +// Created by Jakob Hain on 10/22/23. +// + +#pragma once + +#include "serialize.h" +#include + +namespace rir { + +struct SerialOptions; +class UUID; + +class Tracer { + std::ostream& out; + unsigned maxRawPrintLength; + + static bool shouldTrace(const SerialFlags& flags); + static bool shouldTrace(const SerialFlags& flags, size_t size); + void tracePrefix(char prefixChar, const SerialFlags& flags); + bool traceSpecial(const SerialFlags& flags, const void* data, size_t size); + + protected: + size_t depth; + + Tracer(std::ostream& out, unsigned maxRawPrintLength) + : out(out), maxRawPrintLength(maxRawPrintLength), depth(0) {} + + void traceBytes(char prefixChar, const void* data, size_t size, + const SerialFlags& flags); + void traceInt(char prefixChar, int data, const SerialFlags& flags); + void traceSexp(char prefixChar, SEXP s, const SerialFlags& flags); + void traceSexp(char prefixChar, SEXP s, unsigned size, + const SerialFlags& flags); + void traceSexpDone(char prefixChar, SEXP s, unsigned size, + const SerialFlags& flags); +}; + +class TraceSerializer : public Serializer, private Tracer { + TraceSerializer(ByteBuffer& buffer, const SerialOptions& options, + std::ostream& out = std::cerr); + TraceSerializer(ByteBuffer& buffer, const SerialOptions& options, + std::ostream& out, unsigned maxRawPrintLength) + : Serializer(buffer, options), + Tracer(out, maxRawPrintLength) {} + friend void serialize(SEXP sexp, ByteBuffer& buffer, + const SerialOptions& options); + public: + void writeBytes(const void *data, size_t size, const SerialFlags& flags) override; + void writeInt(int data, const SerialFlags& flags) override; + void write(SEXP s, const SerialFlags& flags) override; +}; + +class TraceDeserializer : public Deserializer, private Tracer { + TraceDeserializer(const ByteBuffer& buffer, const SerialOptions& options, + const UUID& retrieveHash = UUID(), + std::ostream& out = std::cerr); + TraceDeserializer(const ByteBuffer& buffer, const SerialOptions& options, + const UUID& retrieveHash, std::ostream& out, + unsigned maxRawPrintLength) + : Deserializer(buffer, options, retrieveHash), + Tracer(out, maxRawPrintLength) {} + friend SEXP deserialize(const ByteBuffer& sexpBuffer, + const SerialOptions& options, + const UUID& retrieveHash); + public: + void readBytes(void *data, size_t size, const SerialFlags& flags) override; + int readInt(const SerialFlags& flags) override; + SEXP read(const SerialFlags& flags) override; +}; + +void initPirTraceSerializationExcludeFlags(); + +} // namespace rir diff --git a/rir/src/serializeHash/serializeUni.cpp b/rir/src/serializeHash/serializeUni.cpp new file mode 100644 index 000000000..b90856466 --- /dev/null +++ b/rir/src/serializeHash/serializeUni.cpp @@ -0,0 +1,1144 @@ + +// +// Created by Jakob Hain on 8/9/23. +// + +#include "serializeUni.h" +#include "R/Funtab.h" +#include "compiler/parameter.h" +#include "runtime/DispatchTable.h" +#include "runtime/LazyArglist.h" +#include "runtime/LazyEnvironment.h" +#include "runtime/PoolStub.h" +#include "runtime/ProxyEnv.h" +#include "serializeHash/globals.h" +#include "serializeHash/hash/hashRoot_getConnected_common.h" +#include "serializeHash/serialize/rPackFlags.h" +#include "utils/Pool.h" +#include "utils/measuring.h" +#include +#include + +namespace rir { + +unsigned SerialFlags::nextId = 0; + +// Inlay hints are needed to understand the below code +#define V(name) \ +const SerialFlags SerialFlags::name(true, true, true, true, true, true, true); +LIST_OF_INHERIT_SERIAL_FLAGS(V) +#undef V +const SerialFlags SerialFlags::Ast( + true, + false, + true, + true, + true, + false, + true); +const SerialFlags SerialFlags::DtContext( + false, + true, + true, + false, + true, + false, + true); +const SerialFlags SerialFlags::DtBaseline( + true, + true, + true, + true, + true, + true, + true); +const SerialFlags SerialFlags::DtOptimized( + false, + true, + true, + true, + false, + false, + true); +const SerialFlags SerialFlags::FunBody( + true, + true, + true, + true, + true, + true, + true); +const SerialFlags SerialFlags::FunDefaultArg( + true, + true, + true, + true, + true, + true, + true); +const SerialFlags SerialFlags::FunFeedback( + false, + true, + true, + true, + false, + true, + true); +const SerialFlags SerialFlags::FunStats( + false, + true, + true, + false, + false, + false, + true); +const SerialFlags SerialFlags::FunMiscBytes( + true, + true, + true, + false, + true, + false, + true); +const SerialFlags SerialFlags::CodeArglistOrder( + true, + true, + true, + true, + true, + false, + true); +const SerialFlags SerialFlags::CodeOuterFun( + true, + true, + true, + true, + true, + false, + true); +const SerialFlags SerialFlags::CodePromise( + true, + true, + true, + true, + true, + true, + true); +// The values should be the same as FunFeedback's, however the is different +const SerialFlags SerialFlags::CodeFeedback( + false, + true, + true, + true, + false, + true, + true); +const SerialFlags SerialFlags::CodePoolUnknown( + true, + true, + true, + true, + true, + false, + true); +const SerialFlags SerialFlags::CodeNative( + false, + true, + true, + true, + true, + false, + true); +const SerialFlags SerialFlags::CodeAst( + true, + false, + true, + true, + true, + false, + true); +const SerialFlags SerialFlags::CodeMisc( + true, + true, + true, + true, + true, + false, + true); +const SerialFlags SerialFlags::EnvLock( + false, + true, + true, + true, + true, + true, + false); +const SerialFlags SerialFlags::EnvMisc( + false, + true, + true, + true, + true, + true, + true); +const SerialFlags SerialFlags::_Unused( + false, + false, + false, + false, + false, + false, + false); + +static std::vector ById_{ +#define V(name) SerialFlags::name, + LIST_OF_SERIAL_FLAGS(V) +#undef V + SerialFlags::_Unused}; + +const std::vector& SerialFlags::ById = ById_; + +const SerialFlags& SerialFlags::parse(const std::string& name) { +#define V(name_) \ + if (name == #name_) \ + return SerialFlags::name_; + LIST_OF_SERIAL_FLAGS(V) +#undef V + std::cerr << "unknown serial flag: " << name << "\n"; + assert(false && "unknown serial flag, can't parse"); +} + +std::ostream& operator<<(std::ostream& out, const SerialFlags& f) { +#define V(name) \ + if (SerialFlags::name.id_ == f.id_) { \ + out << #name; \ + return out; \ + } + LIST_OF_SERIAL_FLAGS(V) +#undef V + assert(false && "Serial flag is not one of the defined globals, corrupt?"); +} + +void AbstractSerializer::writeConst(unsigned idx, const SerialFlags& flags) { + write(Pool::get(idx), flags); +} + +void AbstractSerializer::writeSrc(unsigned idx, const SerialFlags& flags) { + write(src_pool_at(idx), flags); +} + +unsigned AbstractDeserializer::readConst(const SerialFlags& flags) { + return Pool::insert(read(flags)); +} + +unsigned AbstractDeserializer::readSrc(const SerialFlags& flags) { + return src_pool_add(read(flags)); +} + +/// These SEXPs are added to the ref table the first time they are serialized or +/// deserialized, and serialized as / deserialized from refs subsequent times. +bool canSelfReference(SEXP sexp) { + switch (TYPEOF(sexp)) { + case SYMSXP: + case ENVSXP: + case EXTPTRSXP: + case WEAKREFSXP: + case BCODESXP: + return true; + case EXTERNALSXP: + // SerialModule can't self-reference, but we want to return true for it + // because we want to avoid serializing copies because it's large + return !TypeFeedback::check(sexp) && + !ArglistOrder::check(sexp) && + !PoolStub::check(sexp) && + !ProxyEnv::check(sexp); + case NILSXP: + case LISTSXP: + case CLOSXP: + case PROMSXP: + case LANGSXP: + case SPECIALSXP: + case BUILTINSXP: + case CHARSXP: + case LGLSXP: + case INTSXP: + case REALSXP: + case CPLXSXP: + case STRSXP: + case DOTSXP: + case ANYSXP: + case VECSXP: + case EXPRSXP: + case RAWSXP: + case S4SXP: + return false; + default: + assert(false && "canSelfReference: unhandled type"); + } +} + +static char lastname[8192] = ""; +/// Similar to R_FindNamespace1 (tbh the code in serialize.c is very hacky and +/// I'm not 100% sure I'm following it correctly) +static SEXP findNamespace(SEXP info) { + PROTECT(info); + auto where = Rf_ScalarString(Rf_mkChar(lastname)); + PROTECT(where); + auto s_getNamespace = Rf_install("..getNamespace"); + PROTECT(s_getNamespace); + auto expr = Rf_lcons(s_getNamespace, Rf_lcons(info, Rf_lcons(where, R_NilValue))); + PROTECT(expr); + auto val = Rf_eval(expr, R_GlobalEnv); + UNPROTECT(4); + return val; +} + +/// Code from R +void R_expand_binding_value(SEXP b) { +#if BOXED_BINDING_CELLS + SET_BNDCELL_TAG(b, 0); +#else + int typetag = BNDCELL_TAG(b); + if (typetag) { + union { + SEXP sxpval; + double dval; + int ival; + } vv; + SEXP val; + vv.sxpval = CAR0(b); + switch (typetag) { + case REALSXP: + PROTECT(b); + val = ScalarReal(vv.dval); + SET_BNDCELL(b, val); + INCREMENT_NAMED(val); + UNPROTECT(1); + break; + case INTSXP: + PROTECT(b); + val = ScalarInteger(vv.ival); + SET_BNDCELL(b, val); + INCREMENT_NAMED(val); + UNPROTECT(1); + break; + case LGLSXP: + PROTECT(b); + val = ScalarLogical(vv.ival); + SET_BNDCELL(b, val); + INCREMENT_NAMED(val); + UNPROTECT(1); + break; + } + } +#endif +} + +// Will serialize s if it's an instance of CLS +template +static bool tryWrite(AbstractSerializer& serializer, SEXP s) { + if (CLS* b = CLS::check(s)) { + serializer.writeBytesOf(b->info.magic, SerialFlags::RirMagic); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: writeRir", s, [&]{ + b->serialize(serializer); + }); + return true; + } else { + return false; + } +} + +static void writeRir(AbstractSerializer& serializer, SEXP s) { + if (!tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s) && + !tryWrite(serializer, s)) { + std::cerr << "couldn't serialize EXTERNALSXP: "; + Rf_PrintValue(s); + assert(false); + } +} + +static SEXP readRir(AbstractDeserializer& deserializer) { + return Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: readRir", [&]{ + auto magic = deserializer.readBytesOf(SerialFlags::RirMagic); + switch (magic) { + case DISPATCH_TABLE_MAGIC: + return DispatchTable::deserialize(deserializer)->container(); + case CODE_MAGIC: + return Code::deserialize(deserializer)->container(); + case FUNCTION_MAGIC: + return Function::deserialize(deserializer)->container(); + case ARGLIST_ORDER_MAGIC: + return ArglistOrder::deserialize(deserializer)->container(); + case LAZY_ARGS_MAGIC: + return LazyArglist::deserialize(deserializer)->container(); + case LAZY_ENVIRONMENT_MAGIC: + return LazyEnvironment::deserialize(deserializer)->container(); + case PIR_TYPE_FEEDBACK_MAGIC: + return PirTypeFeedback::deserialize(deserializer)->container(); + case TYPEFEEDBACK_MAGIC: + return TypeFeedback::deserialize(deserializer)->container(); + case SERIAL_MODULE_MAGIC: + return SerialModule::deserialize(deserializer)->container(); + case POOL_STUB_MAGIC: + return PoolStub::deserialize(deserializer)->container(); + case PROXY_ENV_MAGIC: + return ProxyEnv::deserialize(deserializer)->container(); + default: + std::cerr << "unhandled RIR object magic: 0x" << std::hex << magic + << "\n"; + assert(false); + } + }); +} + +static void writeBcLang(AbstractSerializer& serializer, SerializedRefs& bcRefs, + SEXP sexp) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: writeBcLang1", sexp, [&]{ + int type = TYPEOF(sexp); + if (type == LANGSXP || type == LISTSXP) { + if (bcRefs.count(sexp)) { + serializer.writeBytesOf(SpecialType::BcRef); + serializer.writeBytesOf((unsigned)bcRefs.at(sexp)); + return; + } else { + serializer.writeBytesOf(type); + bcRefs[sexp] = bcRefs.size(); + } + + auto attr = ATTRIB(sexp); + serializer.writeBytesOf(attr != R_NilValue); + if (attr != R_NilValue) { + serializer.write(attr); + } + serializer.write(TAG(sexp)); + writeBcLang(serializer, bcRefs, CAR(sexp)); + writeBcLang(serializer, bcRefs, CDR(sexp)); + } else { + serializer.writeBytesOf(type); + serializer.write(sexp); + } + }); +} + +static SEXP readBcLang(AbstractDeserializer& deserializer, + SEXPTYPE type, + DeserializedRefs& bcRefs) { + return Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: readBcLang1", [&]{ + switch (type) { + case (SEXPTYPE)SpecialType::BcRef: + return bcRefs.at(deserializer.readBytesOf()); + case LISTSXP: + case LANGSXP: { + auto result = Rf_allocSExp(type); + PROTECT(result); + bcRefs.push_back(result); + if (deserializer.readBytesOf()) { + SET_ATTRIB(result, deserializer.read()); + } + SET_TAG(result, deserializer.read()); + SETCAR(result, readBcLang(deserializer, deserializer.readBytesOf(), bcRefs)); + SETCDR(result, readBcLang(deserializer, deserializer.readBytesOf(), bcRefs)); + UNPROTECT(1); + return result; + } + default: + return deserializer.read(); + } + }); +} + +static void writeBc(AbstractSerializer& serializer, SerializedRefs& bcRefs, + SEXP sexp) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: writeBc1", sexp, [&]{ + SEXP code = R_bcDecode(BCODE_CODE(sexp)); + serializer.write(code, SerialFlags::RBytecodeCode); + auto consts = BCODE_CONSTS(sexp); + auto n = LENGTH(consts); + serializer.writeBytesOf(n); + for (auto i = 0; i < n; i++) { + auto c = VECTOR_ELT(consts, i); + auto type = TYPEOF(c); + switch (type) { + case BCODESXP: + serializer.writeBytesOf(type); + writeBc(serializer, bcRefs, c); + break; + case LANGSXP: + case LISTSXP: + writeBcLang(serializer, bcRefs, c); + break; + default: + serializer.writeBytesOf(type); + serializer.write(c); + break; + } + } + }); +} + +static SEXP readBc(AbstractDeserializer& deserializer, DeserializedRefs* refs, + DeserializedRefs& bcRefs) { + return Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: readBc1", [&]{ + auto result = Rf_allocSExp(BCODESXP); + if (refs) { + refs->push_back(result); + } + PROTECT(result); + auto bytes = deserializer.read(SerialFlags::RBytecodeCode); + PROTECT(bytes); + SETCAR(result, R_bcEncode(bytes)); + auto n = deserializer.readBytesOf(); + auto consts = Rf_allocVector(VECSXP, n); + PROTECT(consts); + for (auto i = 0; i < n; i++) { + auto type = deserializer.readBytesOf(); + SEXP elem; + switch (type) { + case BCODESXP: + // Don't add this element to refs + elem = readBc(deserializer, nullptr, bcRefs); + break; + case (SEXPTYPE)SpecialType::BcRef: + elem = bcRefs.at(deserializer.readBytesOf()); + break; + case LISTSXP: + case LANGSXP: + elem = readBcLang(deserializer, type, bcRefs); + break; + default: + elem = deserializer.read(); + break; + } + SET_VECTOR_ELT(consts, i, elem); + } + SETCDR(result, consts); + SET_TAG(bytes, R_NilValue); + R_registerBC(bytes, result); + UNPROTECT(3); + return result; + }); +} + +static void writeString(AbstractSerializer& serializer, SEXP sexp, + const SerialFlags& flags) { + assert(TYPEOF(sexp) == CHARSXP); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline char vector", sexp, [&]{ + if (sexp == NA_STRING) { + serializer.writeBytesOf(-1, SerialFlags::StringLength); + } else { + auto n = LENGTH(sexp); + serializer.writeBytesOf(n, SerialFlags::StringLength); + serializer.writeBytes(CHAR(sexp), n * sizeof(char), flags); + } + }); +} + +static SEXP readString(AbstractDeserializer& deserializer, + const SerialFlags& flags) { + return Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline char vector", [&]{ + auto length = deserializer.readBytesOf(SerialFlags::StringLength); + if (length == -1) { + return NA_STRING; + } else if (length < 8192) { + // Store data on stack + // R doesn't allow allocVector(SEXP) because it interns + // strings + char data[8192]; + deserializer.readBytes(data, length, flags); + data[length] = '\0'; + return Rf_mkCharLenCE(data, length, CE_NATIVE); + } else { + // Too large, store data on heap + // R doesn't allow allocVector(CHARSXP) because it interns + // strings + char* data = (char*)malloc(length + 1); + deserializer.readBytes(data, length, flags); + data[length] = '\0'; + auto result = Rf_mkCharLenCE(data, length, CE_NATIVE); + free(data); + return result; + } + }); +} + +void AbstractSerializer::writeInline(SEXP sexp) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline", sexp, [&]{ + auto refs = this->refs(); + + SEXPTYPE type; + if (ALTREP(sexp) && ALTREP_SERIALIZED_CLASS(sexp) && ALTREP_SERIALIZED_STATE(sexp)) { + type = (SEXPTYPE)SpecialType::Altrep; + } else if (global2Index.count(sexp)) { + type = (SEXPTYPE)SpecialType::Global; + } else if (canSelfReference(sexp) && refs && refs->count(sexp)) { + type = (SEXPTYPE)SpecialType::Ref; + } else { + type = TYPEOF(sexp); + } + + bool hasTag_ = type != (SEXPTYPE)SpecialType::Global && + type != (SEXPTYPE)SpecialType::Ref && + type != (SEXPTYPE)SpecialType::Altrep && hasTag(sexp); + // With the CHARSXP cache chains maintained through the ATTRIB + // field the content of that field must not be serialized, so + // we treat it as not there. + auto hasAttr = type != (SEXPTYPE)SpecialType::Global && + type != (SEXPTYPE)SpecialType::Ref && + type != CHARSXP && + (type == (SEXPTYPE)SpecialType::Altrep || + ATTRIB(sexp) != R_NilValue); + auto rFlags = packFlags(type, LEVELS(sexp), OBJECT(sexp), hasAttr, hasTag_); + writeBytesOf(rFlags, SerialFlags::RFlags); + + // Write attrs and tag at the beginning if we (maybe) tail call, at the + // end if we self-reference, and otherwise at the end (otherwise doesn't + // matter as long as we read at the same position) + auto writeAttr = [&]{ + if (hasAttr) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline attribute", sexp, [&]{ + write(ATTRIB(sexp), SerialFlags::RAttrib); + }); + } + }; + auto writeTag = [&]{ + if (hasTag_) { + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline tag", sexp, [&]{ + write(TAG(sexp), SerialFlags::RTag); + }); + } + }; + + if (type == TYPEOF(sexp) && canSelfReference(sexp) && refs && + !refs->count(sexp)) { + (*refs)[sexp] = refs->size(); + } + + switch (type) { + case (SEXPTYPE)SpecialType::Altrep: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline altrep", sexp, [&]{ + auto info = ALTREP_SERIALIZED_CLASS(sexp); + auto state = ALTREP_SERIALIZED_STATE(sexp); + PROTECT(info); + PROTECT(state); + write(info, SerialFlags::AltrepInfo); + write(state, SerialFlags::AltrepState); + UNPROTECT(2); + writeAttr(); + // No tag + }); + break; + case (SEXPTYPE)SpecialType::Global: + writeBytesOf(global2Index.at(sexp), SerialFlags::GlobalId); + // Attr and tag already present + break; + case (SEXPTYPE)SpecialType::Ref: + // If you get an out-of-range here, a RIR object is probably either + // not adding its ref, or the rir object should be excluded from + // `canSelfReference` (and probably also `UUIDPool::internable`) + writeBytesOf((unsigned)refs->at(sexp), SerialFlags::RefId); + // Attr and tag already present + break; + case NILSXP: + // No attr or tag + break; + case SYMSXP: { + auto name = PRINTNAME(sexp); + assert(LENGTH(name) > 0 && + "Empty symbol name, sexp should be a global"); + writeString(*this, name, SerialFlags::SymbolName); + writeAttr(); + // No tag + break; + } + case LISTSXP: + case LANGSXP: + case PROMSXP: + case DOTSXP: + writeAttr(); + writeTag(); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline list elem", sexp, [&]{ + if (BNDCELL_TAG(sexp)) { + R_expand_binding_value(sexp); + } + write(CAR(sexp), SerialFlags::Car); + }); + writeInline(CDR(sexp)); + break; + case CLOSXP: + writeAttr(); + writeTag(); + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline closure sans body", sexp, [&]{ + write(CLOENV(sexp), SerialFlags::ClosureEnv); + write(FORMALS(sexp), SerialFlags::ClosureFormals); + }); + writeInline(BODY(sexp)); + break; + case EXTPTRSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline external pointer", sexp, [&]{ + write(EXTPTR_PROT(sexp), SerialFlags::ExternalPtrProtection); + write(EXTPTR_TAG(sexp), SerialFlags::ExternalPtrTag); + }); + writeAttr(); + // No tag + break; + case WEAKREFSXP: + // Only exists as a reference + writeAttr(); + // No tag + break; + case ENVSXP: + if (R_IsPackageEnv(sexp)) { + writeBytesOf(EnvType::Package, SerialFlags::EnvType); + writeInline(PROTECT(R_PackageEnvName(sexp))); + UNPROTECT(1); + } else if (R_IsNamespaceEnv(sexp)) { + writeBytesOf(EnvType::Namespace, SerialFlags::EnvType); + writeInline(PROTECT(R_NamespaceEnvSpec(sexp))); + UNPROTECT(1); + } else { + writeBytesOf(EnvType::Regular, SerialFlags::EnvType); + writeBytesOf((bool)R_EnvironmentIsLocked(sexp), SerialFlags::EnvLock); + write(ENCLOS(sexp), SerialFlags::EnvMisc); + write(FRAME(sexp), SerialFlags::EnvMisc); + write(HASHTAB(sexp), SerialFlags::EnvMisc); + } + writeAttr(); + // No tag + break; + case SPECIALSXP: + case BUILTINSXP: + writeBytesOf(getBuiltinNr(sexp), SerialFlags::BuiltinNr); + writeAttr(); + // No tag + break; + case CHARSXP: + writeString(*this, sexp, SerialFlags::String); + writeAttr(); + // No tag + break; + case LGLSXP: + case INTSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline int vector", sexp, [&]{ + auto n = XLENGTH(sexp); + writeBytesOf(n, SerialFlags::VectorLength); + writeBytes(INTEGER(sexp), n * sizeof(int), SerialFlags::VectorElt); + }); + writeAttr(); + // No tag + break; + case REALSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline real vector", sexp, [&]{ + auto n = XLENGTH(sexp); + writeBytesOf(n, SerialFlags::VectorLength); + writeBytes(REAL(sexp), n * sizeof(double), SerialFlags::VectorElt); + }); + writeAttr(); + // No tag + break; + case CPLXSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline complex number vector", sexp, [&]{ + auto n = XLENGTH(sexp); + writeBytesOf(n, SerialFlags::VectorLength); + writeBytes(COMPLEX(sexp), n * sizeof(Rcomplex), SerialFlags::VectorElt); + }); + writeAttr(); + // No tag + break; + case RAWSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline byte vector", sexp, [&]{ + auto n = XLENGTH(sexp); + writeBytesOf(n, SerialFlags::VectorLength); + writeBytes(RAW(sexp), n * sizeof(Rbyte), SerialFlags::VectorElt); + }); + writeAttr(); + // No tag + break; + case STRSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline string vector", sexp, [&]{ + auto n = XLENGTH(sexp); + writeBytesOf(n, SerialFlags::VectorLength); + for (int i = 0; i < n; i++) { + write(STRING_ELT(sexp, i), SerialFlags::VectorElt); + } + }); + writeAttr(); + // No tag + break; + case VECSXP: + case EXPRSXP: + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractSerializer::writeInline expression or vector", sexp, [&]{ + auto n = XLENGTH(sexp); + writeBytesOf(n, SerialFlags::VectorLength); + for (int i = 0; i < n; i++) { + write(VECTOR_ELT(sexp, i), SerialFlags::VectorElt); + } + }); + writeAttr(); + // No tag + break; + case S4SXP: + // Only attributes (i.e., slots) count + writeAttr(); + // No tag + break; + case BCODESXP: { + SerializedRefs bcRefs; + writeBc(*this, bcRefs, sexp); + writeAttr(); + // No tag + break; + } + case EXTERNALSXP: + writeRir(*this, sexp); + writeAttr(); + // No tag + break; + default: + Rf_error("hashChild: unknown type %i", type); + } + }); +} + +SEXP AbstractDeserializer::readInline() { + return Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline", [&]{ + auto refs = this->refs(); + + auto rFlags = readBytesOf(SerialFlags::RFlags); + SEXPTYPE type; + int levels; + bool object, hasAttr, hasTag_; + unpackFlags(rFlags, type, levels, object, hasAttr, hasTag_); + + // Read attrs and tag at the beginning if we (maybe) tail call, at the + // end if we self-reference, and otherwise at the end (otherwise doesn't + // matter as long as we wrote at the same position) + SEXP attrib = nullptr; + SEXP tag = nullptr; + auto readAttr = [&]{ + if (hasAttr) { + attrib = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline attribute", [&]{ + return read(SerialFlags::RAttrib); + }); + PROTECT(attrib); + } + }; + auto readTag = [&]{ + if (hasTag_) { + tag = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline tag", [&]{ + return read(SerialFlags::RTag); + }); + PROTECT(tag); + } + }; + + SEXP result; + switch (type) { + case (SEXPTYPE)SpecialType::Altrep: { + auto info = PROTECT(read(SerialFlags::AltrepInfo)); + auto state = PROTECT(read(SerialFlags::AltrepState)); + readAttr(); + // No tag + result = ALTREP_UNSERIALIZE_EX(info, state, attrib, object, levels); + UNPROTECT(2); + break; + } + case (SEXPTYPE)SpecialType::Global: + result = globals[readBytesOf(SerialFlags::GlobalId)]; + // Attr and tag already present + break; + case (SEXPTYPE)SpecialType::Ref: + result = refs->at(readBytesOf(SerialFlags::RefId)); + // Attr and tag already present + break; + case NILSXP: + result = R_NilValue; + // No attr or tag + break; + case SYMSXP: { + auto name = readString(*this, SerialFlags::SymbolName); + result = Rf_installTrChar(name); + // Symbols have read refs (same symbol can be serialized and + // we want it to point to the same SEXP when deserializing) + if (refs) { + refs->push_back(result); + } + readAttr(); + // No tag + break; + } + case LISTSXP: + case LANGSXP: + case PROMSXP: + case DOTSXP: + readAttr(); + readTag(); + result = Rf_allocSExp(type); + PROTECT(result); + if (tag && Rf_isSymbol(tag)) { + snprintf(lastname, 8192, "%s", CHAR(PRINTNAME(tag))); + } + Measuring::timeEventIf(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline list elem", result, [&]{ + SETCAR(result, read(SerialFlags::Car)); + }); + SETCDR(result, readInline()); + if (type == CLOSXP && CLOENV(result) == R_NilValue) { + SET_CLOENV(result, R_BaseEnv); + } + if (type == PROMSXP && PRENV(result) == R_NilValue) { + SET_PRENV(result, R_BaseEnv); + } + snprintf(lastname, 8192, ""); + UNPROTECT(1); + break; + case CLOSXP: + readAttr(); + readTag(); + result = Rf_allocSExp(type); + PROTECT(result); + Measuring::timeEventIf( + pir::Parameter::PIR_MEASURE_SERIALIZATION, + "serializeUni.cpp: AbstractDeserializer::readInline closure sans body", result, + [&] { + SET_CLOENV(result, read(SerialFlags::ClosureEnv)); + SET_FORMALS(result, read(SerialFlags::ClosureFormals)); + }); + SET_BODY(result, readInline()); + UNPROTECT(1); + break; + case EXTPTRSXP: + result = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline external pointer", [&]{ + auto result = Rf_allocSExp(type); + PROTECT(result); + if (refs) { + refs->push_back(result); + } + R_SetExternalPtrAddr(result, nullptr); + R_SetExternalPtrProtected(result, read(SerialFlags::ExternalPtrProtection)); + R_SetExternalPtrTag(result, read(SerialFlags::ExternalPtrTag)); + UNPROTECT(1); + return result; + }); + readAttr(); + // No tag + break; + case WEAKREFSXP: + result = R_MakeWeakRef(R_NilValue, R_NilValue, R_NilValue, FALSE); + if (refs) { + refs->push_back(result); + } + readAttr(); + // No tag + break; + case ENVSXP: + switch (readBytesOf(SerialFlags::EnvType)) { + case EnvType::Package: { + auto name = readInline(); + PROTECT(name); + result = R_FindPackageEnv(name); + if (refs) { + refs->push_back(result); + } + UNPROTECT(1); + break; + } + case EnvType::Namespace: { + auto name = readInline(); + PROTECT(name); + result = findNamespace(name); + if (refs) { + refs->push_back(result); + } + UNPROTECT(1); + break; + } + case EnvType::Regular: { + auto isLocked = readBytesOf(SerialFlags::EnvLock); + result = Rf_allocSExp(type); + PROTECT(result); + if (refs) { + refs->push_back(result); + } + + if (willRead(SerialFlags::EnvMisc)) { + SET_ENCLOS(result, read(SerialFlags::EnvMisc)); + SET_FRAME(result, read(SerialFlags::EnvMisc)); + SET_HASHTAB(result, read(SerialFlags::EnvMisc)); + } else { + SET_ENCLOS(result, R_NilValue); + SET_FRAME(result, R_NilValue); + SET_HASHTAB(result, R_NilValue); + } + + R_RestoreHashCount(result); + if (isLocked) { + R_LockEnvironment(result, FALSE); + } + if (ENCLOS(result) == R_NilValue) { + SET_ENCLOS(result, R_BaseEnv); + } + UNPROTECT(1); + break; + } + } + readAttr(); + // No tag + break; + case SPECIALSXP: + case BUILTINSXP: + result = getBuiltinOrSpecialFun(readBytesOf(SerialFlags::BuiltinNr)); + readAttr(); + // No tag + break; + case CHARSXP: + result = readString(*this, SerialFlags::String); + readAttr(); + // No tag + break; + case LGLSXP: + case INTSXP: + result = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline int vector", [&]{ + auto length = readBytesOf(SerialFlags::VectorLength); + auto sexp = Rf_allocVector(type, length); + readBytes((void*)INTEGER(sexp), length * sizeof(int), SerialFlags::VectorElt); + return sexp; + }); + readAttr(); + // No tag + break; + case REALSXP: + result = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline real vector", [&]{ + auto length = readBytesOf(SerialFlags::VectorLength); + auto sexp = Rf_allocVector(type, length); + readBytes((void*)REAL(sexp), length * sizeof(double), SerialFlags::VectorElt); + return sexp; + }); + readAttr(); + // No tag + break; + case CPLXSXP: + result = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline complex number vector sexp", [&]{ + auto length = readBytesOf(SerialFlags::VectorLength); + auto sexp = Rf_allocVector(type, length); + readBytes((void*)COMPLEX(sexp), length * sizeof(Rcomplex), SerialFlags::VectorElt); + return sexp; + }); + readAttr(); + // No tag + break; + case RAWSXP: + result = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline byte vector", [&]{ + auto length = readBytesOf(SerialFlags::VectorLength); + auto sexp = Rf_allocVector(type, length); + readBytes((void*)RAW(sexp), length * sizeof(Rbyte), SerialFlags::VectorElt); + return sexp; + }); + readAttr(); + // No tag + break; + case STRSXP: + result = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline string vector", [&]{ + auto length = readBytesOf(SerialFlags::VectorLength); + auto sexp = Rf_allocVector(type, length); + PROTECT(sexp); + for (int i = 0; i < length; i++) { + SET_STRING_ELT(sexp, i, read(SerialFlags::VectorElt)); + } + UNPROTECT(1); + return sexp; + }); + readAttr(); + // No tag + break; + case VECSXP: + case EXPRSXP: + result = Measuring::timeEventIf3(pir::Parameter::PIR_MEASURE_SERIALIZATION, "serializeUni.cpp: AbstractDeserializer::readInline expression or vector", [&]{ + auto length = readBytesOf(SerialFlags::VectorLength); + auto sexp = Rf_allocVector(type, length); + PROTECT(sexp); + for (int i = 0; i < length; i++) { + SET_VECTOR_ELT(sexp, i, read(SerialFlags::VectorElt)); + } + UNPROTECT(1); + return sexp; + }); + readAttr(); + // No tag + break; + case S4SXP: + // Only attributes (i.e., slots) count + result = Rf_allocSExp(type); + readAttr(); + // No tag + break; + case BCODESXP: { + DeserializedRefs bcRefs; + result = readBc(*this, refs, bcRefs); + readAttr(); + // No tag + break; + } + case EXTERNALSXP: + result = readRir(*this); + readAttr(); + // No tag + break; + default: + Rf_error("hashChild: unknown type %i", type); + } + + PROTECT(result); + if (type != CHARSXP) { + SETLEVELS(result, levels); + } + SET_OBJECT(result, object); + if (attrib) { + SET_ATTRIB(result, attrib); + if (TYPEOF(result) == ENVSXP && + Rf_getAttrib(result, R_ClassSymbol) != R_NilValue) { + // TODO: This is what R's serialization does, it it needed for RIR's serialization + // We don't write out the object bit for environments, so + // reconstruct it here if needed + SET_OBJECT(result, TRUE); + } + } + if (tag) { + SET_TAG(result, tag); + } + if (attrib) { + UNPROTECT(1); + } + if (tag) { + UNPROTECT(1); + } + UNPROTECT(1); + + assert( + (type == (SEXPTYPE)SpecialType::Altrep || + type == (SEXPTYPE)SpecialType::Global || + type == (SEXPTYPE)SpecialType::Ref || type == TYPEOF(result)) && + "sanity check failed: result deserialized into a different type" + ); + SLOWASSERT( + (type == (SEXPTYPE)SpecialType::Altrep || + type == (SEXPTYPE)SpecialType::Global || + type == (SEXPTYPE)SpecialType::Ref || !canSelfReference(result) || + !refs || + std::find(refs->begin(), refs->end(), result) != refs->end()) && + "sanity check failed: type can self reference but wasn't inserted " + "into ref table" + ); + + return result; + }); +} + +} // namespace rir diff --git a/rir/src/serializeHash/serializeUni.h b/rir/src/serializeHash/serializeUni.h new file mode 100644 index 000000000..7329b1b11 --- /dev/null +++ b/rir/src/serializeHash/serializeUni.h @@ -0,0 +1,315 @@ + +// +// Created by Jakob Hain on 8/9/23. +// + +#pragma once + +#include "R/r_incl.h" +#include "utils/ByteBuffer.h" +#include "utils/EnumSet.h" +#include +#include + +namespace rir { + +#define DESERIALIZE(lhs, fun, flags) if (deserializer.willRead(flags)) lhs = deserializer.fun(flags) + +struct SerialOptions; + +/// Details about serialized children to 1) optimize and 2) filter what gets +/// serialized and deserialized (e.g. when hashing, we leave out some data +/// because we want the hash to be semi-consistent). +/// +/// The flags are additive: having a flag only enables more types and categories +/// of data to be serialized. Most flags control what is serialized. However, +/// `MaybeNotAst` instead allows us to use an optimal serialization algorithm +/// for ASTs when hashing; and `MaybeSexp` is simply a sanity check that allows +/// calling `write(SEXP)` without an assertion failure (and may be removed +/// later, since it's not necessary, not really useful, and most of the time +/// coult be chacked by the compiler). +enum class SerialFlag { + /// Data is serialized when computing hash + Hashed, + /// Data, if SEXP, is not necessarily an AST (ASTs are hashed differently) + MaybeNotAst, + /// Data, if SEXP, is not necessarily a recorded call. Recorded calls are + /// always serialized via hash, but other data is serialized inline on the + /// client, since the client doesn't remember every SEXP. + MaybeNotRecordedCall, + /// Data might be an SEXP (sanity check: assertions fail if we serialize an + /// SEXP without this flag) + MaybeSexp, + /// Data is serialized in source. + InSource, + /// Data is serialized in feedback. + InFeedback, + /// Data is not the IsLocked field of an environment + NotEnvLock, + + FIRST = Hashed, + LAST = NotEnvLock +}; + +/// Wrapper so you can't construct non-sensical collections of flags +class SerialFlags { + static unsigned nextId; + unsigned id_; + EnumSet flags; + + SerialFlags(bool hashed, bool maybeNotAst, bool maybeNotRecordedCall, + bool maybeSexp, bool inSource, bool inFeedback, + bool notEnvLock) + : id_(nextId++), flags() { + if (hashed) flags.set(SerialFlag::Hashed); + if (maybeNotAst) flags.set(SerialFlag::MaybeNotAst); + if (maybeNotRecordedCall) flags.set(SerialFlag::MaybeNotRecordedCall); + if (maybeSexp) flags.set(SerialFlag::MaybeSexp); + if (inSource) flags.set(SerialFlag::InSource); + if (inFeedback) flags.set(SerialFlag::InFeedback); + if (notEnvLock) flags.set(SerialFlag::NotEnvLock); + } + + public: + bool contains(SerialFlag f) const { return flags.contains(f); } + /// Each serial flag has its own identifier which is used for santity + /// checks, since these are static singletons. + unsigned id() const { return id_; } + +#define LIST_OF_INHERIT_SERIAL_FLAGS(V) \ + V(Inherit) \ + V(RFlags) \ + V(RAttrib) \ + V(RTag) \ + V(AltrepInfo) \ + V(AltrepState) \ + V(GlobalId) \ + V(RefId) \ + V(SymbolName) \ + V(Car) \ + V(ClosureEnv) \ + V(ClosureFormals) \ + V(ExternalPtrProtection) \ + V(ExternalPtrTag) \ + V(EnvType) \ + V(BuiltinNr) \ + V(StringLength) \ + V(String) \ + V(VectorLength) \ + V(VectorElt) \ + V(RBytecodeCode) \ + V(RirMagic) + +#define LIST_OF_SERIAL_FLAGS(V) \ + LIST_OF_INHERIT_SERIAL_FLAGS(V) \ + V(Ast) \ + V(DtContext) \ + V(DtBaseline) \ + V(DtOptimized) \ + V(FunBody) \ + V(FunDefaultArg) \ + V(FunFeedback) \ + V(FunStats) \ + V(FunMiscBytes) \ + V(CodeArglistOrder) \ + /** In source, but nearly always if not always will be serialized as a */ \ + /** ref because we've already starter serializing the outer function. */ \ + V(CodeOuterFun) \ + /** Child promise in extra pool */ \ + V(CodePromise) \ + /** Data is part of a record_ bytecode. SEXP is a recorded call in */ \ + /** extra pool. */ \ + V(CodeFeedback) \ + /** Unclassified SEXP in extra pool: original bytecode, any pool entry */ \ + /** in native code. */ \ + V(CodePoolUnknown) \ + /** Code kind (i.e. whether the code is native) and native code. */ \ + /** */ \ + /** Technically in source, will rarely if ever actually be in source: */ \ + /** unless we compile a push_ bc which pushes a native code promise, */ \ + /** not even a dispatch table with native code */ \ + V(CodeNative) \ + V(CodeAst) \ + V(CodeMisc) \ + V(EnvLock) \ + V(EnvMisc) + +#define V(name) static const SerialFlags name; + LIST_OF_SERIAL_FLAGS(V) +#undef V + static const SerialFlags _Unused; // NOLINT(*-reserved-identifier) + + static const std::vector& ById; + + static const SerialFlags& parse(const std::string& name); + friend std::ostream& operator<<(std::ostream& out, const SerialFlags& f); +}; + +/// Map of SEXP to ref which will be written in its place if it gets serialized +/// again (so we don't redundantly and infinitely recurse) +typedef std::unordered_map SerializedRefs; +/// Vector of SEXPs (map of int to SEXP) which will be returned in place of the +/// serialized refs +typedef std::vector DeserializedRefs; + +/// Abstract class to serialize or hash an SEXP +class AbstractSerializer { + protected: + AbstractSerializer() = default; + + /// Serial ref table. Returns `nullptr` if we don't recurse + virtual SerializedRefs* refs() = 0; + /// Write SEXP contents. + /// + /// The implementation is extremely similar but not equivalent to + /// `WriteItem` in `serialize.c` + void writeInline(SEXP s); + + public: + /// Corresponding serial options for byte buffer serialization. + virtual const SerialOptions& serialOptions() const = 0; + /// Whether we will write the data with the given flags. Can be used to + /// optimize by removing null-op calls. + virtual bool willWrite(const SerialFlags& flags) const = 0; + /// Write raw data, can't contain any references + virtual void writeBytes(const void* data, size_t size, + const SerialFlags& flags) = 0; + /// Write raw data, can't contain any references + void writeBytes(const void* data, size_t size) { + writeBytes(data, size, SerialFlags::Inherit); + } + /// Write sizeof(int) bytes of raw data, can't contain any references + virtual void writeInt(int data, const SerialFlags& flags) = 0; + /// Write sizeof(int) bytes of raw data, can't contain any references + void writeInt(int data) { writeInt(data, SerialFlags::Inherit); } + /// Write raw data, can't contain any references + template + inline void writeBytesOf(T c, + const SerialFlags& flags = SerialFlags::Inherit) { + if (sizeof(c) == sizeof(int)) { + int result; + // min is redundant, but prevents overflow warnings from linters + memcpy(&result, &c, std::min(sizeof(int), sizeof(T))); + writeInt(result, flags); + } else { + writeBytes((void*)&c, sizeof(c), flags); + } + } + /// Write SEXP (recurse). If non-trivial, may actually write the SEXP + /// contents later instead of actually recursing + virtual void write(SEXP s, const SerialFlags& flags) = 0; + /// Write SEXP (recurse). If non-trivial, may actually write the SEXP + /// contents later instead of actually recursing + void write(SEXP s) { write(s, SerialFlags::Inherit); } + /// Write SEXP which could be nullptr + void writeNullable(SEXP s, + const SerialFlags& flags = SerialFlags::Inherit) { + writeBytesOf(s != nullptr, flags); + if (s) { + write(s, flags); + } + } + /// Write SEXP in constant pool ([cp_pool_at]) + void writeConst(unsigned idx, + const SerialFlags& flags = SerialFlags::Inherit); + /// Write SEXP in source pool ([src_pool_at]) + void writeSrc(unsigned idx, const SerialFlags& flags = SerialFlags::Ast); + + // Helpers + void writeSexpVector(const std::vector& vec, + const SerialFlags& flags = SerialFlags::Inherit) { + writeBytesOf(vec.size(), flags); + for (auto s : vec) { + write(s, flags); + } + } +}; + +/// Abstract class to deserialize an SEXP +class AbstractDeserializer { + protected: + AbstractDeserializer() = default; + + /// Serial ref table. Returns `nullptr` if we don't recurse + virtual DeserializedRefs* refs() = 0; + /// Read SEXP + /// + /// The implementation is extremely similar but not equivalent to `ReadItem` + /// in `serialize.c` + SEXP readInline(); + + public: + /// Corresponding serial options for byte buffer deserialization. + virtual const SerialOptions& serialOptions() const = 0; + /// Whether we will write the data with the given flags. Otherwise we will + /// set the data to 0/null. Can be used to optimize by removing null-op + /// calls AND needed when the data isn't null by default. + virtual bool willRead(const SerialFlags& flags) const = 0; + /// Read raw data, can't contain any references + virtual void readBytes(void* data, size_t size, + const SerialFlags& flags) = 0; + /// Read raw data, can't contain any references + void readBytes(void* data, size_t size) { + readBytes(data, size, SerialFlags::Inherit); + } + /// Read sizeof(int) bytes of raw data, can't contain any references + virtual int readInt(const SerialFlags& flags) = 0; + /// Read sizeof(int) bytes of raw data, can't contain any references + int readInt() { return readInt(SerialFlags::Inherit); } + /// Read raw data, can't contain any references + template + inline T readBytesOf(const SerialFlags& flags = SerialFlags::Inherit) { + if (sizeof(T) == sizeof(int)) { + auto integer = readInt(flags); + T result; + // min is redundant, but prevents overflow warnings from linters + memcpy(&result, &integer, std::min(sizeof(int), sizeof(T))); + return result; + } else { + T result; + readBytes((void*)&result, sizeof(result), flags); + return result; + } + } + /// Read SEXP (recurse). If non-trivial, the returned SEXP may be an empty + /// container which gets filled with deserialized data later, instead of + /// actually recursing + virtual SEXP read(const SerialFlags& flags) = 0; + /// Read SEXP (recurse). If non-trivial, the returned SEXP may be an empty + /// container which gets filled with deserialized data later, instead of + /// actually recursing + SEXP read() { return read(SerialFlags::Inherit); } + /// Read SEXP which could be nullptr + SEXP readNullable(const SerialFlags& flags = SerialFlags::Inherit) { + if (readBytesOf(flags)) { + return read(flags); + } else { + return nullptr; + } + } + /// Read SEXP in constant pool ([cp_pool_add]) + unsigned readConst(const SerialFlags& flags = SerialFlags::Inherit); + /// Read SEXP in source pool ([src_pool_add]) + unsigned readSrc(const SerialFlags& flags = SerialFlags::Ast); + virtual void addRef(SEXP s) { + if (refs()) { + refs()->push_back(s); + } + } + + // Helpers + std::vector readSexpVector(const SerialFlags& flags = SerialFlags::Inherit) { + auto size = readBytesOf(flags); + std::vector result(size); + for (size_t i = 0; i < size; ++i) { + result[i] = read(flags); + } + return result; + } +}; + +/// These SEXPs are added to the ref table the first time they are serialized or +/// deserialized, and serialized as / deserialized from refs subsequent times. +bool canSelfReference(SEXP sexp); + +} // namespace rir diff --git a/rir/src/utils/BimapVector.h b/rir/src/utils/BimapVector.h new file mode 100644 index 000000000..3070c2c97 --- /dev/null +++ b/rir/src/utils/BimapVector.h @@ -0,0 +1,47 @@ +// +// Created by Jakob Hain on 10/9/23. +// + +#pragma once + +#include + +/// Bimap of `std::vector` and `std::unordered_map`. +/// The vector can have multiple copies of the same item, and the map will map +/// to the last occurrence. +template class BimapVector { + std::vector ltr_; + std::unordered_map rtl_; + + public: + BimapVector() = default; + explicit BimapVector(std::vector ltr_) : ltr_(ltr_) { + for (size_t i = 0; i < ltr_.size(); i++) { + rtl_[ltr_[i]] = i; + } + } + + const std::vector& ltr() const { return ltr_; } + const std::unordered_map& rtl() const { return rtl_; } + + size_t size() const { return ltr_.size(); } + bool empty() const { return ltr_.empty(); } + bool count(const T& t) const { return rtl_.count(t); } + const T& at(size_t i) const { + assert(i < ltr_.size() && "BimapVector index out of bounds"); + return ltr_.at(i); + } + size_t at(const T& t) const { + assert(rtl_.count(t) && "BimapVector does not contain this element"); + return rtl_.at(t); + } + + void push_back(const T& t) { + ltr_.push_back(t); + rtl_[t] = ltr_.size() - 1; + } + + bool operator==(const BimapVector& other) const { + return ltr_ == other.ltr_; + } +}; diff --git a/rir/src/utils/ByteBuffer.cpp b/rir/src/utils/ByteBuffer.cpp new file mode 100644 index 000000000..b9b2de638 --- /dev/null +++ b/rir/src/utils/ByteBuffer.cpp @@ -0,0 +1,407 @@ +/** +ByteBuffer +ByteBuffer.cpp +Copyright 2011 - 2013 Ramsey Kant + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Modified 2015 by Ashley Davis (SgtCoDFish) +*/ + +#include "ByteBuffer.h" + +#ifdef BB_USE_NS +// cppcheck-suppress syntaxError +namespace bb { +#endif + + /** +* ByteBuffer constructor +* Reserves specified size in internal vector +* +* @param size Size (in bytes) of space to preallocate internally. Default is set in DEFAULT_SIZE + */ + ByteBuffer::ByteBuffer(uint32_t size) { + buf.reserve(size); + clear(); +#ifdef BB_UTILITY + name = ""; +#endif + } + + /** +* ByteBuffer constructor +* Consume an entire uint8_t array of length len in the ByteBuffer +* +* @param arr uint8_t array of data (should be of length len) +* @param size Size of space to allocate + */ + ByteBuffer::ByteBuffer(uint8_t* arr, uint32_t size) { + // If the provided array is NULL, allocate a blank buffer of the provided size + if (arr == NULL) { + buf.reserve(size); + clear(); + } else { // Consume the provided array + buf.reserve(size); + clear(); + putBytes(arr, size); + } + +#ifdef BB_UTILITY + name = ""; +#endif + } + + /** +* Bytes Remaining +* Returns the number of bytes from the current read position till the end of the buffer +* +* @return Number of bytes from rpos to the end (size()) + */ + uint32_t ByteBuffer::bytesRemaining() { + return size() - rpos; + } + + /** +* Clear +* Clears out all data from the internal vector (original preallocated size remains), resets the positions to 0 + */ + void ByteBuffer::clear() { + rpos = 0; + wpos = 0; + buf.clear(); + } + + /** +* Clone +* Allocate an exact copy of the ByteBuffer on the heap and return a pointer +* +* @return A pointer to the newly cloned ByteBuffer. NULL if no more memory available + */ + std::unique_ptr ByteBuffer::clone() { + std::unique_ptr ret = std::make_unique(buf.size()); + + // Copy data + for (uint32_t i = 0; i < buf.size(); i++) { + ret->put((uint8_t) get(i)); + } + + // Reset positions + ret->setReadPos(0); + ret->setWritePos(0); + + return ret; + } + + /** +* Equals, test for data equivilancy +* Compare this ByteBuffer to another by looking at each byte in the internal buffers and making sure they are the same +* +* @param other A pointer to a ByteBuffer to compare to this one +* @return True if the internal buffers match. False if otherwise + */ + bool ByteBuffer::equals(ByteBuffer* other) { + // If sizes aren't equal, they can't be equal + if (size() != other->size()) + return false; + + // Compare byte by byte + uint32_t len = size(); + for (uint32_t i = 0; i < len; i++) { + if ((uint8_t) get(i) != (uint8_t) other->get(i)) + return false; + } + + return true; + } + + /** +* Resize +* Reallocates memory for the internal buffer of size newSize. Read and write positions will also be reset +* +* @param newSize The amount of memory to allocate + */ + void ByteBuffer::resize(uint32_t newSize) { + buf.resize(newSize); + rpos = 0; + wpos = 0; + } + + /** +* Size +* Returns the size of the internal buffer...not necessarily the length of bytes used as data! +* +* @return size of the internal buffer + */ + uint32_t ByteBuffer::size() const { + return buf.size(); + } + + uint8_t* ByteBuffer::data() { + return buf.data(); + } + + // Replacement + + /** +* Replace +* Replace occurance of a particular uint8_t, key, with the uint8_t rep +* +* @param key uint8_t to find for replacement +* @param rep uint8_t to replace the found key with +* @param start Index to start from. By default, start is 0 +* @param firstOccuranceOnly If true, only replace the first occurance of the key. If false, replace all occurances. False by default + */ + void ByteBuffer::replace(uint8_t key, uint8_t rep, uint32_t start, bool firstOccuranceOnly) { + uint32_t len = buf.size(); + for (uint32_t i = start; i < len; i++) { + uint8_t data = read(i); + // Wasn't actually found, bounds of buffer were exceeded + if ((key != 0) && (data == 0)) + break; + + // Key was found in array, perform replacement + if (data == key) { + buf[i] = rep; + if (firstOccuranceOnly) + return; + } + } + } + + // Read Functions + + uint8_t ByteBuffer::peek() const { + return read(rpos); + } + + uint8_t ByteBuffer::get() const { + return read(); + } + + uint8_t ByteBuffer::get(uint32_t index) const { + return read(index); + } + + void ByteBuffer::getBytes(uint8_t* buf, uint32_t len) const { + for (uint32_t i = 0; i < len; i++) { + buf[i] = read(); + } + } + + bool ByteBuffer::getBool() const { + return read(); + } + + char ByteBuffer::getChar() const { + return read(); + } + + char ByteBuffer::getChar(uint32_t index) const { + return read(index); + } + + double ByteBuffer::getDouble() const { + return read(); + } + + double ByteBuffer::getDouble(uint32_t index) const { + return read(index); + } + + float ByteBuffer::getFloat() const { + return read(); + } + + float ByteBuffer::getFloat(uint32_t index) const { + return read(index); + } + + uint32_t ByteBuffer::getInt() const { + return read(); + } + + uint32_t ByteBuffer::getInt(uint32_t index) const { + return read(index); + } + + uint64_t ByteBuffer::peekLong() const { + return read(rpos); + } + + uint64_t ByteBuffer::getLong() const { + return read(); + } + + uint64_t ByteBuffer::getLong(uint32_t index) const { + return read(index); + } + + uint16_t ByteBuffer::getShort() const { + return read(); + } + + uint16_t ByteBuffer::getShort(uint32_t index) const { + return read(index); + } + + // Write Functions + + void ByteBuffer::put(ByteBuffer* src) { + uint32_t len = src->size(); + for (uint32_t i = 0; i < len; i++) + append(src->get(i)); + } + + void ByteBuffer::put(uint8_t b) { + append(b); + } + + void ByteBuffer::put(uint8_t b, uint32_t index) { + insert(b, index); + } + + void ByteBuffer::putBytes(uint8_t* b, uint32_t len) { + // Insert the data one byte at a time into the internal buffer at position i+starting index + for (uint32_t i = 0; i < len; i++) + append(b[i]); + } + + void ByteBuffer::putBytes(uint8_t* b, uint32_t len, uint32_t index) { + wpos = index; + + // Insert the data one byte at a time into the internal buffer at position i+starting index + for (uint32_t i = 0; i < len; i++) + append(b[i]); + } + + void ByteBuffer::putBool(bool b) { + append(b); + } + + void ByteBuffer::putBool(bool b, uint32_t index) { + insert(b, index); + } + + void ByteBuffer::putChar(char value) { + append(value); + } + + void ByteBuffer::putChar(char value, uint32_t index) { + insert(value, index); + } + + void ByteBuffer::putDouble(double value) { + append(value); + } + + void ByteBuffer::putDouble(double value, uint32_t index) { + insert(value, index); + } + void ByteBuffer::putFloat(float value) { + append(value); + } + + void ByteBuffer::putFloat(float value, uint32_t index) { + insert(value, index); + } + + void ByteBuffer::putInt(uint32_t value) { + append(value); + } + + void ByteBuffer::putInt(uint32_t value, uint32_t index) { + insert(value, index); + } + + void ByteBuffer::putLong(uint64_t value) { + append(value); + } + + void ByteBuffer::putLong(uint64_t value, uint32_t index) { + insert(value, index); + } + + void ByteBuffer::putShort(uint16_t value) { + append(value); + } + + void ByteBuffer::putShort(uint16_t value, uint32_t index) { + insert(value, index); + } + +// Utility Functions +#ifdef BB_UTILITY + void ByteBuffer::setName(const std::string& n) { + name = n; + } + + std::string ByteBuffer::getName() { + return name; + } + + void ByteBuffer::printInfo() { + uint32_t length = buf.size(); + std::cout << "ByteBuffer " << name.c_str() << " Length: " << length << ". Info Print" << std::endl; + } + + void ByteBuffer::printAH() { + uint32_t length = buf.size(); + std::cout << "ByteBuffer " << name.c_str() << " Length: " << length << ". ASCII & Hex Print" << std::endl; + + for (uint32_t i = 0; i < length; i++) { + std::printf("0x%02x ", buf[i]); + } + + std::printf("\n"); + for (uint32_t i = 0; i < length; i++) { + std::printf("%c ", buf[i]); + } + + std::printf("\n"); + } + + void ByteBuffer::printAscii() { + uint32_t length = buf.size(); + std::cout << "ByteBuffer " << name.c_str() << " Length: " << length << ". ASCII Print" << std::endl; + + for (uint32_t i = 0; i < length; i++) { + std::printf("%c ", buf[i]); + } + + std::printf("\n"); + } + + void ByteBuffer::printHex() { + uint32_t length = buf.size(); + std::cout << "ByteBuffer " << name.c_str() << " Length: " << length << ". Hex Print" << std::endl; + + for (uint32_t i = 0; i < length; i++) { + std::printf("0x%02x ", buf[i]); + } + + std::printf("\n"); + } + + void ByteBuffer::printPosition() { + uint32_t length = buf.size(); + std::cout << "ByteBuffer " << name.c_str() << " Length: " << length << " Read Pos: " << rpos << ". Write Pos: " + << wpos << std::endl; + } + +#ifdef BB_USE_NS +// cppcheck-suppress syntaxError +} // namespace bb +#endif + +#endif diff --git a/rir/src/utils/ByteBuffer.h b/rir/src/utils/ByteBuffer.h new file mode 100644 index 000000000..f9579d194 --- /dev/null +++ b/rir/src/utils/ByteBuffer.h @@ -0,0 +1,207 @@ +/** +ByteBuffer +ByteBuffer.h +Copyright 2011 - 2013 Ramsey Kant + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Modified 2015 by Ashley Davis (SgtCoDFish) +*/ + +#ifndef _ByteBuffer_H_ +#define _ByteBuffer_H_ + +// Default number of uint8_ts to allocate in the backing buffer if no size is provided +#define BB_DEFAULT_SIZE 4096 + +// If defined, utility functions within the class are enabled +// #define BB_UTILITY + +// If defined, places the class into a namespace called bb +// #define BB_USE_NS + +#include +#include +#include + +#include +#include + +#ifdef BB_UTILITY +#include +#include +#endif + +#ifdef BB_USE_NS +// cppcheck-suppress syntaxError +namespace bb { +#endif + + class ByteBuffer { + public: + ByteBuffer(uint32_t size = BB_DEFAULT_SIZE); + ByteBuffer(uint8_t* arr, uint32_t size); + ~ByteBuffer() = default; + + uint32_t bytesRemaining(); // Number of uint8_ts from the current read position till the end of the buffer + void clear(); // Clear our the vector and reset read and write positions + std::unique_ptr clone(); // Return a new instance of a ByteBuffer with the exact same contents and the same state (rpos, wpos) + //ByteBuffer compact(); // TODO? + bool equals(ByteBuffer* other); // Compare if the contents are equivalent + void resize(uint32_t newSize); + uint32_t size() const; // Size of internal vector + uint8_t* data(); + + // Basic Searching (Linear) + template int32_t find(T key, uint32_t start = 0) { + int32_t ret = -1; + uint32_t len = buf.size(); + for (uint32_t i = start; i < len; i++) { + T data = read(i); + // Wasn't actually found, bounds of buffer were exceeded + if ((key != 0) && (data == 0)) + break; + + // Key was found in array + if (data == key) { + ret = (int32_t) i; + break; + } + } + return ret; + } + + // Replacement + void replace(uint8_t key, uint8_t rep, uint32_t start = 0, bool firstOccuranceOnly = false); + + // Read + + uint8_t peek() const; // Relative peek. Reads and returns the next uint8_t in the buffer from the current position but does not increment the read position + uint8_t get() const; // Relative get method. Reads the uint8_t at the buffers current position then increments the position + uint8_t get(uint32_t index) const; // Absolute get method. Read uint8_t at index + void getBytes(uint8_t* buf, uint32_t len) const; // Absolute read into array buf of length len + bool getBool() const; // Relative + char getChar() const; // Relative + char getChar(uint32_t index) const; // Absolute + double getDouble() const; + double getDouble(uint32_t index) const; + float getFloat() const; + float getFloat(uint32_t index) const; + uint32_t getInt() const; + uint32_t getInt(uint32_t index) const; + uint64_t peekLong() const; + uint64_t getLong() const; + uint64_t getLong(uint32_t index) const; + uint16_t getShort() const; + uint16_t getShort(uint32_t index) const; + + // Write + + void put(ByteBuffer* src); // Relative write of the entire contents of another ByteBuffer (src) + void put(uint8_t b); // Relative write + void put(uint8_t b, uint32_t index); // Absolute write at index + void putBytes(uint8_t* b, uint32_t len); // Relative write + void putBytes(uint8_t* b, uint32_t len, uint32_t index); // Absolute write starting at index + void putBool(bool b); // Relative write + void putBool(bool b, uint32_t index); // Absolute write at index + void putChar(char value); // Relative + void putChar(char value, uint32_t index); // Absolute + void putDouble(double value); + void putDouble(double value, uint32_t index); + void putFloat(float value); + void putFloat(float value, uint32_t index); + void putInt(uint32_t value); + void putInt(uint32_t value, uint32_t index); + void putLong(uint64_t value); + void putLong(uint64_t value, uint32_t index); + void putShort(uint16_t value); + void putShort(uint16_t value, uint32_t index); + + // Buffer Position Accessors & Mutators + + void setReadPos(uint32_t r) { + rpos = r; + } + + uint32_t getReadPos() const { + return rpos; + } + + void setWritePos(uint32_t w) { + wpos = w; + } + + uint32_t getWritePos() const { + return wpos; + } + + // Utility Functions +#ifdef BB_UTILITY + void setName(const std::string& n); + std::string getName(); + void printInfo(); + void printAH(); + void printAscii(); + void printHex(); + void printPosition(); +#endif + + private: + uint32_t wpos; + mutable uint32_t rpos; + std::vector buf; + +#ifdef BB_UTILITY + std::string name; +#endif + + template T read() const { + T data = read(rpos); + rpos += sizeof(T); + return data; + } + + template T read(uint32_t index) const { + if (index + sizeof(T) <= buf.size()) + // cppcheck-suppress invalidPointerCast + return *(reinterpret_cast((uint8_t*)&buf[index])); + return 0; + } + + template void append(T data) { + uint32_t s = sizeof(data); + + if (size() < (wpos + s)) + buf.resize(wpos + s); + memcpy(&buf[wpos], reinterpret_cast(&data), s); + //printf("writing %c to %i\n", (uint8_t)data, wpos); + + wpos += s; + } + + template void insert(T data, uint32_t index) { + if ((index + sizeof(data)) > size()) { + buf.resize(size() + (index + sizeof(data))); + } + + memcpy(&buf[index], reinterpret_cast(&data), sizeof(data)); + wpos = index + sizeof(data); + } + }; + +#ifdef BB_USE_NS +// cppcheck-suppress syntaxError +} // namespace bb +#endif + +#endif diff --git a/rir/src/utils/FunctionWriter.h b/rir/src/utils/FunctionWriter.h index d32699888..d6fab2f94 100644 --- a/rir/src/utils/FunctionWriter.h +++ b/rir/src/utils/FunctionWriter.h @@ -5,6 +5,7 @@ #include "bc/BC_inc.h" #include "interpreter/instance.h" #include "runtime/Function.h" +#include "runtime/TypeFeedback.h" #include #include @@ -40,7 +41,7 @@ class FunctionWriter { } void finalize(Code* body, const FunctionSignature& signature, - const Context& context) { + const Context& context, TypeFeedback* feedback) { assert(function_ == nullptr && "Trying to finalize a second time"); size_t dataSize = defaultArgs.size() * sizeof(SEXP); @@ -48,8 +49,9 @@ class FunctionWriter { SEXP store = Rf_allocVector(EXTERNALSXP, functionSize); void* payload = INTEGER(store); - Function* fun = new (payload) Function(functionSize, body->container(), - defaultArgs, signature, context); + Function* fun = + new (payload) Function(functionSize, body->container(), defaultArgs, + signature, context, feedback); preserve(store); assert(fun->info.magic == FUNCTION_MAGIC); diff --git a/rir/src/utils/HTMLBuilder/Document.h b/rir/src/utils/HTMLBuilder/Document.h new file mode 100644 index 000000000..6e171e160 --- /dev/null +++ b/rir/src/utils/HTMLBuilder/Document.h @@ -0,0 +1,105 @@ +/** +* @file Document.h +* @ingroup HtmlBuilder +* @brief Root Element of the HTML Document Object Model. +* +* Copyright (c) 2017-2021 Sebastien Rombauts (sebastien.rombauts@gmail.com) +* +* Distributed under the MIT License (MIT) (See accompanying file LICENSE.txt +* or copy at http://opensource.org/licenses/MIT) +*/ +#pragma once + +#include "Element.h" + +#include +#include +#include + +/// A simple C++ HTML Generator library. +namespace HTML { + +// Note: to configure indentation & minification, define this at compile time before including HTML headers. +#ifndef HTML_INDENTATION +#define HTML_INDENTATION 2 +#endif +#ifndef HTML_ENDLINE +#define HTML_ENDLINE "\n" +#endif + +/** +* @brief Root Element \ of the HTML Document Object Model. +* +* The Document is a specialized Element with restriction on what can be done on it, +* since many aspects of the \ root tag are well defined and constrained. +*/ +class Document : public Element { + public: + Document() : + Element(), mHead(*static_cast(&mChildren[0])), mBody(*static_cast(&mChildren[1])) { + } + explicit Document(const char* apTitle) : + Element(), mHead(*static_cast(&mChildren[0])), mBody(*static_cast(&mChildren[1])) { + mHead << HTML::Title(apTitle); + } + explicit Document(const std::string& aTitle) : + Element(), mHead(*static_cast(&mChildren[0])), mBody(*static_cast(&mChildren[1])) { + mHead << HTML::Title(aTitle); + } + Document(const char* apTitle, Style&& aStyle) : + Element(), mHead(*static_cast(&mChildren[0])), mBody(*static_cast(&mChildren[1])) { + mHead << HTML::Title(apTitle); + mHead << std::move(aStyle); + } + Document(const char* apTitle, const Style& aStyle) : + Element(), mHead(*static_cast(&mChildren[0])), mBody(*static_cast(&mChildren[1])) { + mHead << HTML::Title(apTitle); + mHead << Style(aStyle); + } + + Document& operator<<(Element&& aElement) { + mBody << std::move(aElement); + return *this; + } + + Element& head() { + return mHead; + } + Element& body() { + return mBody; + } + + void lang(const char* apLang) { + mHead.addAttribute("lang", apLang); + } + + friend std::ostream& operator<< (std::ostream& aStream, const Document& aElement); + + std::string toString() const { + std::ostringstream stream; + stream << *this; + return stream.str(); + } + + operator std::string() const { + return toString(); + } + + private: + std::ostream& toString(std::ostream& aStream) const { + aStream << "" HTML_ENDLINE; + Element::toString(aStream); + return aStream; + } + + private: + Head& mHead; ///< Reference to the first child Element \ + Body& mBody; ///< Reference to the second child Element \ +}; + +inline std::ostream& operator<< (std::ostream& aStream, const Document& aDocument) { + return aDocument.toString(aStream); +} + +} // namespace HTML + diff --git a/rir/src/utils/HTMLBuilder/Element.h b/rir/src/utils/HTMLBuilder/Element.h new file mode 100644 index 000000000..8504283c0 --- /dev/null +++ b/rir/src/utils/HTMLBuilder/Element.h @@ -0,0 +1,1012 @@ +/** +* @file Element.h +* @ingroup HtmlBuilder +* @brief Definitions of an Element in the HTML Document Object Model, and various specialized Element types. +* +* Copyright (c) 2017-2021 Sebastien Rombauts (sebastien.rombauts@gmail.com) +* +* Distributed under the MIT License (MIT) (See accompanying file LICENSE.txt +* or copy at http://opensource.org/licenses/MIT) +*/ +#pragma once + +#include +#include +#include +#include +#include +#include + +/// A simple C++ HTML Generator library. +namespace HTML { + +// Note: to configure indentation & minification, define this at compile time before including HTML headers. +#ifndef HTML_INDENTATION +#define HTML_INDENTATION 2 +#endif +#ifndef HTML_ENDLINE +#define HTML_ENDLINE "\n" +#endif + +/// Convert a boolean to string like std::boolalpha in a std::ostream +constexpr const char* to_string(bool aBool) { + return aBool ? "true" : "false"; +} + +/** +* @brief Definitions of an Element in the HTML Document Object Model, and various specialized Element types. +* +* An Element represents any HTML node in the Document Object Model. +*/ +class Element { + public: + explicit Element(const char* apName, const char* apContent = nullptr) : + mName(apName), mContent(apContent ? apContent : "") {} + Element(const char* apName, std::string&& aContent) : + mName(apName), mContent(aContent) {} + Element(const char* apName, const std::string& aContent) : + mName(apName), mContent(aContent) {} + + Element&& addAttribute(const char* apName, const char* apValue) { + if (apName && apValue) { + mAttributes.push_back({ apName, apValue }); + } + return std::move(*this); + } + Element&& addAttribute(const char* apName, const std::string& aValue) { + mAttributes.push_back({apName, aValue}); + return std::move(*this); + } + Element&& addOrAppendAttribute(const char* apName, const char* apValue) { + if (apName && apValue) { + for (auto& attribute : mAttributes) { + if (attribute.Name == apName) { + attribute.Value += " "; + attribute.Value += apValue; + return std::move(*this); + } + } + } + return addAttribute(apName, apValue); + } + Element&& addOrAppendAttribute(const char* apName, const std::string& aValue) { + for (auto& attribute : mAttributes) { + if (attribute.Name == apName) { + attribute.Value += " "; + attribute.Value += aValue; + return std::move(*this); + } + } + return addAttribute(apName, aValue); + } + Element&& addAttribute(const char* apName, const unsigned int aValue) { + mAttributes.push_back({apName, std::to_string(aValue)}); + return std::move(*this); + } + Element&& operator<<(Element&& aElement) { + mChildren.push_back(std::move(aElement)); + return std::move(*this); + } + Element&& operator<<(const char* apContent); + Element&& operator<<(std::string&& aContent); + Element&& operator<<(const std::string& aContent); + + friend std::ostream& operator<<(std::ostream& aStream, const Element& aElement); + std::string toString() const { + std::ostringstream stream; + stream << *this; + return stream.str(); + } + + Element&& id(const char* apValue) { + return addAttribute("id", apValue); + } + Element&& id(const std::string& aValue) { + return addAttribute("id", aValue); + } + + Element&& cls(const char* apValue) { + return addOrAppendAttribute("class", apValue); + } + Element&& cls(const std::string& aValue) { + return addOrAppendAttribute("class", aValue); + } + + Element&& title(const char* apValue) { + return addAttribute("title", apValue); + } + Element&& title(const std::string& aValue) { + return addAttribute("title", aValue); + } + + Element&& style(const char* apValue) { + return addAttribute("style", apValue); + } + Element&& style(const std::string& aValue) { + return addAttribute("style", aValue); + } + + struct Attribute { + std::string Name; + std::string Value; + }; + + protected: + /// Constructor reserved for the Root \ Element as well as the Empty + Element(); + + std::ostream& toString(std::ostream& aStream, const size_t aIndentation = 0) const { + toStringOpen(aStream, aIndentation); + toStringContent(aStream, aIndentation); + toStringClose(aStream, aIndentation); + return aStream; + } + + private: + void toStringOpen(std::ostream& aStream, const size_t aIndentation) const { + if (!mName.empty()) { + std::fill_n(std::ostream_iterator(aStream), aIndentation, ' '); + aStream << '<' << mName; + + for (const auto& attr : mAttributes) { + aStream << ' ' << attr.Name; + if (!attr.Value.empty()) { + aStream << "=\"" << attr.Value << "\""; + } + } + + if (mContent.empty()) { + // Note: using children for content is less efficient/breaking the assumption + if (!mChildren.empty() || mbVoid) { + aStream << ">" HTML_ENDLINE; + } else { + aStream << ">"; + } + } else { + aStream << '>'; + } + } + } + void toStringContent(std::ostream& aStream, const size_t aIndentation) const { + if (!mName.empty()) { + aStream << mContent; + for (auto& child : mChildren) { + child.toString(aStream, aIndentation + HTML_INDENTATION); + } + } else { + std::fill_n(std::ostream_iterator(aStream), aIndentation, ' '); + aStream << mContent << HTML_ENDLINE; + } + } + void toStringClose(std::ostream& aStream, const size_t aIndentation) const { + if (!mName.empty()) { + if (!mChildren.empty()) { + std::fill_n(std::ostream_iterator(aStream), aIndentation, ' '); + } + // Note: using children for content is less efficient/breaking the assumption + if (!mContent.empty() || !mChildren.empty() || !mbVoid) { + aStream << "" HTML_ENDLINE; + } + } + } + + protected: + std::string mName; + std::string mContent; + std::vector mAttributes; + std::vector mChildren; + + // Self-closing elements complete list: + //
+ // + bool mbVoid = false; +}; + +inline std::ostream& operator<<(std::ostream& aStream, const Element& aElement) { + return aElement.toString(aStream); +} + +/// Empty Element, useful as a default parameter for instance +class Empty : public Element { + public: + Empty() : Element() {} +}; + +/// Raw content text (unnamed Element) to use as text values between child Elements +class Text : public Element { + public: + explicit Text(const char* apContent) : Element("", apContent) {} + explicit Text(std::string&& aContent) : Element("", aContent) {} + explicit Text(const std::string& aContent) : Element("", aContent) {} +}; + +inline Element&& Element::operator<<(const char* apContent) { + return *this << Text(apContent); +} + +inline Element&& Element::operator<<(std::string&& aContent) { + return *this << Text(std::move(aContent)); +} + +inline Element&& Element::operator<<(const std::string& aContent) { + return *this << Text(aContent); +} + +/// \ Element required in \ +class Title : public Element { + public: + explicit Title(const char* apContent) : Element("title", apContent) {} + explicit Title(const std::string& aContent) : Element("title", aContent) {} +}; + +/// \ Element for inline CSS in \ +class Style : public Element { + public: + explicit Style(const char* apContent) : Element("style", apContent) {} + explicit Style(const std::string& aContent) : Element("style", aContent) {} +}; + +/// \ Element for inline Javascript in \ +class Script : public Element { + public: + Script() : Element("script") {} + explicit Script(const char* apSrc) : Element("script") { + if (apSrc) { + addAttribute("src", apSrc); + } + } + explicit Script(const char* apSrc, const char* apContent) : Element("script", apContent) { + if (apSrc) { + addAttribute("src", apSrc); + } + } + Script&& integrity(const std::string& aValue) { + addAttribute("integrity", aValue); + return std::move(*this); + } + Script&& crossorigin(const std::string& aValue) { + addAttribute("crossorigin", aValue); + return std::move(*this); + } +}; + +/// \ metadata about the Document in \ +class Meta : public Element { + public: + Meta() : Element("meta") {} + explicit Meta(const char* apCharset) : Element("meta") { + addAttribute("charset", apCharset); + mbVoid = true; + } + explicit Meta(const char* apName, const char* apContent) : Element("meta") { + addAttribute("name", apName); + addAttribute("content", apContent); + mbVoid = true; + } +}; + +/// \ Element to reference external CSS or Javascript files +class Rel : public Element { + public: + Rel(const char* apRel, const char* apUrl, const char* apType = nullptr) : Element("link") { + addAttribute("rel", apRel); + addAttribute("href", apUrl); + if (apType) { + addAttribute("type", apType); + } + mbVoid = true; + } + + Rel&& integrity(const std::string& aValue) { + addAttribute("integrity", aValue); + return std::move(*this); + } + Rel&& crossorigin(const std::string& aValue) { + addAttribute("crossorigin", aValue); + return std::move(*this); + } +}; + +/// \ Element in \ +class Base : public Element { + public: + Base(const std::string& aContent, const std::string& aUrl, const char* apTarget) : Element("base", aContent) { + addAttribute("href", aUrl); + if (apTarget) { + addAttribute("target", apTarget); + } + } +}; + +/// \ required as the first child Element in every HTML Document +class Head : public Element { + public: + Head() : Element("head") {} + + Head&& operator<<(Element&& aElement) = delete; + Head&& operator<<(Title&& aTitle) { + mChildren.push_back(std::move(aTitle)); + return std::move(*this); + } + Head&& operator<<(Style&& aStyle) { + mChildren.push_back(std::move(aStyle)); + return std::move(*this); + } + Head&& operator<<(Script&& aScript) { + mChildren.push_back(std::move(aScript)); + return std::move(*this); + } + Head&& operator<<(Meta&& aMeta) { + mChildren.push_back(std::move(aMeta)); + return std::move(*this); + } + Head&& operator<<(Rel&& aRel) { + mChildren.push_back(std::move(aRel)); + return std::move(*this); + } + Head&& operator<<(Base&& aBase) { + mChildren.push_back(std::move(aBase)); + return std::move(*this); + } +}; + +/// \ required as the second child Element in every HTML Document +class Body : public Element { + public: + Body() : Element("body") {} +}; + +// Constructor of the Root \ Element +inline Element::Element() : mName("html"), mChildren{Head(), Body()} { +} + + +/// \ Line break Element +class Break : public Element { + public: + Break() : Element("br") { + mbVoid = true; + } +}; + +/// \ Table Header Column Element +class ColHeader : public Element { + public: + explicit ColHeader(const char* apContent = nullptr) : Element("th", apContent) {} + explicit ColHeader(std::string&& aContent) : Element("th", aContent) {} + explicit ColHeader(const std::string& aContent) : Element("th", aContent) {} + + ColHeader&& operator<<(Element&& aElement) { + mChildren.push_back(std::move(aElement)); + return std::move(*this); + } + + ColHeader&& rowSpan(const unsigned int aNbRow) { + if (0 < aNbRow) { + addAttribute("rowspan", aNbRow); + } + return std::move(*this); + } + ColHeader&& colSpan(const unsigned int aNbCol) { + if (0 < aNbCol) { + addAttribute("colspan", aNbCol); + } + return std::move(*this); + } +}; + +/// \ Table Column Element +class Col : public Element { + public: + explicit Col(const char* apContent = nullptr) : Element("td", apContent) {} + explicit Col(std::string&& aContent) : Element("td", aContent) {} + explicit Col(const std::string& aContent) : Element("td", aContent) {} + explicit Col(const bool abContent) : Element("td", to_string(abContent)) {} + explicit Col(const int aContent) : Element("td", std::to_string(aContent)) {} + explicit Col(const unsigned int aContent) : Element("td", std::to_string(aContent)) {} + explicit Col(const long long aContent) : Element("td", std::to_string(aContent)) {} + explicit Col(const unsigned long long aContent) : Element("td", std::to_string(aContent)) {} + explicit Col(const float aContent) : Element("td", std::to_string(aContent)) {} + explicit Col(const double aContent) : Element("td", std::to_string(aContent)) {} + + Col&& operator<<(Element&& aElement) { + mChildren.push_back(std::move(aElement)); + return std::move(*this); + } + + Col&& rowSpan(const unsigned int aNbRow) { + if (0 < aNbRow) { + addAttribute("rowspan", aNbRow); + } + return std::move(*this); + } + Col&& colSpan(const unsigned int aNbCol) { + if (0 < aNbCol) { + addAttribute("colspan", aNbCol); + } + return std::move(*this); + } + Col&& style(const std::string& aValue) { + Element::style(aValue); + return std::move(*this); + } +}; + +/// \ Table Row Element +class Row : public Element { + public: + Row() : Element("tr") {} + + Row&& operator<<(Element&& aElement) = delete; + Row&& operator<<(ColHeader&& aCol) { + mChildren.push_back(std::move(aCol)); + return std::move(*this); + } + Row&& operator<<(Col&& aCol) { + mChildren.push_back(std::move(aCol)); + return std::move(*this); + } + Row&& style(const std::string& aValue) { + Element::style(aValue); + return std::move(*this); + } +}; + +/// \ Table Caption Element +class Caption : public Element { + public: + explicit Caption(const char* apContent) : Element("caption", apContent) {} +}; + +/// \ Element +class Table : public Element { + public: + Table() : Element("table") {} + + Table&& operator<<(Element&& aElement) = delete; + Table&& operator<<(Row&& aRow) { + mChildren.push_back(std::move(aRow)); + return std::move(*this); + } + Table&& operator<<(Caption&& aCaption) { + mChildren.push_back(std::move(aCaption)); + return std::move(*this); + } +}; + +/// \ List Item Element to put in List +class ListItem : public Element { + public: + ListItem() : Element("li") {} + explicit ListItem(const char* apContent) : Element("li", apContent) {} + explicit ListItem(const std::string& aContent) : Element("li", aContent) {} + + ListItem&& operator<<(Element&& aElement) { + mChildren.push_back(std::move(aElement)); + return std::move(*this); + } + + ListItem&& cls(const std::string& aValue) { + addOrAppendAttribute("class", aValue); + return std::move(*this); + } +}; + +/// \ Ordered List or \ Unordered List Element to use with ListItem +class List : public Element { + public: + explicit List(const bool abOrdered = false) : Element(abOrdered?"ol":"ul") {} + List(const bool abOrdered, const char* apClass) : Element(abOrdered ?"ol":"ul") { + cls(apClass); + } + + List&& operator<<(Element&& aElement) = delete; + List&& operator<<(ListItem&& aItem) { + mChildren.push_back(std::move(aItem)); + return std::move(*this); + } +}; + +/// \ Element +class Form : public Element { + public: + explicit Form(const char* apAction = nullptr, const char* apMethod = nullptr) : Element("form") { + if (apAction) { + addAttribute("action", apAction); + } + if (apMethod) { + addAttribute("method", apMethod); + } + } +}; + +/// \ Element to use in Form +class Input : public Element { + public: + explicit Input(const char* apType = nullptr, const char* apName = nullptr, + const char* apValue = nullptr, const char* apContent = nullptr) : Element("input", apContent) { + if (apType) { + addAttribute("type", apType); + } + if (apName) { + addAttribute("name", apName); + } + if (apValue) { + addAttribute("value", apValue); + } + mbVoid = true; + } + + Input&& addAttribute(const char* apName, const std::string& aValue) { + Element::addAttribute(apName, aValue); + return std::move(*this); + } + Input&& addAttribute(const char* apName, const unsigned int aValue) { + Element::addAttribute(apName, aValue); + return std::move(*this); + } + + Input&& id(const std::string& aValue) { + return addAttribute("id", aValue); + } + Input&& cls(const std::string& aValue) { + addOrAppendAttribute("class", aValue); + return std::move(*this); + } + Input&& title(const std::string& aValue) { + return addAttribute("title", aValue); + } + Input&& style(const std::string& aValue) { + return addAttribute("style", aValue); + } + + Input&& size(const unsigned int aSize) { + return addAttribute("size", aSize); + } + Input&& maxlength(const unsigned int aMaxlength) { + return addAttribute("maxlength", aMaxlength); + } + Input&& placeholder(const std::string& aPlaceholder) { + return addAttribute("placeholder", aPlaceholder); + } + Input&& min(const std::string& aMin) { + return addAttribute("min", aMin); + } + Input&& min(const unsigned int aMin) { + return addAttribute("min", aMin); + } + Input&& max(const std::string& aMax) { // NOLINT(build/include_what_you_use) false positive + return addAttribute("max", aMax); + } + Input&& max(const unsigned int aMax) { // NOLINT(build/include_what_you_use) false positive + return addAttribute("max", aMax); + } + + Input&& checked(const bool abChecked = true) { + if (abChecked) { + addAttribute("checked", ""); + } + return std::move(*this); + } + Input&& autocomplete() { + return addAttribute("autocomplete", ""); + } + Input&& autofocus() { + return addAttribute("autofocus", ""); + } + Input&& disabled() { + return addAttribute("disabled", ""); + } + Input&& readonly() { + return addAttribute("readonly", ""); + } + Input&& required() { + return addAttribute("required", ""); + } +}; + +/// \ Radio Element to use in Form +class InputRadio : public Input { + public: + explicit InputRadio(const char* apName, const char* apValue = nullptr, const char* apContent = nullptr) : + Input("radio", apName, apValue, apContent) { + } +}; + +/// \ Checkbox Element to use in Form +class InputCheckbox : public Input { + public: + explicit InputCheckbox(const char* apName, const char* apValue = nullptr, const char* apContent = nullptr) : + Input("checkbox", apName, apValue, apContent) { + } +}; + +/// \ hidden Element to use in Form +class InputHidden : public Input { + public: + explicit InputHidden(const char* apName, const char* apValue = nullptr) : + Input("hidden", apName, apValue) { + } +}; + +/// \ text Element to use in Form +class InputText : public Input { + public: + explicit InputText(const char* apName, const char* apValue = nullptr) : + Input("text", apName, apValue) { + } +}; + +/// \ Element to use in Form +class TextArea : public Element { + public: + explicit TextArea(const char* apName, const unsigned int aCols = 0, const unsigned int aRows = 0) : + Element("textarea") { + addAttribute("name", apName); + if (0 < aCols) { + addAttribute("cols", aCols); + } + if (0 < aRows) { + addAttribute("rows", aRows); + } + } + TextArea&& maxlength(const unsigned int aMaxlength) { + addAttribute("maxlength", aMaxlength); + return std::move(*this); + } +}; + +/// \ Number Element to use in Form +class InputNumber : public Input { + public: + explicit InputNumber(const char* apName, const char* apValue = nullptr) : + Input("number", apName, apValue) { + } +}; + +/// \ Range Element to use in Form +class InputRange : public Input { + public: + explicit InputRange(const char* apName, const char* apValue = nullptr) : + Input("range", apName, apValue) { + } +}; + +/// \ Date Element to use in Form +class InputDate : public Input { + public: + explicit InputDate(const char* apName, const char* apValue = nullptr) : + Input("date", apName, apValue) { + } +}; + +/// \ Time Element to use in Form +class InputTime : public Input { + public: + explicit InputTime(const char* apName, const char* apValue = nullptr) : + Input("time", apName, apValue) { + } +}; + +/// \ E-mail Element to use in Form +class InputEmail : public Input { + public: + explicit InputEmail(const char* apName, const char* apValue = nullptr) : + Input("email", apName, apValue) { + } +}; + +/// \ URL Element to use in Form +class InputUrl : public Input { + public: + explicit InputUrl(const char* apName, const char* apValue = nullptr) : + Input("url", apName, apValue) { + } +}; + +/// \ Password Element to use in Form +class InputPassword : public Input { + public: + explicit InputPassword(const char* apName) : + Input("password", apName) { + } +}; + +/// \ Submit Button Element to use in Form +class InputSubmit : public Input { + public: + explicit InputSubmit(const char* apValue = nullptr, const char* apName = nullptr) : + Input("submit", apName, apValue) { + } +}; + +/// \ Reset Button Element to use in Form +class InputReset : public Input { + public: + explicit InputReset(const char* apValue = nullptr) : + Input("reset", nullptr, apValue) { + } +}; + +/// \ List Element to use in Form with DataList +class InputList : public Input { + public: + explicit InputList(const char* apName, const char* apList) : Input(nullptr, apName) { + addAttribute("list", apList); + } +}; + +/// \ Element for InputList, to use with Option Elements +class DataList : public Element { + public: + explicit DataList(const char* apId) : Element("datalist") { + addAttribute("id", apId); + } +}; + +/// \ Element to use with Option Elements +class Select : public Element { + public: + explicit Select(const char* apName) : Element("select") { + addAttribute("name", apName); + } +}; + +/// \ Element for Select and DataList +class Option : public Element { + public: + explicit Option(const char* apValue, const char* apContent = nullptr) : Element("option", apContent) { + addAttribute("value", apValue); + } + + Option&& selected(const bool abSelected = true) { + if (abSelected) { + addAttribute("selected", ""); + } + return std::move(*this); + } +}; + +/// \ Element +class Header1 : public Element { + public: + explicit Header1(const std::string& aContent) : Element("h1", aContent) {} +}; + +/// \ Element +class Header2 : public Element { + public: + explicit Header2(const std::string& aContent) : Element("h2", aContent) {} +}; + +/// \ Element +class Header3 : public Element { + public: + explicit Header3(const std::string& aContent) : Element("h3", aContent) {} +}; + +/// \ bold Element +class Bold : public Element { + public: + explicit Bold(const std::string& aContent) : Element("b", aContent) {} +}; + +/// \ italic Element +class Italic : public Element { + public: + explicit Italic(const std::string& aContent) : Element("i", aContent) {} +}; + +/// \ Element for side-comment text and small print, including copyright and legal text +class Small : public Element { + public: + Small() : Element("small") {} + explicit Small(const char* apContent) : Element("small", apContent) {} + explicit Small(std::string&& aContent) : Element("small", aContent) {} + explicit Small(const std::string& aContent) : Element("small", aContent) {} +}; + +/// \ Element for important text +class Strong : public Element { + public: + Strong() : Element("strong") {} + explicit Strong(const char* apContent) : Element("strong", apContent) {} + explicit Strong(std::string&& aContent) : Element("strong", aContent) {} + explicit Strong(const std::string& aContent) : Element("strong", aContent) {} +}; + +/// \ paragraph Element +class Paragraph : public Element { + public: + explicit Paragraph(const std::string& aContent) : Element("p", aContent) {} +}; + +/// \ division Element to group elements in a rectangular block. +class Div : public Element { + public: + Div() : Element("div") {} + explicit Div(const char* apClass) : Element("div") { + cls(apClass); + } + + Div&& cls(const std::string& aValue) { + addOrAppendAttribute("class", aValue); + return std::move(*this); + } +}; + +/// \ Element to group inline-elements in a document. +class Span : public Element { + public: + explicit Span(const std::string& aContent) : Element("span", aContent) {} +}; + +/// \ pre-formatted Element to display text in mono-space font. +class Pre : public Element { + public: + explicit Pre(const std::string& aContent) : Element("pre", aContent) {} +}; + +/// \ Hyper-Link Element +class Link : public Element { + public: + Link() : Element("a") {} + explicit Link(const char* apContent) : Element("a", apContent) {} + explicit Link(const char* apContent, const char* apUrl = nullptr) : Element("a", apContent) { + if (apUrl) { + addAttribute("href", apUrl); + } + } + Link(const std::string& aContent, const std::string& aUrl) : Element("a", aContent) { + if (!aUrl.empty()) { + addAttribute("href", aUrl); + } + } + Link&& target(const char* apValue) { + addAttribute("target", apValue); + return std::move(*this); + } +}; + +/// \ Image Element +class Image : public Element { + public: + Image(const std::string& aSrc, const std::string& aAlt, unsigned int aWidth = 0, unsigned int aHeight = 0) : + Element("img") { + addAttribute("src", aSrc); + addAttribute("alt", aAlt); + if (0 < aWidth) { + addAttribute("width", aWidth); + } + if (0 < aHeight) { + addAttribute("height", aHeight); + } + mbVoid = true; + } +}; + +/// \ Button Element +class Button : public Element { + public: + Button(const char* apContent, const char* apType = "button") : + Element("button", apContent) { + addAttribute("type", apType); + } +}; + +/// \ Element +class Progress : public Element { + public: + Progress(const unsigned int aValue, const unsigned int aMax) : Element("progress") { + addAttribute("value", aValue); + addAttribute("max", aMax); + } +}; + +/// \ gauge Element +class Meter : public Element { + public: + Meter(const unsigned int aValue, const unsigned int aMin, const unsigned int aMax) : Element("meter") { + addAttribute("value", aValue); + addAttribute("min", aMin); + addAttribute("max", aMax); + } +}; + +/// \ semantic Element +class Mark : public Element { + public: + explicit Mark(const std::string& aContent) : Element("mark", aContent) {} +}; + +/// \ semantic Element +class Time : public Element { + public: + explicit Time(const std::string& aContent, const std::string& aDateTime) : Element("time", aContent) { + addAttribute("datetime", aDateTime); + } +}; + +/// \ semantic Element +class Header : public Element { + public: + Header() : Element("header") {} +}; + +/// \ semantic Element +class Footer : public Element { + public: + Footer() : Element("footer") {} +}; + +/// \ semantic Element +class Section : public Element { + public: + Section() : Element("section") {} +}; + +/// \ semantic Element +class Article : public Element { + public: + Article() : Element("article") {} +}; + +/// \ semantic Element +class Nav : public Element { + public: + Nav() : Element("nav") {} + explicit Nav(const char* apClass) : Element("nav") { + cls(apClass); + } +}; + +/// \ semantic Element +class Aside : public Element { + public: + Aside() : Element("aside") {} +}; + +/// \ semantic Element +class Main : public Element { + public: + Main() : Element("main") {} +}; + +/// \ semantic Element +class Figure : public Element { + public: + Figure() : Element("figure") {} +}; + +/// \ semantic Element to use with Figure +class FigCaption : public Element { + public: + explicit FigCaption(const std::string& aContent) : Element("figcaption", aContent) {} +}; + +/** @brief \ semantic Element containing detailed information to use with Summary. +* +* @verbatim +
+ Copyright 2017-2021. +

By Sebastien Rombauts.

+

sebastien.rombauts@gmail.com.

+
@endverbatim +*/ +class Details : public Element { + public: + explicit Details(const char* apOpen = nullptr) : Element("details") { + if (apOpen) { + addAttribute("open", apOpen); + } + } +}; + +/// \ semantic Element to use inside a Details section to specify a visible heading +class Summary : public Element { + public: + explicit Summary(const std::string& aContent) : Element("summary", aContent) {} +}; + + +} // namespace HTML diff --git a/rir/src/utils/HTMLBuilder/HTML.h b/rir/src/utils/HTMLBuilder/HTML.h new file mode 100644 index 000000000..e6d989232 --- /dev/null +++ b/rir/src/utils/HTMLBuilder/HTML.h @@ -0,0 +1,19 @@ +/** +* @file HTML.h +* @ingroup HtmlBuilder +* @brief A simple C++ HTML Generator library. +* +* Copyright (c) 2017-2021 Sebastien Rombauts (sebastien.rombauts@gmail.com) +* +* Distributed under the MIT License (MIT) (See accompanying file LICENSE.txt +* or copy at http://opensource.org/licenses/MIT) +*/ +#pragma once + +/** +* @defgroup HtmlBuilder HtmlBuilder +* @brief A simple C++ header-only HTML Generator library, using a Document Object Model (DOM). +*/ + +#include "Element.h" +#include "Document.h" diff --git a/rir/src/utils/HTMLBuilder/escapeHtml.h b/rir/src/utils/HTMLBuilder/escapeHtml.h new file mode 100644 index 000000000..32ac4d332 --- /dev/null +++ b/rir/src/utils/HTMLBuilder/escapeHtml.h @@ -0,0 +1,34 @@ +// +// Created by Jakob Hain on 7/29/23. +// + +#pragma once + +#include + +static std::string escapeHtml(const std::string& s) { + std::string res; + res.reserve(s.size()); + for (auto c : s) { + switch (c) { + case '&': + res += "&"; + break; + case '\"': + res += """; + break; + case '\'': + res += "'"; + break; + case '<': + res += "<"; + break; + case '>': + res += ">"; + break; + default: + res += c; + } + } + return res; +} \ No newline at end of file diff --git a/rir/src/utils/Map.h b/rir/src/utils/Map.h index fda62f0ca..ed069dc55 100644 --- a/rir/src/utils/Map.h +++ b/rir/src/utils/Map.h @@ -66,6 +66,29 @@ class SmallMap { return end() - 1; } + void erase(const K& k) { + if (big) { + auto p = index.find(k); + if (p != index.end()) { + auto idx = p->second; + index.erase(p); + container[idx] = container.back(); + index[container[idx].first] = idx; + container.pop_back(); + return; + } + } else { + for (auto it = container.begin(), end = container.end(); it != end; ++it) { + if (it->first == k) { + *it = container.back(); + container.pop_back(); + return; + } + } + } + assert(false); + } + V& at(const K& k) { if (big) return container[index.at(k)].second; diff --git a/rir/src/utils/RangeSet.h b/rir/src/utils/RangeSet.h new file mode 100644 index 000000000..5463bbce5 --- /dev/null +++ b/rir/src/utils/RangeSet.h @@ -0,0 +1,239 @@ +//! Source: https://github.com/hl037/rangeset.hpp/blob/master/rangeset.hpp +#pragma once + +#include +#include +#include + +/** + * Range set ot type T. + * + * A range set is a set comprising zero or more nonempty, disconnected ranges of type T. + * + * This class supports adding and removing ranges from the set, and testing if a given object or range is contained in the set. + * + * @tparam T type of the contained range end points (anything with an absolute order defined) + * + * @tparam MERGE_TOUCHING if true (default) inserting [10, 20) then [20, 30) will merge both the range to [10;30). If set to false, both will live in the range set. To merge then, one would have to insert [19, 21) + */ +template +class RangeSet{ + private: + /** \internal + * Representation of an end_point (lower/upper bound) of a range. + * RangeSet should alternate lower and upper bounds. + */ + struct end_point_t{ + T v; + enum { + BEFORE=0, + LOWER=MERGE_TOUCHING ? 1 : 2, + UPPER=3-LOWER, + AFTER=2 + } dir; + bool operator<(const end_point_t & oth) const{ + return v == oth.v ? dir < oth.dir : v < oth.v; + } + bool operator ==(const end_point_t & oth) const{ + return v == oth.v && dir == oth.dir; + } + }; + + std::set data; + + public: + /** + * The iterator is bidirectionnal. Its dereferenced value is a std::pair. + */ + struct const_iterator{ + using difference_type = long; + using value_type = std::pair; + using pointer = const value_type *; + using reference = const value_type &; + using iterator_category = std::bidirectional_iterator_tag; + + using _sub = typename std::set::const_iterator; + + value_type val; + _sub lower; + _sub end; + protected: + inline void update(){ + if(lower != end){ + val = {lower->v, std::next(lower)->v}; + } + } + public: + inline const_iterator() : lower{} {} + inline const_iterator(const _sub & lower, const _sub & end) : lower{lower}, end{end}{ update(); } + + inline reference operator*() const { return val; } + inline pointer operator->() const { return &val; } + inline const_iterator & operator++() { ++++lower; update(); return *this; } + inline const_iterator operator++(int) { const_iterator res{*this}; ++*this; return res; } + inline const_iterator & operator--() { ----lower; update(); return *this; } + inline const_iterator operator--(int) { const_iterator res{*this}; --*this; return res; } + + inline bool operator==(const const_iterator & oth) const { return lower == oth.lower; } + inline bool operator!=(const const_iterator & oth) const { return !(*this == oth); } + }; + + /** + * Add the range [start, end) (or "[start; end[" in other notation) to the set. + * If overlap occurs, the ranges are merged. if STRICT_OVERLAP is False, [start, mid) and [mid, end) will be merged to [start, end). Else, they will coeexist. + */ + void insert(T start, T end){ + if(end <= start){ + return; + } + auto && upper = data.upper_bound({end, end_point_t::UPPER}); // end) < upper OR upper == end() + // At the container begining + if(upper == data.begin()){ // [start , end) < [ upper=begin(), end() ) + data.insert(data.begin(), {end, end_point_t::UPPER}); + data.insert(data.begin(), {start, end_point_t::LOWER}); + return; + } + + if(upper == data.end() or upper->dir == end_point_t::LOWER){ // ')' < end < '[' + if(std::prev(upper)->v != end){ // if not same value, insert, else just skip and take upper's precedent + data.insert(upper, {end, end_point_t::UPPER}); + } + --upper; + } + + auto && lower = data.upper_bound({start, end_point_t::LOWER});// [start < lower + + if((lower == upper || lower->dir == end_point_t::LOWER) && lower->v != start + && (lower == data.begin() || std::prev(lower)->dir == end_point_t::UPPER)){ + data.insert(lower, {start, end_point_t::LOWER}); + } + + if(lower != upper){ + data.erase(lower, upper); + } + } + + inline void insert(const std::pair & range){ + insert(range.first, range.second); + } + + void insert_all(const RangeSet & oth){ + for(const auto& r : oth){ + insert(r.first, r.second); + } + } + + /** + * Remove the interval [start, end) (or "[start; end[" in other notation) from the set. + */ + void remove(const T & start, const T & end){ + auto && lower = data.lower_bound({start, end_point_t::LOWER}); + // At the container end + if(lower == data.end()){ + return; //nothing to do... + } + + bool lower_inserted = false; + if(lower->dir == end_point_t::UPPER){ + if(lower->v == start){ + ++lower; + } + else{ + data.insert(lower, {start, end_point_t::UPPER}); + --lower; + lower_inserted = true; + } + } + + auto && upper = data.lower_bound({end, end_point_t::LOWER}); + + if(upper != data.end() && upper->dir == end_point_t::UPPER){ + if(upper->v == end){ + ++upper; + } + else{ + data.insert(upper, {end, end_point_t::LOWER}); + --upper; + } + } + + if(lower_inserted){ + ++lower; + } + if(lower != upper){ + data.erase(lower, upper); + } + } + + inline void remove(const std::pair & range){ + remove(range.first, range.second); + } + + /** + * Remove unit ranges from the set (could be faster than remove) + */ + inline void erase(const_iterator it_begin, const_iterator it_end){ + if(it_begin == end()){ + return; + } + data.erase(it_begin.lower, it_end.lower); + } + + inline void erase(const_iterator it){ + if(it == end()){ + return; + } + auto it2 = it.lower; + ++++it2; + data.erase(it.lower, it2); + } + + /** + * Find the unit range that contains a specific value. + * Returns end() if not v is not in the set. + */ + const_iterator find(const T & v) const { + auto && upper = data.upper_bound({v, end_point_t::AFTER}); // v < lower + if(upper == data.begin() || upper == data.end() || upper->dir == end_point_t::LOWER){ + return end(); + } + else { + return const_iterator(--upper, data.end()); + } + } + + /** + * Find the unit range that contains the sub range [start, end) (or [start; end[ ) + */ + const_iterator find(const T & start, const T & end) const { + auto && upper = data.upper_bound({start, end_point_t::AFTER}); // v < lower + if(upper == data.begin() || upper == data.end() || upper->dir == end_point_t::LOWER || upper->v < end){ + return end(); + } + else { + return const_iterator(--upper, data.end()); + } + } + inline const_iterator find(const std::pair & range) const { + return find(range.first, range.second); + } + + /** + * Return the number of unit range in the set (The number of iterator beetwin begin() and end()) + */ + inline size_t size() const { return data.size() / 2; } + + /** + * Return an iterator to the first unit range. When dereferencing an iterator, the value is a std::pair describing the interval [ res.first, res.end ) + */ + inline const_iterator begin() const { return const_iterator{data.begin(), data.end()}; } + /** + * Return a past-the-end iterator of this set. + */ + inline const_iterator end() const { return const_iterator{data.end(), data.end()}; } + + public: + RangeSet()=default; + ~RangeSet()=default; + +}; \ No newline at end of file diff --git a/rir/src/utils/Terminal.h b/rir/src/utils/Terminal.h index 9648a3f99..b7d51196c 100644 --- a/rir/src/utils/Terminal.h +++ b/rir/src/utils/Terminal.h @@ -2,6 +2,7 @@ #define RIR_TERMINAL_ #include +#include #include struct ConsoleColor { @@ -14,10 +15,36 @@ struct ConsoleColor { return false; } static void red(std::ostream& out) { out << "\033[1;31m"; } + static void green(std::ostream& out) { out << "\033[1;32m"; } static void yellow(std::ostream& out) { out << "\033[1;33m"; } static void blue(std::ostream& out) { out << "\033[1;34m"; } static void magenta(std::ostream& out) { out << "\033[1;35m"; } static void clear(std::ostream& out) { out << "\033[0m"; } }; +namespace console { + template struct SetColor { + friend std::ostream& operator<<(std::ostream& out, const SetColor& r) { + color(out); + return out; + } + }; + template struct WithColor { + std::string msg; + explicit WithColor(std::string msg) : msg(std::move(msg)) {} + + friend std::ostream& operator<<(std::ostream& out, const WithColor& r) { + out << SetColor{} << r.msg << SetColor{}; + return out; + } + }; + __attribute__((unused)) static const SetColor clear{}; + __attribute__((unused)) static WithColor with_red(std::string msg) { + return WithColor(std::move(msg)); + } + __attribute__((unused)) static WithColor with_green(std::string msg) { + return WithColor(std::move(msg)); + } +} // namespace console + #endif diff --git a/rir/src/utils/UUID.cpp b/rir/src/utils/UUID.cpp deleted file mode 100644 index 00e2e77bc..000000000 --- a/rir/src/utils/UUID.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "UUID.h" -#include "R/Serialize.h" - -#include - -namespace rir { - -static size_t nextUuid = 0; - -// Generates a random UUID -UUID UUID::random() { return UUID(++nextUuid); } - -UUID UUID::deserialize(SEXP refTable, R_inpstream_t inp) { - UUID uuid; - InBytes(inp, &uuid.uuid, sizeof(uuid.uuid)); - return uuid; -} - -void UUID::serialize(SEXP refTable, R_outpstream_t out) const { - OutBytes(out, &uuid, sizeof(uuid)); -} - -std::string UUID::str() { - std::ostringstream str; - str << uuid; - return str.str(); -} - -bool UUID::operator==(const UUID& other) const { return uuid == other.uuid; } - -} // namespace rir diff --git a/rir/src/utils/UUID.h b/rir/src/utils/UUID.h deleted file mode 100644 index 9ed9eedb8..000000000 --- a/rir/src/utils/UUID.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include - -#include - -namespace rir { - -class UUID { - size_t uuid; - - UUID() {} - explicit UUID(size_t v) : uuid(v) {} - - public: - // Generates a random UUID - static UUID random(); - static UUID deserialize(SEXP refTable, R_inpstream_t inp); - void serialize(SEXP refTable, R_outpstream_t out) const; - std::string str(); - - bool operator==(const UUID& other) const; - friend struct std::hash; -}; - -} // namespace rir - -namespace std { -template <> -struct hash { - std::size_t operator()(const rir::UUID& v) const { return v.uuid; } -}; -} // namespace std diff --git a/rir/src/utils/ctpl.h b/rir/src/utils/ctpl.h new file mode 100644 index 000000000..566fc3de7 --- /dev/null +++ b/rir/src/utils/ctpl.h @@ -0,0 +1,252 @@ +/********************************************************* +* +* Copyright (C) 2014 by Vitaliy Vitsentiy +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*********************************************************/ + + +#ifndef __ctpl_stl_thread_pool_H__ +#define __ctpl_stl_thread_pool_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +// thread pool to run user's functors with signature +// ret func(int id, other_params) +// where id is the index of the thread that runs the functor +// ret is some return type + + +namespace ctpl { + +namespace detail { +template +class Queue { + public: + bool push(T const & value) { + std::unique_lock lock(this->mutex); + this->q.push(value); + return true; + } + // deletes the retrieved element, do not use for non integral types + bool pop(T & v) { + std::unique_lock lock(this->mutex); + if (this->q.empty()) + return false; + v = this->q.front(); + this->q.pop(); + return true; + } + bool empty() { + std::unique_lock lock(this->mutex); + return this->q.empty(); + } + private: + std::queue q; + std::mutex mutex; +}; +} + + class thread_pool { + + public: + + thread_pool() { this->init(); } + explicit thread_pool(int nThreads) { this->init(); this->resize(nThreads); } + + // the destructor waits for all the functions in the queue to be finished + ~thread_pool() { + this->stop(true); + } + + // get the number of running threads in the pool + int size() { return static_cast(this->threads.size()); } + + // number of idle threads + int n_idle() { return this->nWaiting; } + std::thread & get_thread(int i) { return *this->threads[i]; } + + // change the number of threads in the pool + // should be called from one thread, otherwise be careful to not interleave, also with this->stop() + // nThreads must be >= 0 + void resize(int nThreads) { + if (!this->isStop && !this->isDone) { + int oldNThreads = static_cast(this->threads.size()); + if (oldNThreads <= nThreads) { // if the number of threads is increased + this->threads.resize(nThreads); + this->flags.resize(nThreads); + + for (int i = oldNThreads; i < nThreads; ++i) { + this->flags[i] = std::make_shared>(false); + this->set_thread(i); + } + } + else { // the number of threads is decreased + for (int i = oldNThreads - 1; i >= nThreads; --i) { + *this->flags[i] = true; // this thread will finish + this->threads[i]->detach(); + } + { + // stop the detached threads that were waiting + std::unique_lock lock(this->mutex); + this->cv.notify_all(); + } + this->threads.resize(nThreads); // safe to delete because the threads are detached + this->flags.resize(nThreads); // safe to delete because the threads have copies of shared_ptr of the flags, not originals + } + } + } + + // empty the queue + void clear_queue() { + std::function * _f; + while (this->q.pop(_f)) + delete _f; // empty the queue + } + + // pops a functional wrapper to the original function + std::function pop() { + std::function * _f = nullptr; + this->q.pop(_f); + std::unique_ptr> func(_f); // at return, delete the function even if an exception occurred + std::function f; + if (_f) + // cppcheck-suppress deallocuse + f = *_f; + return f; + } + + // wait for all computing threads to finish and stop all threads + // may be called asynchronously to not pause the calling thread while waiting + // if isWait == true, all the functions in the queue are run, otherwise the queue is cleared without running the functions + void stop(bool isWait = false) { + if (!isWait) { + if (this->isStop) + return; + this->isStop = true; + for (int i = 0, n = this->size(); i < n; ++i) { + *this->flags[i] = true; // command the threads to stop + } + this->clear_queue(); // empty the queue + } + else { + if (this->isDone || this->isStop) + return; + this->isDone = true; // give the waiting threads a command to finish + } + { + std::unique_lock lock(this->mutex); + this->cv.notify_all(); // stop all waiting threads + } + for (int i = 0; i < static_cast(this->threads.size()); ++i) { // wait for the computing threads to finish + if (this->threads[i]->joinable()) + this->threads[i]->join(); + } + // if there were no threads in the pool but some functors in the queue, the functors are not deleted by the threads + // therefore delete them here + this->clear_queue(); + this->threads.clear(); + this->flags.clear(); + } + + template + auto push(F && f, Rest&&... rest) ->std::future { + auto pck = std::make_shared>( + std::bind(std::forward(f), std::placeholders::_1, std::forward(rest)...) + ); + auto _f = new std::function([pck](int id) { + (*pck)(id); + }); + this->q.push(_f); + std::unique_lock lock(this->mutex); + this->cv.notify_one(); + return pck->get_future(); + } + + // run the user's function that excepts argument int - id of the running thread. returned value is templatized + // operator returns std::future, where the user can get the result and rethrow the catched exceptins + template + auto push(F && f) ->std::future { + auto pck = std::make_shared>(std::forward(f)); + auto _f = new std::function([pck](int id) { + (*pck)(id); + }); + this->q.push(_f); + std::unique_lock lock(this->mutex); + this->cv.notify_one(); + return pck->get_future(); + } + + + private: + + // deleted + thread_pool(const thread_pool &);// = delete; + thread_pool(thread_pool &&);// = delete; + thread_pool & operator=(const thread_pool &);// = delete; + thread_pool & operator=(thread_pool &&);// = delete; + + void set_thread(int i) { + std::shared_ptr> flag(this->flags[i]); // a copy of the shared ptr to the flag + auto f = [this, i, flag/* a copy of the shared ptr to the flag */]() { + std::atomic & _flag = *flag; + std::function * _f; + bool isPop = this->q.pop(_f); + while (true) { + while (isPop) { // if there is anything in the queue + std::unique_ptr> func(_f); // at return, delete the function even if an exception occurred + (*_f)(i); + if (_flag) + return; // the thread is wanted to stop, return even if the queue is not empty yet + else + isPop = this->q.pop(_f); + } + // the queue is empty here, wait for the next command + std::unique_lock lock(this->mutex); + ++this->nWaiting; + this->cv.wait(lock, [this, &_f, &isPop, &_flag](){ isPop = this->q.pop(_f); return isPop || this->isDone || _flag; }); + --this->nWaiting; + if (!isPop) + return; // if the queue is empty and this->isDone == true or *flag then return + } + }; + this->threads[i].reset(new std::thread(f)); // compiler may not support std::make_unique() + } + + void init() { this->nWaiting = 0; this->isStop = false; this->isDone = false; } + + std::vector> threads; + std::vector>> flags; + detail::Queue *> q; + std::atomic isDone; + std::atomic isStop; + std::atomic nWaiting; // how many threads are waiting + + std::mutex mutex; + std::condition_variable cv; +}; + +} + +#endif // __ctpl_stl_thread_pool_H__ diff --git a/rir/src/utils/measuring.cpp b/rir/src/utils/measuring.cpp index 50ef9680d..154362df6 100644 --- a/rir/src/utils/measuring.cpp +++ b/rir/src/utils/measuring.cpp @@ -7,27 +7,49 @@ #include #include #include +#include +#include "R/Printing.h" +#include "RangeSet.h" +#include "runtime/Code.h" +#include "runtime/DispatchTable.h" +#include "runtime/Function.h" #include "utils/measuring.h" namespace rir { +using TimePoint = std::chrono::time_point; +using Duration = std::chrono::duration; + +struct Measuring::TimingEvent { + const std::string& name; + TimePoint start; +}; + namespace { struct MeasuringImpl { + struct TimedEvent { + TimePoint start; + TimePoint end; + }; struct Timer { + bool canNest = false; double timer = 0; bool timerActive = false; - std::chrono::time_point start; + TimePoint start; size_t alreadyRunning = 0; size_t notStarted = 0; }; + std::unordered_map>> timedEvents; std::unordered_map timers; std::unordered_map events; - std::chrono::time_point start; - std::chrono::time_point end; + std::unordered_map associatedLatestDumps; + TimePoint start; + TimePoint end; size_t threshold = 0; const unsigned width = 40; + const unsigned maxTimedEventsToPrint = 1000; bool shouldOutput = false; MeasuringImpl() : start(std::chrono::high_resolution_clock::now()) {} @@ -49,13 +71,28 @@ struct MeasuringImpl { } } + void updateAssociatedDump(SEXP associated, bool associatedIsInitialized) { +#ifdef DUMP_MEASURE_ASSOCIATEDS + std::stringstream s; + if (!associatedIsInitialized) { + s << "(not yet initialized)\n"; + } else { + printRirObject(associated, s); + } + std::string str = s.str(); + if (!str.empty()) { + associatedLatestDumps[associated] = str; + } +#endif + } + void dump(std::ostream& out) { if (!shouldOutput) return; - std::chrono::duration duration = end - start; + std::chrono::duration totalLifetime = end - start; out << "\n---== Measuring breakdown ===---\n\n"; - out << " Total lifetime: " << format(duration.count()) << "\n\n"; + out << " Total lifetime: " << format(totalLifetime.count()) << "\n\n"; { std::map> @@ -67,7 +104,7 @@ struct MeasuringImpl { key += 1e-20; double notStopped = 0; if (t.second.timerActive) { - duration = end - t.second.start; + Duration duration = end - t.second.start; notStopped = duration.count(); } orderedTimers.emplace( @@ -78,7 +115,7 @@ struct MeasuringImpl { if (!orderedTimers.empty()) { out << " Timers (" << format(totalTimers) << " in total, or " << std::setprecision(2) - << (totalTimers / duration.count() * 100) << "%):\n"; + << (totalTimers / totalLifetime.count() * 100) << "%):\n"; for (auto& t : orderedTimers) { auto& name = std::get<0>(t.second); out << " " << std::setw(width) << name << "\t" @@ -120,12 +157,125 @@ struct MeasuringImpl { } } + { + std::map> + timedEventSuperSumsOrderedByDuration; + std::map> + timedEventSumsOrderedByDuration; + std::map> + timedEventsOrderedChronologically; + RangeSet totalTimedEventsRange; + size_t totalTimedEventsCount = 0; + for (auto& a : timedEvents) { + auto& name = a.first; + RangeSet superRange; + size_t superCount = 0; + for (auto& b : a.second) { + auto& associated = b.first; + RangeSet range; + for (auto& e : b.second) { + auto duration = e.end - e.start; + timedEventsOrderedChronologically.emplace( + e.start, + std::make_tuple(name, associated, duration)); + range.insert(e.start, e.end); + } + auto sum = sumRangeSet(range); + timedEventSumsOrderedByDuration.emplace( + sum, std::make_tuple(name, associated, b.second.size())); + superRange.insert_all(range); + superCount += b.second.size(); + } + auto superSum = sumRangeSet(superRange); + timedEventSuperSumsOrderedByDuration.emplace( + superSum, std::make_tuple(name, superCount)); + totalTimedEventsRange.insert_all(superRange); + totalTimedEventsCount += superCount; + } + if (!timedEventsOrderedChronologically.empty()) { + auto totalTimedEventsDuration = sumRangeSet(totalTimedEventsRange); + out << " Timed events (total count = " << totalTimedEventsCount << ", time = " + << format(totalTimedEventsDuration) << ", ratio to total lifetime = " + << std::setprecision(2) + << (totalTimedEventsDuration.count() / totalLifetime.count() * 100) << "%):\n"; + out << " Super sums ordered by duration:\n"; + size_t totalCount = 0; + for (auto it = timedEventSuperSumsOrderedByDuration.rbegin(); + it != timedEventSuperSumsOrderedByDuration.rend(); ++it) { + auto& t = *it; + auto& name = std::get<0>(t.second); + auto count = std::get<1>(t.second); + auto duration = t.first; + out << " " << std::setw((int)width) << name << "\t" + << count << "\t" << format(duration); + out << "\n"; + if (totalCount++ > maxTimedEventsToPrint) { + out << " ... (omitted)\n"; + break; + } + } + out << " Sums ordered by duration:\n"; + std::unordered_set printedAssociateds; + totalCount = 0; + for (auto it = timedEventSumsOrderedByDuration.rbegin(); + it != timedEventSumsOrderedByDuration.rend(); ++it) { + auto& t = *it; + auto& name = std::get<0>(t.second); + auto& associated = std::get<1>(t.second); + auto count = std::get<2>(t.second); + auto duration = t.first; + out << " " << std::setw((int)width) << name << "\t" + << associated << "\t" << count << "\t" + << format(duration); + out << "\n"; + printedAssociateds.insert(associated); + if (totalCount++ > maxTimedEventsToPrint) { + out << " ... (omitted)\n"; + break; + } + } + out << " All ordered chronologically:\n"; + totalCount = 0; + for (auto& t : timedEventsOrderedChronologically) { + auto& name = std::get<0>(t.second); + auto& associated = std::get<1>(t.second); + auto duration = std::get<2>(t.second); + out << " " << std::setw((int)width) << name << "\t" + << associated << "\t" << format(duration); + out << "\n"; + printedAssociateds.insert(associated); + if (totalCount++ > maxTimedEventsToPrint) { + out << " ... (omitted)\n"; + break; + } + } + out << " Associated latest dumps:\n"; + for (auto& a : printedAssociateds) { + if (associatedLatestDumps.count(a)) { + out << " " << std::setw((int)width) << a; + out << "\n" << associatedLatestDumps.at(a) << "\n"; + } + } + out << "\n"; + } + } + out << std::flush; } + static std::string format(Duration secs) { + return format(secs.count()); + } + static std::string format(double secs) { std::stringstream ss; - if (secs < 60) + if (secs < 0.000001) + ss << secs * 1000 * 1000 * 1000 << " ns"; + else if (secs < 0.001) + ss << secs * 1000 * 1000 << " µs"; + else if (secs < 1) + ss << secs * 1000 << " ms"; + else if (secs < 60) ss << secs << " secs"; else if (secs < 60 * 60) ss << secs / 60 << " min"; @@ -148,15 +298,50 @@ struct MeasuringImpl { ss << n; return ss.str(); } + + static Duration sumRangeSet(const RangeSet& set) { + auto sum = Duration::zero(); + for (auto& r : set) { + sum += r.second - r.first; + } + return sum; + } }; } // namespace std::unique_ptr m = std::make_unique(); -void Measuring::startTimer(const std::string& name) { +Measuring::TimingEvent* Measuring::startTimingEvent(const std::string& name) { + startTimer(name, true); m->shouldOutput = true; + auto start = std::chrono::high_resolution_clock::now(); + return new Measuring::TimingEvent{name, start}; +} + +void Measuring::stopTimingEvent(rir::Measuring::TimingEvent* timing, + SEXP associated, + bool associatedIsInitialized) { + assert(timing); + countTimer(timing->name, true); + m->updateAssociatedDump(associated, associatedIsInitialized); + auto end = std::chrono::high_resolution_clock::now(); + MeasuringImpl::TimedEvent timed{timing->start, end}; + m->timedEvents[timing->name][associated].push_back(timed); + delete timing; +} + +void Measuring::startTimer(const std::string& name, bool canNest) { + m->shouldOutput = true; + + auto isNewTimer = !m->timers.count(name); auto& t = m->timers[name]; + if (isNewTimer) { + t.canNest = canNest; + } else { + assert(t.canNest == canNest && "canNest must be consistent with timer of the same name"); + } + if (t.timerActive) { t.alreadyRunning++; } else { @@ -165,12 +350,23 @@ void Measuring::startTimer(const std::string& name) { } } -void Measuring::countTimer(const std::string& name) { +void Measuring::countTimer(const std::string& name, bool canNest) { auto end = std::chrono::high_resolution_clock::now(); m->shouldOutput = true; + + auto isNewTimer = !m->timers.count(name); auto& t = m->timers[name]; + if (isNewTimer) { + t.canNest = canNest; + } else { + assert(t.canNest == canNest && + "canNest must be consistent with timer of the same name"); + } + if (!t.timerActive) { t.notStarted++; + } else if (canNest && t.alreadyRunning > 0) { + t.alreadyRunning--; } else { t.timerActive = false; std::chrono::duration duration = end - t.start; diff --git a/rir/src/utils/measuring.h b/rir/src/utils/measuring.h index cb00b3eb2..40a63638f 100644 --- a/rir/src/utils/measuring.h +++ b/rir/src/utils/measuring.h @@ -1,17 +1,93 @@ #ifndef MEASURING_H #define MEASURING_H +#include "R/r_incl.h" #include namespace rir { class Measuring { + struct TimingEvent; + + static TimingEvent* startTimingEvent(const std::string& name); + static void stopTimingEvent(TimingEvent* timing, SEXP associated, + bool associatedIsInitialized); public: - static void startTimer(const std::string& name); - static void countTimer(const std::string& name); + template static ALWAYS_INLINE SEXP + timeEventIf3(bool cond, const std::string& name, F code, + F2 associatedIsInitialized) { + auto timing = cond ? startTimingEvent(name) : nullptr; + auto associated = code(); + if (timing) { + PROTECT(associated); + auto isInitialized = associatedIsInitialized(associated); + stopTimingEvent(timing, associated, isInitialized); + UNPROTECT(1); + } + return associated; + } + template static ALWAYS_INLINE SEXP + timeEventIf3(bool cond, const std::string& name, F code) { + auto timing = cond ? startTimingEvent(name) : nullptr; + auto associated = code(); + if (timing) { + PROTECT(associated); + stopTimingEvent(timing, associated, true); + UNPROTECT(1); + } + return associated; + } + template static ALWAYS_INLINE SEXP + timeEventIf2(bool cond, const std::string& name, SEXP associated, + bool associatedWillBeInitialized, F code) { + PROTECT(associated); + auto timing = cond ? startTimingEvent(name) : nullptr; + auto result = code(); + if (cond) { + stopTimingEvent(timing, associated, associatedWillBeInitialized); + } + UNPROTECT(1); + return result; + } + template static ALWAYS_INLINE SEXP + timeEventIf2(bool cond, const std::string& name, SEXP associated, F code) { + return timeEventIf2(cond, name, associated, true, code); + } + template static ALWAYS_INLINE void + timeEventIf(bool cond, const std::string& name, SEXP associated, + bool associatedWillBeInitialized, F code) { + PROTECT(associated); + auto timing = cond ? startTimingEvent(name) : nullptr; + code(); + if (timing) { + stopTimingEvent(timing, associated, associatedWillBeInitialized); + } + UNPROTECT(1); + } + template static ALWAYS_INLINE void + timeEventIf(bool cond, const std::string& name, SEXP associated, F code) { + timeEventIf(cond, name, associated, true, code); + } + + static void startTimer(const std::string& name, bool canNest = false); + static void countTimer(const std::string& name, bool canNest = false); static void addTime(const std::string& name, double time); + static inline void startTimerIf(bool cond, const std::string& name, + bool canNest = false) { + if (cond) { + startTimer(name, canNest); + } + } + static inline void countTimerIf(bool cond, const std::string& name, + bool canNest = false) { + if (cond) { + countTimer(name, canNest); + } + } + static void setEventThreshold(size_t n); static void countEvent(const std::string& name, size_t n = 1); + static void reset(bool outputOld = false); }; diff --git a/rir/tests/macos-llvm-regression.R b/rir/tests/macos-llvm-regression.R new file mode 100644 index 000000000..562ca99d6 --- /dev/null +++ b/rir/tests/macos-llvm-regression.R @@ -0,0 +1,18 @@ +f <- function() { + gc(FALSE) + 1 +} + +for (i in 1:14) + print(f()) + +# Above fails SOMETIMES +# Below code fails ALWAYS + +f <- function(expr) { + gc(FALSE) + expr +} + +for (i in 1:5) + print(f(print(1))) \ No newline at end of file diff --git a/rir/tests/pir_check.R b/rir/tests/pir_check.R index f160ce659..0068a338b 100644 --- a/rir/tests/pir_check.R +++ b/rir/tests/pir_check.R @@ -1,12 +1,7 @@ -jitOn <- as.numeric(Sys.getenv("R_ENABLE_JIT", unset=2)) != 0 -jitOn <- jitOn && (Sys.getenv("PIR_ENABLE", unset="on") == "on") - -if (!jitOn) +jitOn <- as.numeric(Sys.getenv("R_ENABLE_JIT", unset=2)) == 0 && (Sys.getenv("PIR_ENABLE", unset="on") == "on") +if (!jitOn || Sys.getenv("PIR_GLOBAL_SPECIALIZATION_LEVEL") != "") quit() -if (Sys.getenv("PIR_GLOBAL_SPECIALIZATION_LEVEL") != "") - q() - # Sanity check for loop peeling, and testing that enabling/disabling works # These loop peeling tests may be a bit brittle. # Loop peeling should be enabled by default diff --git a/rir/tests/regression_intern_llvm_grid.R b/rir/tests/regression_intern_llvm_grid.R new file mode 100644 index 000000000..6a9e135d5 --- /dev/null +++ b/rir/tests/regression_intern_llvm_grid.R @@ -0,0 +1,31 @@ +pkgname <- "grid" +source(file.path(R.home("share"), "R", "examples-header.R")) +options(warn = 1) +library('grid') + +base::assign(".oldSearch", base::search(), pos = 'CheckExEnv') +base::assign(".old_wd", base::getwd(), pos = 'CheckExEnv') +cleanEx() +nameEx("Grid") +### * Grid + +flush(stderr()); flush(stdout()) + +### Name: Grid +### Title: Grid Graphics +### Aliases: Grid +### Keywords: dplot + +### ** Examples + +## Diagram of a simple layout +grid.show.layout(grid.layout(4,2, + heights=unit(rep(1, 4), + c("lines", "lines", "lines", "null")), + widths=unit(c(1, 1), "inches"))) +## Diagram of a sample viewport +grid.show.viewport(viewport(x=0.6, y=0.6, + width=unit(1, "inches"), height=unit(1, "inches"))) +## A flash plotting example +grid.multipanel(vp=viewport(0.5, 0.5, 0.8, 0.8)) +# R: /opt/rir/rir/src/runtime/TypeFeedback.cpp:146: void rir::TypeFeedback::addConnected(rir::ConnectedCollectorOld&) const: Assertion `false && "Feedback should never be hashed (don't call addConnected)"' failed. diff --git a/rir/tests/regression_intern_reg_s4.R b/rir/tests/regression_intern_reg_s4.R new file mode 100644 index 000000000..0cc5229e6 --- /dev/null +++ b/rir/tests/regression_intern_reg_s4.R @@ -0,0 +1,262 @@ +####--- S4 Methods (and Classes) --- see also ../src/library/methods/tests/ + +#### Instead of adding more tests depending on recommended packages, +#### re-facror into a separate script and treat like eval-etc-2.R + +options(useFancyQuotes=FALSE) +require(methods) +assertError <- tools::assertError # "import" +##too fragile: showMethods(where = "package:methods") + +## When this test comes too late, it failed too early in R <= 3.2.2 +require(stats4) +detach("package:methods") +require("methods") +cc <- methods::getClassDef("standardGeneric") +cc ## (auto) print failed here, in R <= 3.2.2 +stopifnot(.isMethodsDispatchOn()) ## was FALSE in R <= 3.2.2 + + +## Needs cached primitive generic for '$' +new("envRefClass")# failed in R <= 3.2.0 + +##-- S4 classes with S3 slots [moved from ./reg-tests-1.R] +setClass("test1", representation(date="POSIXct")) +x <- new("test1", date=as.POSIXct("2003-10-09")) +stopifnot(format(x @ date) == "2003-10-09") +## line 2 failed in 1.8.0 because of an extraneous space in "%in%" + +stopifnot(all.equal(3:3, 3.), all.equal(1., 1:1)) + +## trace (requiring methods): +f <- function(x, y) { c(x,y)} +xy <- 0 +trace(f, quote(x <- c(1, x)), exit = quote(xy <<- x), print = FALSE) +fxy <- f(2,3) +stopifnot(identical(fxy, c(1,2,3))) +stopifnot(identical(xy, c(1,2))) +untrace(f) + +## a generic and its methods + +setGeneric("f") +setMethod("f", c("character", "character"), function(x, y) paste(x,y)) + +## trace the generic +trace("f", quote(x <- c("A", x)), exit = quote(xy <<- c(x, "Z")), print = FALSE) + +## should work for any method + +stopifnot(identical(f(4,5), c("A",4,5)), + identical(xy, c("A", 4, "Z"))) + +stopifnot(identical(f("B", "C"), paste(c("A","B"), "C")), + identical(xy, c("A", "B", "Z"))) + +## trace a method +trace("f", sig = c("character", "character"), quote(x <- c(x, "D")), + exit = quote(xy <<- xyy <<- c(x, "W")), print = FALSE) + +stopifnot(identical(f("B", "C"), paste(c("A","B","D"), "C"))) +stopifnot(identical(xyy, c("A", "B", "D", "W"))) +# got broken by Luke's lexical scoping fix: +#stopifnot(identical(xy, xyy)) + +## but the default method is unchanged +stopifnot(identical(f(4,5), c("A",4,5)), + identical(xy, c("A", 4, "Z"))) + +removeGeneric("f") +## end of moved from trace.Rd + + +## print/show dispatch [moved from ./reg-tests-2.R ] +## The results have waffled back and forth. +## Currently (R 2.4.0) the intent is that automatic printing of S4 +## objects should correspond to a call to show(), as per the green +## book, p. 332. Therefore, the show() method is called, once defined, +## for auto-printing foo, regardless of the S3 or S4 print() method. +## (But most of this example is irrelevant if one avoids S3 methods for +## S4 classes, as one should.) +setClass("bar", representation(a="numeric")) +foo <- new("bar", a=pi) +foo +show(foo) +print(foo) + +setMethod("show", "bar", function(object){cat("show method\n")}) +show(foo) +foo +print(foo) +# suppressed because output depends on current choice of S4 type or +# not. Can reinstate when S4 type is obligatory +# print(foo, digits = 4) + +## DON'T DO THIS: S3 methods for S4 classes are a design error JMC iii.9.09 +## print.bar <- function(x, ...) cat("print method\n") +## foo +## print(foo) +## show(foo) + +setMethod("print", "bar", function(x, ...){cat("S4 print method\n")}) +foo +print(foo) +show(foo) +## calling print() with more than one argument suppresses the show() +## method, largely to prevent an infinite loop if there is in fact no +## show() method for this class. A better solution would be desirable. +print(foo, digits = 4) + +cn <- "integer or NULL" +setClassUnion(cn, members = c("integer", "NULL")) +setClass("c1", representation(x = "integer", code = cn)) +stopifnot(exprs = { + cn %in% extends(getClass("NULL")) + cn %in% extends(getClass(".NULL")) + cn %in% extends(getClass("integer")) +}) +nc <- new("c1", x = 1:2) +str(nc)# gave ^ANULL^A in 2.0.0 +## + +showMethods("coerce", classes=c("matrix", "numeric")) +## {gave wrong result for a while in R 2.4.0} + +## Most for "mle" in stats4: +for(f in c("coef", "confint", "logLik", "plot", "profile", + "show", "summary", "update", "vcov")) + if(!hasMethods(f)) stop("no S4 methods found for ", f) + + +##--- "[" fiasco before R 2.2.0 : +d2 <- data.frame(b= I(matrix(1:6,3,2))) +## all is well: +d2[2,] +stopifnot(identical(d2[-1,], d2[2:3,])) +## Now make "[" into S4 generic by defining a trivial method +setClass("Mat", representation(Dim = "integer", "VIRTUAL")) +setMethod("[", signature(x = "Mat", + i = "missing", j = "missing", drop = "ANY"), + function (x, i, j, drop) x) +## Can even remove the method: it doesn't help +removeMethod("[", signature(x = "Mat", + i = "missing", j = "missing", drop = "ANY")) +d2[1:2,] ## used to fail badly; now okay +stopifnot(identical(d2[-1,], d2[2:3,])) +## failed in R <= 2.1.x + + +## Fritz' S4 "odditiy" +setClass("X", representation(bar="numeric")) +setClass("Y", contains="X") +## Now we define a generic foo() and two different methods for "X" and +## "Y" objects for arg missing: +setGeneric("foo", function(object, arg) standardGeneric("foo")) +setMethod("foo", signature(object= "X", arg="missing"), + function(object, arg) cat("an X object with bar =", object@bar, "\n")) +setMethod("foo", signature(object= "Y", arg="missing"), + function(object, arg) cat("a Y object with bar =", object@bar, "\n")) +## Finally we create a method where arg is "logical" only for class +## "X", hence class "Y" should inherit that: +setMethod("foo", signature(object= "X", arg= "logical"), + function(object, arg) cat("Hello World!\n") ) +## now create objects and call methods: +y <- new("Y", bar=2) +## showMethods("foo") +foo(y) +foo(y, arg=TRUE)## Hello World! +## OK, inheritance worked, and we have +## showMethods("foo") +foo(y) +## still 'Y' -- was 'X object' in R < 2.3 + + +## Multiple inheritance +setClass("A", representation(x = "numeric")) +setClass("B", representation(y = "character")) +setClass("C", contains = c("A", "B"), representation(z = "logical")) +new("C") +setClass("C", contains = c("A", "B"), representation(z = "logical"), + prototype = prototype(x = 1.5, y = "test", z = TRUE)) +(cc <- new("C")) +## failed reconcilePropertiesAndPrototype(..) after svn r37018 +stopifnot(identical(selectSuperClasses("C", dropVirtual = TRUE), c("A", "B")), + 0 == length(.selectSuperClasses(getClass("B")@contains))) + +## "Logic" group -- was missing in R <= 2.4.0 +stopifnot(all(getGroupMembers("Logic") %in% c("&", "|")), + any(getGroupMembers("Ops") == "Logic")) +setClass("brob", contains="numeric") +b <- new("brob", 3.14) +logic.brob.error <- function(nm) + stop("logic operator '", nm, "' not applicable to brobs") +logic2 <- function(e1,e2) logic.brob.error(.Generic) +setMethod("Logic", signature("brob", "ANY"), logic2) +setMethod("Logic", signature("ANY", "brob"), logic2) +## Now ensure that using group members gives error: +assertError(b & b) +assertError(b | 1) +assertError(TRUE & b) + + +## methods' hidden cbind() / rbind: +setClass("myMat", representation(x = "numeric")) +setMethod("cbind2", signature(x = "myMat", y = "missing"), function(x,y) x) +m <- new("myMat", x = c(1, pi)) +stopifnot(identical(m, methods:::cbind(m)), identical(m, cbind(m))) + + +## explicit print or show on a basic class with an S4 bit +## caused infinite recursion +setClass("Foo", representation(name="character"), contains="matrix") +(f <- new("Foo", name="Sam", matrix())) +f2 <- new("Foo", .Data = diag(2), name="Diag")# explicit .Data +(m <- as(f, "matrix")) +## this has no longer (2.7.0) an S4 bit: set it explicitly just for testing: +stopifnot(isS4(m. <- asS4(m)), + identical(m, f@.Data), + .hasSlot(f, "name"))# failed in R <= 2.13.1 +show(m.) +print(m.) +## fixed in 2.5.0 patched + +## callGeneric inside a method with new arguments {hence using .local()}: +setGeneric("Gfun", function(x, ...) standardGeneric("Gfun"), + useAsDefault = function(x, ...) sum(x, ...)) +setClass("myMat", contains="matrix") +setClass("mmat2", contains="matrix") +setClass("mmat3", contains="mmat2") +setMethod(Gfun, signature(x = "myMat"), + function(x, extrarg = TRUE) { + cat("in 'myMat' method for 'Gfun() : extrarg=", extrarg, "\n") + Gfun(unclass(x)) + }) +setMethod(Gfun, signature(x = "mmat2"), + function(x, extrarg = TRUE) { + cat("in 'mmat2' method for 'Gfun() : extrarg=", extrarg, "\n") + x <- unclass(x) + callGeneric() + }) +setMethod(Gfun, signature(x = "mmat3"), + function(x, extrarg = TRUE) { + cat("in 'mmat3' method for 'Gfun() : extrarg=", extrarg, "\n") + x <- as(x, "mmat2") + callGeneric() + }) +wrapG <- function(x, a1, a2) { + myextra <- missing(a1) && missing(a2) + Gfun(x, extrarg = myextra) +} + +(mm <- new("myMat", diag(3))) +Gfun(mm) +stopifnot(identical(wrapG(mm), Gfun(mm, TRUE)), + identical(wrapG(mm,,2), Gfun(mm, FALSE))) + +Gfun(mm, extrarg = FALSE) +m2 <- new("mmat2", diag(3)) +Gfun(m2) +Gfun(m2, extrarg = FALSE) +## The last two gave Error ...... variable ".local" was not found +(m3 <- new("mmat3", diag(3))) +Gfun(m3) diff --git a/rir/tests/regression_reg-packages.R b/rir/tests/regression_reg-packages.R index b84b84659..cada2e181 100644 --- a/rir/tests/regression_reg-packages.R +++ b/rir/tests/regression_reg-packages.R @@ -1,3 +1,8 @@ +# Serialization can cause namespaces to get attached multiple times, because serializing +# and deserializing namespaces affects them (TODO: investigate) +if (Sys.getenv("RIR_SERIALIZE_CHAOS") != "") + q() + unlockBinding(".make_numeric_version", .BaseNamespaceEnv) .BaseNamespaceEnv$.make_numeric_version <- function(x, strict = TRUE, regexp, classes = NULL) diff --git a/rir/tests/test_mark_function.r b/rir/tests/test_mark_function.r index fec672cd2..d618a68e7 100644 --- a/rir/tests/test_mark_function.r +++ b/rir/tests/test_mark_function.r @@ -1,4 +1,4 @@ -if (Sys.getenv("R_ENABLE_JIT") == 0 || Sys.getenv("PIR_ENABLE") == "force" || Sys.getenv("PIR_ENABLE") == "off" || Sys.getenv("RIR_SERIALIZE_CHAOS") == "1" || Sys.getenv("PIR_GLOBAL_SPECIALIZATION_LEVEL") != "") +if (Sys.getenv("R_ENABLE_JIT") == 0 || Sys.getenv("PIR_ENABLE") == "force" || Sys.getenv("PIR_ENABLE") == "off" || Sys.getenv("RIR_SERIALIZE_CHAOS") > 0 || Sys.getenv("PIR_GLOBAL_SPECIALIZATION_LEVEL") != "" || Sys.getenv("PIR_CLIENT_ADDR") != "") quit() add_noinline1 <- rir.compile(function(a,b) a+b) diff --git a/tools/R b/tools/R index e7de0fdeb..c964755b4 100755 --- a/tools/R +++ b/tools/R @@ -13,5 +13,6 @@ PKG="$SCRIPTPATH/../rir/" export EXTRA_LOAD_SO="`ls $RIR_BUILD/librir.*`" export EXTRA_LOAD_R="$PKG/R/rir.R" +export PIR_PRETTY_GRAPH_DEPENDENCY_LOCATION="$SCRIPTPATH/rirPrettyGraph" $R_HOME/bin/`basename "$0"` "$@" diff --git a/tools/build-gnur.sh b/tools/build-gnur.sh index 8cb64c401..5bfb48eb9 100755 --- a/tools/build-gnur.sh +++ b/tools/build-gnur.sh @@ -65,7 +65,7 @@ function build_r { echo "-> configure $NAME" cd $R_DIR if [ $USING_OSX -eq 1 ]; then - CFLAGS="-O2 -g -DSWITCH_TO_NAMED=$SND" ./configure --enable-R-shlib --with-internal-tzcode --with-ICU=no || cat config.log + CFLAGS="-O2 -g -DSWITCH_TO_NAMED=$SND -I/opt/homebrew/include" LDFLAGS="-L/opt/homebrew/lib" ./configure --enable-R-shlib --with-internal-tzcode --with-ICU=no --with-x=no --with-aqua=no || cat config.log else CFLAGS="-O2 -g -DSWITCH_TO_NAMED=$SND" ./configure fi @@ -100,7 +100,12 @@ function build_r { fi echo "-> building $NAME" - make -j8 + if [ $USING_OSX -eq 1 ]; then + # We need `C_INCLUDE_PATH` here AND we need to include `/opt/homebrew/bin` in `./configure` + C_INCLUDE_PATH=/opt/homebrew/include make -j8 + else + make -j8 + fi } build_r custom-r diff --git a/tools/build-llvm.sh b/tools/build-llvm.sh index ad167b730..00461bd40 100755 --- a/tools/build-llvm.sh +++ b/tools/build-llvm.sh @@ -11,25 +11,25 @@ fi SRC_DIR=`cd ${SCRIPTPATH}/.. && pwd` . "${SCRIPTPATH}/script_include.sh" -if [ -d "${SRC_DIR}/external/llvm-8.0.0" ]; then - echo "${SRC_DIR}/external/llvm-8.0.0 already exists. Remove it to install a debug version of llvm." +if [ -d "${SRC_DIR}/external/llvm-12" ]; then + echo "${SRC_DIR}/external/llvm-12 already exists. Remove it to install a debug version of llvm." exit 1 fi cd "${SRC_DIR}/external" -if [ ! -f llvm-8.0.0.src.tar.xz ]; then - wget http://releases.llvm.org/8.0.0/llvm-8.0.0.src.tar.xz +if [ ! -f llvm-12.0.0.src.tar.xz ]; then + wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/llvm-12.0.0.src.tar.xz fi -if [ ! -d "llvm-8.0.0.src" ]; then - tar xf llvm-8.0.0.src.tar.xz - mkdir llvm-8.0.0 - cd llvm-8.0.0.src +if [ ! -d "llvm-12.0.0.src" ]; then + tar xf llvm-12.0.0.src.tar.xz + mkdir llvm-12 + cd llvm-12.0.0.src mkdir build cd build cmake -DCMAKE_BUILD_TYPE=Debug -GNinja .. ninja else - cd llvm-8.0.0.src/build + cd llvm-12.0.0.src/build fi -cmake -DCMAKE_INSTALL_PREFIX=../../llvm-8.0.0 -P cmake_install.cmake +cmake -DCMAKE_INSTALL_PREFIX=../../llvm-12 -P cmake_install.cmake diff --git a/tools/build-zeromq.sh b/tools/build-zeromq.sh new file mode 100755 index 000000000..b0a0252c8 --- /dev/null +++ b/tools/build-zeromq.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -e + +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +if [ ! -d "$SCRIPTPATH" ]; then + echo "Could not determine absolute dir of $0" + exit 1 +fi +. "${SCRIPTPATH}/script_include.sh" +SRC_DIR="${SCRIPTPATH}/.." +EXTERNAL_DIR="${SRC_DIR}/external" + +if [ -d "${EXTERNAL_DIR}/zeromq" ] && [ -n "${FORCE}" ]; then + echo "-> removing old zeromq build..." + rm -rf "${EXTERNAL_DIR}/zeromq" "${EXTERNAL_DIR}/zeromq-4.3.4" "${EXTERNAL_DIR}/cppzmq-4.9.0" +fi + +if [ ! -d "${EXTERNAL_DIR}/zeromq" ]; then + echo "-> building libzmq..." + # https://github.com/... is the path to the libzmq source release + wget -qO- https://github.com/zeromq/libzmq/releases/download/v4.3.4/zeromq-4.3.4.tar.gz | tar -xz -C "${EXTERNAL_DIR}" + cd "${EXTERNAL_DIR}/zeromq-4.3.4" + # --disable-Werror must be passed because of https://github.com/zeromq/libzmq/issues/4391, which is still open :( + ./configure --prefix="${EXTERNAL_DIR}/zeromq" --enable-debug --disable-Werror && make -j "$(ncores)" && make install + echo "-> building cppzmq" + # https://github.com/... is the path to the cppzmq source release + wget -qO- https://github.com/zeromq/cppzmq/archive/refs/tags/v4.9.0.tar.gz | tar -xz -C "${EXTERNAL_DIR}" + cd "${EXTERNAL_DIR}/cppzmq-4.9.0" + if [ "${USE_NINJA}" -eq 0 ]; then + GNINJA="" + else + GNINJA="-GNinja" + fi + # TODO: Switch to release if CMake is in a release configuration + cmake "${GNINJA}" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="${EXTERNAL_DIR}/zeromq" -B build && cmake --build build --target install + # We don't enable exceptions. cppzmq throws exceptions. There isn't really a good alternative API. + # What do we do? Replace all `throw ...` with `zeromq_error()`. Right now this aborts; in the future, if we actually + # want to handle exceptions, we can use longjmp (poor man's exception) + echo "-> patching cppzmq..." + sub() { + sed -i.bak "s/$1/$2/g" "${EXTERNAL_DIR}/zeromq/include/zmq.hpp" "${EXTERNAL_DIR}/zeromq/include/zmq_addon.hpp" + rm "${EXTERNAL_DIR}/zeromq/include/zmq.hpp.bak" "${EXTERNAL_DIR}/zeromq/include/zmq_addon.hpp.bak" + } + sub "throw error_t();" "rir::zeromq_error(__func__);" + sub "throw std::exception();" "rir::zeromq_error(__func__);" +else + echo "-> zeromq already built, run with FORCE=1 to force rebuild. Skipping..." +fi diff --git a/tools/fetch-llvm.sh b/tools/fetch-llvm.sh index 47f9d3a8a..d15ddd48c 100755 --- a/tools/fetch-llvm.sh +++ b/tools/fetch-llvm.sh @@ -11,6 +11,10 @@ fi SRC_DIR=`cd ${SCRIPTPATH}/.. && pwd` . "${SCRIPTPATH}/script_include.sh" +if [[ $(uname -m) == "arm64" ]]; then + echo "there is no LLVM 12 distribution for ARM64, so we will try to build instead" + exec "${SCRIPTPATH}/build-llvm.sh" +fi if [[ "$OSTYPE" == "darwin"* ]]; then USING_OSX=1 diff --git a/tools/git-clang-format b/tools/git-clang-format index 0c45762ea..fc647695e 100755 --- a/tools/git-clang-format +++ b/tools/git-clang-format @@ -128,15 +128,15 @@ def main(): if opts.verbose >= 1: ignored_files.difference_update(changed_lines) if ignored_files: - print 'Ignoring changes in the following files (wrong extension):' + print('Ignoring changes in the following files (wrong extension):') for filename in ignored_files: - print ' ', filename + print(' ', filename) if changed_lines: - print 'Running clang-format on the following files:' + print('Running clang-format on the following files:') for filename in changed_lines: - print ' ', filename + print(' ', filename) if not changed_lines: - print 'no modified files to format' + print('no modified files to format') return # The computed diff outputs absolute paths, so we must cd before accessing # those files. @@ -146,20 +146,20 @@ def main(): binary=opts.binary, style=opts.style) if opts.verbose >= 1: - print 'old tree:', old_tree - print 'new tree:', new_tree + print('old tree:', old_tree) + print('new tree:', new_tree) if old_tree == new_tree: if opts.verbose >= 0: - print 'clang-format did not modify any files' + print('clang-format did not modify any files') elif opts.diff: print_diff(old_tree, new_tree) else: changed_files = apply_changes(old_tree, new_tree, force=opts.force, patch_mode=opts.patch) if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1: - print 'changed files:' + print('changed files:') for filename in changed_files: - print ' ', filename + print(' ', filename) def load_git_config(non_string_options=None): @@ -235,7 +235,7 @@ def get_object_type(value): """Returns a string description of an object's type, or None if it is not a valid git object.""" cmd = ['git', 'cat-file', '-t', value] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() if p.returncode != 0: return None @@ -262,7 +262,7 @@ def compute_diff(commit, files): (if non-empty). Zero context lines are used in the patch.""" cmd = ['git', 'diff-index', '-p', '-U0', commit, '--'] cmd.extend(files) - p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p = popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) p.stdin.close() return p @@ -298,7 +298,7 @@ def filter_by_extension(dictionary, allowed_extensions): `allowed_extensions` must be a collection of lowercase file extensions, excluding the period.""" allowed_extensions = frozenset(allowed_extensions) - for filename in dictionary.keys(): + for filename in list(dictionary.keys()): base_ext = filename.rsplit('.', 1) if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions: del dictionary[filename] @@ -323,7 +323,7 @@ def run_clang_format_and_save_to_tree(changed_lines, binary='clang-format', Returns the object ID (SHA-1) of the created tree.""" def index_info_generator(): - for filename, line_ranges in changed_lines.iteritems(): + for filename, line_ranges in changed_lines.items(): mode = oct(os.stat(filename).st_mode) blob_id = clang_format_to_blob(filename, line_ranges, binary=binary, style=style) @@ -341,7 +341,7 @@ def create_tree(input_lines, mode): assert mode in ('--stdin', '--index-info') cmd = ['git', 'update-index', '--add', '-z', mode] with temporary_index_file(): - p = subprocess.Popen(cmd, stdin=subprocess.PIPE) + p = popen(cmd, stdin=subprocess.PIPE) for line in input_lines: p.stdin.write('%s\0' % line) p.stdin.close() @@ -363,8 +363,7 @@ def clang_format_to_blob(filename, line_ranges, binary='clang-format', '-lines=%s:%s' % (start_line, start_line+line_count-1) for start_line, line_count in line_ranges]) try: - clang_format = subprocess.Popen(clang_format_cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + clang_format = popen(clang_format_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) except OSError as e: if e.errno == errno.ENOENT: die('cannot find executable "%s"' % binary) @@ -372,8 +371,7 @@ def clang_format_to_blob(filename, line_ranges, binary='clang-format', raise clang_format.stdin.close() hash_object_cmd = ['git', 'hash-object', '-w', '--path='+filename, '--stdin'] - hash_object = subprocess.Popen(hash_object_cmd, stdin=clang_format.stdout, - stdout=subprocess.PIPE) + hash_object = popen(hash_object_cmd, stdin=clang_format.stdout, stdout=subprocess.PIPE) clang_format.stdout.close() stdout = hash_object.communicate()[0] if hash_object.returncode != 0: @@ -431,10 +429,9 @@ def apply_changes(old_tree, new_tree, force=False, patch_mode=False): if not force: unstaged_files = run('git', 'diff-files', '--name-status', *changed_files) if unstaged_files: - print >>sys.stderr, ('The following files would be modified but ' - 'have unstaged changes:') - print >>sys.stderr, unstaged_files - print >>sys.stderr, 'Please commit, stage, or stash them first.' + eprint('The following files would be modified but have unstaged changes:') + eprint(unstaged_files) + eprint('Please commit, stage, or stash them first.') sys.exit(2) if patch_mode: # In patch mode, we could just as well create an index from the new tree @@ -452,32 +449,40 @@ def apply_changes(old_tree, new_tree, force=False, patch_mode=False): return changed_files +def popen(*args, **kwargs): + return subprocess.Popen(*args, encoding='utf-8', **kwargs) + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + def run(*args, **kwargs): stdin = kwargs.pop('stdin', '') verbose = kwargs.pop('verbose', True) strip = kwargs.pop('strip', True) for name in kwargs: raise TypeError("run() got an unexpected keyword argument '%s'" % name) - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=subprocess.PIPE) + p = popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=subprocess.PIPE) stdout, stderr = p.communicate(input=stdin) if p.returncode == 0: if stderr: if verbose: - print >>sys.stderr, '`%s` printed to stderr:' % ' '.join(args) - print >>sys.stderr, stderr.rstrip() + eprint('`%s` printed to stderr:' % ' '.join(args)) + eprint(stderr.rstrip()) if strip: stdout = stdout.rstrip('\r\n') return stdout if verbose: - print >>sys.stderr, '`%s` returned %s' % (' '.join(args), p.returncode) + eprint('`%s` returned %s' % (' '.join(args), p.returncode)) if stderr: - print >>sys.stderr, stderr.rstrip() + eprint(stderr.rstrip()) sys.exit(2) def die(message): - print >>sys.stderr, 'error:', message + eprint('error:', message) sys.exit(2) diff --git a/tools/rirPrettyGraph/README.md b/tools/rirPrettyGraph/README.md new file mode 100644 index 000000000..ccd4051e7 --- /dev/null +++ b/tools/rirPrettyGraph/README.md @@ -0,0 +1,5 @@ +# rirPrettyGraph + +Uses [cytoscape.js](http://js.cytoscape.org/) to render RIR objects as graphs. + +Printing RIR objects with `RirObjectPrintStyle::PrettyGraph` will output HTML code which references the code in this folder. \ No newline at end of file diff --git a/tools/rirPrettyGraph/cytoscape-style.js b/tools/rirPrettyGraph/cytoscape-style.js new file mode 100644 index 000000000..d0904d2e8 --- /dev/null +++ b/tools/rirPrettyGraph/cytoscape-style.js @@ -0,0 +1,115 @@ +// This is a STYLESHEET in Cytoscape CSS, see https://js.cytoscape.org/#style +// It's a string embedded in JavaScript because that's easiest to load from main.js +// language=CSS +const style = ` +node { + label: data(name); + compound-sizing-wrt-labels: include; + text-valign: center; + text-halign: center; + font-size: 12px; + /* Shape and color for misc, rare RIR structures */ + shape: triangle; + background-color: #41485A; +} + +node.Code, node.DispatchTable, node.Function { + border-color: #422006; +} + +node.Code { + shape: rectangle; + background-color: #D7983A; + border-width: 2px; + border-opacity: 0.5; +} + +node.DispatchTable { + shape: hexagon; + background-color: #F6DB95; +} + +node.Function { + shape: pentagon; + background-color: #F7B46F; +} + +node.other { + shape: ellipse; + background-color: #528A74; + border-color: #082f49; +} + +node.main { + border-width: 16px; + border-style: double; +} + +edge { + label: data(label); + curve-style: bezier; + target-arrow-shape: triangle; + text-rotation: autorotate; + /* margin-x in case the arrow is vertical, if horizontal it will barely be noticed */ + text-margin-x: -10px; + text-margin-y: -10px; + font-size: 10px; +} + +edge.other-body, edge.DispatchTable-entry, edge.Function-body, edge.Code-arglist-order { + line-color: #422006; + target-arrow-color: #422006; + color: #422006; + width: 4px; +} + +edge.DispatchTable-nested-closure, edge.Function-default-arg, edge.Code-promise { + line-color: #3f6212; + target-arrow-color: #3f6212; + color: #3f6212; + width: 4px; +} + +edge.Code-push, edge.Code-guard, edge.Code-call { + line-color: #075985; + target-arrow-color: #075985; + color: #075985; + /** solid for parent-child relationships, + * dotted for "far away" (e.g. globals), + * dashed for everything else */ + line-style: dashed; + width: 2px; +} + +edge.Code-target { + line-color: #701a75; + target-arrow-color: #701a75; + color: #701a75; + /** solid for parent-child relationships, + * dotted for "far away" (e.g. globals), + * dashed for everything else */ + line-style: dashed; + width: 2px; +} + +edge.Code-unknown-extra-pool { + line-color: #52525b; + target-arrow-color: #52525b; + color: #52525b; + line-style: dashed; + width: 2px; +} + +edge.Code-unexpected-name, edge.Code-unexpected-ast, edge.Code-unexpected-builtin, edge.Code-unexpected { + line-color: #dc2626; + target-arrow-color: #dc2626; + color: #dc2626; + line-style: dashed; + width: 4px; +} + +edge.far-away { + line-style: dotted; + target-arrow-shape: vee; +} +` \ No newline at end of file diff --git a/tools/rirPrettyGraph/dependencies/cose-base.js b/tools/rirPrettyGraph/dependencies/cose-base.js new file mode 100644 index 000000000..49ccc6601 --- /dev/null +++ b/tools/rirPrettyGraph/dependencies/cose-base.js @@ -0,0 +1,3214 @@ +(function webpackUniversalModuleDefinition(root, factory) { + if(typeof exports === 'object' && typeof module === 'object') + module.exports = factory(require("layout-base")); + else if(typeof define === 'function' && define.amd) + define(["layout-base"], factory); + else if(typeof exports === 'object') + exports["coseBase"] = factory(require("layout-base")); + else + root["coseBase"] = factory(root["layoutBase"]); +})(this, function(__WEBPACK_EXTERNAL_MODULE__551__) { +return /******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 45: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var coseBase = {}; + +coseBase.layoutBase = __webpack_require__(551); +coseBase.CoSEConstants = __webpack_require__(806); +coseBase.CoSEEdge = __webpack_require__(767); +coseBase.CoSEGraph = __webpack_require__(880); +coseBase.CoSEGraphManager = __webpack_require__(578); +coseBase.CoSELayout = __webpack_require__(765); +coseBase.CoSENode = __webpack_require__(991); +coseBase.ConstraintHandler = __webpack_require__(902); + +module.exports = coseBase; + +/***/ }), + +/***/ 806: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var FDLayoutConstants = __webpack_require__(551).FDLayoutConstants; + +function CoSEConstants() {} + +//CoSEConstants inherits static props in FDLayoutConstants +for (var prop in FDLayoutConstants) { + CoSEConstants[prop] = FDLayoutConstants[prop]; +} + +CoSEConstants.DEFAULT_USE_MULTI_LEVEL_SCALING = false; +CoSEConstants.DEFAULT_RADIAL_SEPARATION = FDLayoutConstants.DEFAULT_EDGE_LENGTH; +CoSEConstants.DEFAULT_COMPONENT_SEPERATION = 60; +CoSEConstants.TILE = true; +CoSEConstants.TILING_PADDING_VERTICAL = 10; +CoSEConstants.TILING_PADDING_HORIZONTAL = 10; +CoSEConstants.TRANSFORM_ON_CONSTRAINT_HANDLING = true; +CoSEConstants.ENFORCE_CONSTRAINTS = true; +CoSEConstants.APPLY_LAYOUT = true; +CoSEConstants.RELAX_MOVEMENT_ON_CONSTRAINTS = true; +CoSEConstants.TREE_REDUCTION_ON_INCREMENTAL = true; // this should be set to false if there will be a constraint +// This constant is for differentiating whether actual layout algorithm that uses cose-base wants to apply only incremental layout or +// an incremental layout on top of a randomized layout. If it is only incremental layout, then this constant should be true. +CoSEConstants.PURE_INCREMENTAL = CoSEConstants.DEFAULT_INCREMENTAL; + +module.exports = CoSEConstants; + +/***/ }), + +/***/ 767: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var FDLayoutEdge = __webpack_require__(551).FDLayoutEdge; + +function CoSEEdge(source, target, vEdge) { + FDLayoutEdge.call(this, source, target, vEdge); +} + +CoSEEdge.prototype = Object.create(FDLayoutEdge.prototype); +for (var prop in FDLayoutEdge) { + CoSEEdge[prop] = FDLayoutEdge[prop]; +} + +module.exports = CoSEEdge; + +/***/ }), + +/***/ 880: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var LGraph = __webpack_require__(551).LGraph; + +function CoSEGraph(parent, graphMgr, vGraph) { + LGraph.call(this, parent, graphMgr, vGraph); +} + +CoSEGraph.prototype = Object.create(LGraph.prototype); +for (var prop in LGraph) { + CoSEGraph[prop] = LGraph[prop]; +} + +module.exports = CoSEGraph; + +/***/ }), + +/***/ 578: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var LGraphManager = __webpack_require__(551).LGraphManager; + +function CoSEGraphManager(layout) { + LGraphManager.call(this, layout); +} + +CoSEGraphManager.prototype = Object.create(LGraphManager.prototype); +for (var prop in LGraphManager) { + CoSEGraphManager[prop] = LGraphManager[prop]; +} + +module.exports = CoSEGraphManager; + +/***/ }), + +/***/ 765: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var FDLayout = __webpack_require__(551).FDLayout; +var CoSEGraphManager = __webpack_require__(578); +var CoSEGraph = __webpack_require__(880); +var CoSENode = __webpack_require__(991); +var CoSEEdge = __webpack_require__(767); +var CoSEConstants = __webpack_require__(806); +var ConstraintHandler = __webpack_require__(902); +var FDLayoutConstants = __webpack_require__(551).FDLayoutConstants; +var LayoutConstants = __webpack_require__(551).LayoutConstants; +var Point = __webpack_require__(551).Point; +var PointD = __webpack_require__(551).PointD; +var DimensionD = __webpack_require__(551).DimensionD; +var Layout = __webpack_require__(551).Layout; +var Integer = __webpack_require__(551).Integer; +var IGeometry = __webpack_require__(551).IGeometry; +var LGraph = __webpack_require__(551).LGraph; +var Transform = __webpack_require__(551).Transform; +var LinkedList = __webpack_require__(551).LinkedList; + +function CoSELayout() { + FDLayout.call(this); + + this.toBeTiled = {}; // Memorize if a node is to be tiled or is tiled + this.constraints = {}; // keep layout constraints +} + +CoSELayout.prototype = Object.create(FDLayout.prototype); + +for (var prop in FDLayout) { + CoSELayout[prop] = FDLayout[prop]; +} + +CoSELayout.prototype.newGraphManager = function () { + var gm = new CoSEGraphManager(this); + this.graphManager = gm; + return gm; +}; + +CoSELayout.prototype.newGraph = function (vGraph) { + return new CoSEGraph(null, this.graphManager, vGraph); +}; + +CoSELayout.prototype.newNode = function (vNode) { + return new CoSENode(this.graphManager, vNode); +}; + +CoSELayout.prototype.newEdge = function (vEdge) { + return new CoSEEdge(null, null, vEdge); +}; + +CoSELayout.prototype.initParameters = function () { + FDLayout.prototype.initParameters.call(this, arguments); + if (!this.isSubLayout) { + if (CoSEConstants.DEFAULT_EDGE_LENGTH < 10) { + this.idealEdgeLength = 10; + } else { + this.idealEdgeLength = CoSEConstants.DEFAULT_EDGE_LENGTH; + } + + this.useSmartIdealEdgeLengthCalculation = CoSEConstants.DEFAULT_USE_SMART_IDEAL_EDGE_LENGTH_CALCULATION; + this.gravityConstant = FDLayoutConstants.DEFAULT_GRAVITY_STRENGTH; + this.compoundGravityConstant = FDLayoutConstants.DEFAULT_COMPOUND_GRAVITY_STRENGTH; + this.gravityRangeFactor = FDLayoutConstants.DEFAULT_GRAVITY_RANGE_FACTOR; + this.compoundGravityRangeFactor = FDLayoutConstants.DEFAULT_COMPOUND_GRAVITY_RANGE_FACTOR; + + // variables for tree reduction support + this.prunedNodesAll = []; + this.growTreeIterations = 0; + this.afterGrowthIterations = 0; + this.isTreeGrowing = false; + this.isGrowthFinished = false; + } +}; + +// This method is used to set CoSE related parameters used by spring embedder. +CoSELayout.prototype.initSpringEmbedder = function () { + FDLayout.prototype.initSpringEmbedder.call(this); + + // variables for cooling + this.coolingCycle = 0; + this.maxCoolingCycle = this.maxIterations / FDLayoutConstants.CONVERGENCE_CHECK_PERIOD; + this.finalTemperature = 0.04; + this.coolingAdjuster = 1; +}; + +CoSELayout.prototype.layout = function () { + var createBendsAsNeeded = LayoutConstants.DEFAULT_CREATE_BENDS_AS_NEEDED; + if (createBendsAsNeeded) { + this.createBendpoints(); + this.graphManager.resetAllEdges(); + } + + this.level = 0; + return this.classicLayout(); +}; + +CoSELayout.prototype.classicLayout = function () { + this.nodesWithGravity = this.calculateNodesToApplyGravitationTo(); + this.graphManager.setAllNodesToApplyGravitation(this.nodesWithGravity); + this.calcNoOfChildrenForAllNodes(); + this.graphManager.calcLowestCommonAncestors(); + this.graphManager.calcInclusionTreeDepths(); + this.graphManager.getRoot().calcEstimatedSize(); + this.calcIdealEdgeLengths(); + + if (!this.incremental) { + var forest = this.getFlatForest(); + + // The graph associated with this layout is flat and a forest + if (forest.length > 0) { + this.positionNodesRadially(forest); + } + // The graph associated with this layout is not flat or a forest + else { + // Reduce the trees when incremental mode is not enabled and graph is not a forest + this.reduceTrees(); + // Update nodes that gravity will be applied + this.graphManager.resetAllNodesToApplyGravitation(); + var allNodes = new Set(this.getAllNodes()); + var intersection = this.nodesWithGravity.filter(function (x) { + return allNodes.has(x); + }); + this.graphManager.setAllNodesToApplyGravitation(intersection); + + this.positionNodesRandomly(); + } + } else { + if (CoSEConstants.TREE_REDUCTION_ON_INCREMENTAL) { + // Reduce the trees in incremental mode if only this constant is set to true + this.reduceTrees(); + // Update nodes that gravity will be applied + this.graphManager.resetAllNodesToApplyGravitation(); + var allNodes = new Set(this.getAllNodes()); + var intersection = this.nodesWithGravity.filter(function (x) { + return allNodes.has(x); + }); + this.graphManager.setAllNodesToApplyGravitation(intersection); + } + } + + if (Object.keys(this.constraints).length > 0) { + ConstraintHandler.handleConstraints(this); + this.initConstraintVariables(); + } + + this.initSpringEmbedder(); + if (CoSEConstants.APPLY_LAYOUT) { + this.runSpringEmbedder(); + } + + return true; +}; + +CoSELayout.prototype.tick = function () { + this.totalIterations++; + + if (this.totalIterations === this.maxIterations && !this.isTreeGrowing && !this.isGrowthFinished) { + if (this.prunedNodesAll.length > 0) { + this.isTreeGrowing = true; + } else { + return true; + } + } + + if (this.totalIterations % FDLayoutConstants.CONVERGENCE_CHECK_PERIOD == 0 && !this.isTreeGrowing && !this.isGrowthFinished) { + if (this.isConverged()) { + if (this.prunedNodesAll.length > 0) { + this.isTreeGrowing = true; + } else { + return true; + } + } + + this.coolingCycle++; + + if (this.layoutQuality == 0) { + // quality - "draft" + this.coolingAdjuster = this.coolingCycle; + } else if (this.layoutQuality == 1) { + // quality - "default" + this.coolingAdjuster = this.coolingCycle / 3; + } + + // cooling schedule is based on http://www.btluke.com/simanf1.html -> cooling schedule 3 + this.coolingFactor = Math.max(this.initialCoolingFactor - Math.pow(this.coolingCycle, Math.log(100 * (this.initialCoolingFactor - this.finalTemperature)) / Math.log(this.maxCoolingCycle)) / 100 * this.coolingAdjuster, this.finalTemperature); + this.animationPeriod = Math.ceil(this.initialAnimationPeriod * Math.sqrt(this.coolingFactor)); + } + // Operations while tree is growing again + if (this.isTreeGrowing) { + if (this.growTreeIterations % 10 == 0) { + if (this.prunedNodesAll.length > 0) { + this.graphManager.updateBounds(); + this.updateGrid(); + this.growTree(this.prunedNodesAll); + // Update nodes that gravity will be applied + this.graphManager.resetAllNodesToApplyGravitation(); + var allNodes = new Set(this.getAllNodes()); + var intersection = this.nodesWithGravity.filter(function (x) { + return allNodes.has(x); + }); + this.graphManager.setAllNodesToApplyGravitation(intersection); + + this.graphManager.updateBounds(); + this.updateGrid(); + if (CoSEConstants.PURE_INCREMENTAL) this.coolingFactor = FDLayoutConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL / 2;else this.coolingFactor = FDLayoutConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL; + } else { + this.isTreeGrowing = false; + this.isGrowthFinished = true; + } + } + this.growTreeIterations++; + } + // Operations after growth is finished + if (this.isGrowthFinished) { + if (this.isConverged()) { + return true; + } + if (this.afterGrowthIterations % 10 == 0) { + this.graphManager.updateBounds(); + this.updateGrid(); + } + if (CoSEConstants.PURE_INCREMENTAL) this.coolingFactor = FDLayoutConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL / 2 * ((100 - this.afterGrowthIterations) / 100);else this.coolingFactor = FDLayoutConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL * ((100 - this.afterGrowthIterations) / 100); + this.afterGrowthIterations++; + } + + var gridUpdateAllowed = !this.isTreeGrowing && !this.isGrowthFinished; + var forceToNodeSurroundingUpdate = this.growTreeIterations % 10 == 1 && this.isTreeGrowing || this.afterGrowthIterations % 10 == 1 && this.isGrowthFinished; + + this.totalDisplacement = 0; + this.graphManager.updateBounds(); + this.calcSpringForces(); + this.calcRepulsionForces(gridUpdateAllowed, forceToNodeSurroundingUpdate); + this.calcGravitationalForces(); + this.moveNodes(); + this.animate(); + + return false; // Layout is not ended yet return false +}; + +CoSELayout.prototype.getPositionsData = function () { + var allNodes = this.graphManager.getAllNodes(); + var pData = {}; + for (var i = 0; i < allNodes.length; i++) { + var rect = allNodes[i].rect; + var id = allNodes[i].id; + pData[id] = { + id: id, + x: rect.getCenterX(), + y: rect.getCenterY(), + w: rect.width, + h: rect.height + }; + } + + return pData; +}; + +CoSELayout.prototype.runSpringEmbedder = function () { + this.initialAnimationPeriod = 25; + this.animationPeriod = this.initialAnimationPeriod; + var layoutEnded = false; + + // If aminate option is 'during' signal that layout is supposed to start iterating + if (FDLayoutConstants.ANIMATE === 'during') { + this.emit('layoutstarted'); + } else { + // If aminate option is 'during' tick() function will be called on index.js + while (!layoutEnded) { + layoutEnded = this.tick(); + } + + this.graphManager.updateBounds(); + } +}; + +// overrides moveNodes method in FDLayout +CoSELayout.prototype.moveNodes = function () { + var lNodes = this.getAllNodes(); + var node; + + // calculate displacement for each node + for (var i = 0; i < lNodes.length; i++) { + node = lNodes[i]; + node.calculateDisplacement(); + } + + if (Object.keys(this.constraints).length > 0) { + this.updateDisplacements(); + } + + // move each node + for (var i = 0; i < lNodes.length; i++) { + node = lNodes[i]; + node.move(); + } +}; + +// constraint related methods: initConstraintVariables and updateDisplacements + +// initialize constraint related variables +CoSELayout.prototype.initConstraintVariables = function () { + var self = this; + this.idToNodeMap = new Map(); + this.fixedNodeSet = new Set(); + + var allNodes = this.graphManager.getAllNodes(); + + // fill idToNodeMap + for (var i = 0; i < allNodes.length; i++) { + var node = allNodes[i]; + this.idToNodeMap.set(node.id, node); + } + + // calculate fixed node weight for given compound node + var calculateCompoundWeight = function calculateCompoundWeight(compoundNode) { + var nodes = compoundNode.getChild().getNodes(); + var node; + var fixedNodeWeight = 0; + for (var i = 0; i < nodes.length; i++) { + node = nodes[i]; + if (node.getChild() == null) { + if (self.fixedNodeSet.has(node.id)) { + fixedNodeWeight += 100; + } + } else { + fixedNodeWeight += calculateCompoundWeight(node); + } + } + return fixedNodeWeight; + }; + + if (this.constraints.fixedNodeConstraint) { + // fill fixedNodeSet + this.constraints.fixedNodeConstraint.forEach(function (nodeData) { + self.fixedNodeSet.add(nodeData.nodeId); + }); + + // assign fixed node weights to compounds if they contain fixed nodes + var allNodes = this.graphManager.getAllNodes(); + var node; + + for (var i = 0; i < allNodes.length; i++) { + node = allNodes[i]; + if (node.getChild() != null) { + var fixedNodeWeight = calculateCompoundWeight(node); + if (fixedNodeWeight > 0) { + node.fixedNodeWeight = fixedNodeWeight; + } + } + } + } + + if (this.constraints.relativePlacementConstraint) { + var nodeToDummyForVerticalAlignment = new Map(); + var nodeToDummyForHorizontalAlignment = new Map(); + this.dummyToNodeForVerticalAlignment = new Map(); + this.dummyToNodeForHorizontalAlignment = new Map(); + this.fixedNodesOnHorizontal = new Set(); + this.fixedNodesOnVertical = new Set(); + + // fill maps and sets + this.fixedNodeSet.forEach(function (nodeId) { + self.fixedNodesOnHorizontal.add(nodeId); + self.fixedNodesOnVertical.add(nodeId); + }); + + if (this.constraints.alignmentConstraint) { + if (this.constraints.alignmentConstraint.vertical) { + var verticalAlignment = this.constraints.alignmentConstraint.vertical; + for (var i = 0; i < verticalAlignment.length; i++) { + this.dummyToNodeForVerticalAlignment.set("dummy" + i, []); + verticalAlignment[i].forEach(function (nodeId) { + nodeToDummyForVerticalAlignment.set(nodeId, "dummy" + i); + self.dummyToNodeForVerticalAlignment.get("dummy" + i).push(nodeId); + if (self.fixedNodeSet.has(nodeId)) { + self.fixedNodesOnHorizontal.add("dummy" + i); + } + }); + } + } + if (this.constraints.alignmentConstraint.horizontal) { + var horizontalAlignment = this.constraints.alignmentConstraint.horizontal; + for (var i = 0; i < horizontalAlignment.length; i++) { + this.dummyToNodeForHorizontalAlignment.set("dummy" + i, []); + horizontalAlignment[i].forEach(function (nodeId) { + nodeToDummyForHorizontalAlignment.set(nodeId, "dummy" + i); + self.dummyToNodeForHorizontalAlignment.get("dummy" + i).push(nodeId); + if (self.fixedNodeSet.has(nodeId)) { + self.fixedNodesOnVertical.add("dummy" + i); + } + }); + } + } + } + + if (CoSEConstants.RELAX_MOVEMENT_ON_CONSTRAINTS) { + + this.shuffle = function (array) { + var j, x, i; + for (i = array.length - 1; i >= 2 * array.length / 3; i--) { + j = Math.floor(Math.random() * (i + 1)); + x = array[i]; + array[i] = array[j]; + array[j] = x; + } + return array; + }; + + this.nodesInRelativeHorizontal = []; + this.nodesInRelativeVertical = []; + this.nodeToRelativeConstraintMapHorizontal = new Map(); + this.nodeToRelativeConstraintMapVertical = new Map(); + this.nodeToTempPositionMapHorizontal = new Map(); + this.nodeToTempPositionMapVertical = new Map(); + + // fill arrays and maps + this.constraints.relativePlacementConstraint.forEach(function (constraint) { + if (constraint.left) { + var nodeIdLeft = nodeToDummyForVerticalAlignment.has(constraint.left) ? nodeToDummyForVerticalAlignment.get(constraint.left) : constraint.left; + var nodeIdRight = nodeToDummyForVerticalAlignment.has(constraint.right) ? nodeToDummyForVerticalAlignment.get(constraint.right) : constraint.right; + + if (!self.nodesInRelativeHorizontal.includes(nodeIdLeft)) { + self.nodesInRelativeHorizontal.push(nodeIdLeft); + self.nodeToRelativeConstraintMapHorizontal.set(nodeIdLeft, []); + if (self.dummyToNodeForVerticalAlignment.has(nodeIdLeft)) { + self.nodeToTempPositionMapHorizontal.set(nodeIdLeft, self.idToNodeMap.get(self.dummyToNodeForVerticalAlignment.get(nodeIdLeft)[0]).getCenterX()); + } else { + self.nodeToTempPositionMapHorizontal.set(nodeIdLeft, self.idToNodeMap.get(nodeIdLeft).getCenterX()); + } + } + if (!self.nodesInRelativeHorizontal.includes(nodeIdRight)) { + self.nodesInRelativeHorizontal.push(nodeIdRight); + self.nodeToRelativeConstraintMapHorizontal.set(nodeIdRight, []); + if (self.dummyToNodeForVerticalAlignment.has(nodeIdRight)) { + self.nodeToTempPositionMapHorizontal.set(nodeIdRight, self.idToNodeMap.get(self.dummyToNodeForVerticalAlignment.get(nodeIdRight)[0]).getCenterX()); + } else { + self.nodeToTempPositionMapHorizontal.set(nodeIdRight, self.idToNodeMap.get(nodeIdRight).getCenterX()); + } + } + + self.nodeToRelativeConstraintMapHorizontal.get(nodeIdLeft).push({ right: nodeIdRight, gap: constraint.gap }); + self.nodeToRelativeConstraintMapHorizontal.get(nodeIdRight).push({ left: nodeIdLeft, gap: constraint.gap }); + } else { + var nodeIdTop = nodeToDummyForHorizontalAlignment.has(constraint.top) ? nodeToDummyForHorizontalAlignment.get(constraint.top) : constraint.top; + var nodeIdBottom = nodeToDummyForHorizontalAlignment.has(constraint.bottom) ? nodeToDummyForHorizontalAlignment.get(constraint.bottom) : constraint.bottom; + + if (!self.nodesInRelativeVertical.includes(nodeIdTop)) { + self.nodesInRelativeVertical.push(nodeIdTop); + self.nodeToRelativeConstraintMapVertical.set(nodeIdTop, []); + if (self.dummyToNodeForHorizontalAlignment.has(nodeIdTop)) { + self.nodeToTempPositionMapVertical.set(nodeIdTop, self.idToNodeMap.get(self.dummyToNodeForHorizontalAlignment.get(nodeIdTop)[0]).getCenterY()); + } else { + self.nodeToTempPositionMapVertical.set(nodeIdTop, self.idToNodeMap.get(nodeIdTop).getCenterY()); + } + } + if (!self.nodesInRelativeVertical.includes(nodeIdBottom)) { + self.nodesInRelativeVertical.push(nodeIdBottom); + self.nodeToRelativeConstraintMapVertical.set(nodeIdBottom, []); + if (self.dummyToNodeForHorizontalAlignment.has(nodeIdBottom)) { + self.nodeToTempPositionMapVertical.set(nodeIdBottom, self.idToNodeMap.get(self.dummyToNodeForHorizontalAlignment.get(nodeIdBottom)[0]).getCenterY()); + } else { + self.nodeToTempPositionMapVertical.set(nodeIdBottom, self.idToNodeMap.get(nodeIdBottom).getCenterY()); + } + } + self.nodeToRelativeConstraintMapVertical.get(nodeIdTop).push({ bottom: nodeIdBottom, gap: constraint.gap }); + self.nodeToRelativeConstraintMapVertical.get(nodeIdBottom).push({ top: nodeIdTop, gap: constraint.gap }); + } + }); + } else { + var subGraphOnHorizontal = new Map(); // subgraph from vertical RP constraints + var subGraphOnVertical = new Map(); // subgraph from vertical RP constraints + + // construct subgraphs from relative placement constraints + this.constraints.relativePlacementConstraint.forEach(function (constraint) { + if (constraint.left) { + var left = nodeToDummyForVerticalAlignment.has(constraint.left) ? nodeToDummyForVerticalAlignment.get(constraint.left) : constraint.left; + var right = nodeToDummyForVerticalAlignment.has(constraint.right) ? nodeToDummyForVerticalAlignment.get(constraint.right) : constraint.right; + if (subGraphOnHorizontal.has(left)) { + subGraphOnHorizontal.get(left).push(right); + } else { + subGraphOnHorizontal.set(left, [right]); + } + if (subGraphOnHorizontal.has(right)) { + subGraphOnHorizontal.get(right).push(left); + } else { + subGraphOnHorizontal.set(right, [left]); + } + } else { + var top = nodeToDummyForHorizontalAlignment.has(constraint.top) ? nodeToDummyForHorizontalAlignment.get(constraint.top) : constraint.top; + var bottom = nodeToDummyForHorizontalAlignment.has(constraint.bottom) ? nodeToDummyForHorizontalAlignment.get(constraint.bottom) : constraint.bottom; + if (subGraphOnVertical.has(top)) { + subGraphOnVertical.get(top).push(bottom); + } else { + subGraphOnVertical.set(top, [bottom]); + } + if (subGraphOnVertical.has(bottom)) { + subGraphOnVertical.get(bottom).push(top); + } else { + subGraphOnVertical.set(bottom, [top]); + } + } + }); + + // function to construct components from a given graph + // also returns an array that keeps whether each component contains fixed node + var constructComponents = function constructComponents(graph, fixedNodes) { + var components = []; + var isFixed = []; + var queue = new LinkedList(); + var visited = new Set(); + var count = 0; + + graph.forEach(function (value, key) { + if (!visited.has(key)) { + components[count] = []; + isFixed[count] = false; + var currentNode = key; + queue.push(currentNode); + visited.add(currentNode); + components[count].push(currentNode); + + while (queue.length != 0) { + currentNode = queue.shift(); + if (fixedNodes.has(currentNode)) { + isFixed[count] = true; + } + var neighbors = graph.get(currentNode); + neighbors.forEach(function (neighbor) { + if (!visited.has(neighbor)) { + queue.push(neighbor); + visited.add(neighbor); + components[count].push(neighbor); + } + }); + } + count++; + } + }); + + return { components: components, isFixed: isFixed }; + }; + + var resultOnHorizontal = constructComponents(subGraphOnHorizontal, self.fixedNodesOnHorizontal); + this.componentsOnHorizontal = resultOnHorizontal.components; + this.fixedComponentsOnHorizontal = resultOnHorizontal.isFixed; + var resultOnVertical = constructComponents(subGraphOnVertical, self.fixedNodesOnVertical); + this.componentsOnVertical = resultOnVertical.components; + this.fixedComponentsOnVertical = resultOnVertical.isFixed; + } + } +}; + +// updates node displacements based on constraints +CoSELayout.prototype.updateDisplacements = function () { + var self = this; + if (this.constraints.fixedNodeConstraint) { + this.constraints.fixedNodeConstraint.forEach(function (nodeData) { + var fixedNode = self.idToNodeMap.get(nodeData.nodeId); + fixedNode.displacementX = 0; + fixedNode.displacementY = 0; + }); + } + + if (this.constraints.alignmentConstraint) { + if (this.constraints.alignmentConstraint.vertical) { + var allVerticalAlignments = this.constraints.alignmentConstraint.vertical; + for (var i = 0; i < allVerticalAlignments.length; i++) { + var totalDisplacementX = 0; + for (var j = 0; j < allVerticalAlignments[i].length; j++) { + if (this.fixedNodeSet.has(allVerticalAlignments[i][j])) { + totalDisplacementX = 0; + break; + } + totalDisplacementX += this.idToNodeMap.get(allVerticalAlignments[i][j]).displacementX; + } + var averageDisplacementX = totalDisplacementX / allVerticalAlignments[i].length; + for (var j = 0; j < allVerticalAlignments[i].length; j++) { + this.idToNodeMap.get(allVerticalAlignments[i][j]).displacementX = averageDisplacementX; + } + } + } + if (this.constraints.alignmentConstraint.horizontal) { + var allHorizontalAlignments = this.constraints.alignmentConstraint.horizontal; + for (var i = 0; i < allHorizontalAlignments.length; i++) { + var totalDisplacementY = 0; + for (var j = 0; j < allHorizontalAlignments[i].length; j++) { + if (this.fixedNodeSet.has(allHorizontalAlignments[i][j])) { + totalDisplacementY = 0; + break; + } + totalDisplacementY += this.idToNodeMap.get(allHorizontalAlignments[i][j]).displacementY; + } + var averageDisplacementY = totalDisplacementY / allHorizontalAlignments[i].length; + for (var j = 0; j < allHorizontalAlignments[i].length; j++) { + this.idToNodeMap.get(allHorizontalAlignments[i][j]).displacementY = averageDisplacementY; + } + } + } + } + + if (this.constraints.relativePlacementConstraint) { + + if (CoSEConstants.RELAX_MOVEMENT_ON_CONSTRAINTS) { + // shuffle array to randomize node processing order + if (this.totalIterations % 10 == 0) { + this.shuffle(this.nodesInRelativeHorizontal); + this.shuffle(this.nodesInRelativeVertical); + } + + this.nodesInRelativeHorizontal.forEach(function (nodeId) { + if (!self.fixedNodesOnHorizontal.has(nodeId)) { + var displacement = 0; + if (self.dummyToNodeForVerticalAlignment.has(nodeId)) { + displacement = self.idToNodeMap.get(self.dummyToNodeForVerticalAlignment.get(nodeId)[0]).displacementX; + } else { + displacement = self.idToNodeMap.get(nodeId).displacementX; + } + self.nodeToRelativeConstraintMapHorizontal.get(nodeId).forEach(function (constraint) { + if (constraint.right) { + var diff = self.nodeToTempPositionMapHorizontal.get(constraint.right) - self.nodeToTempPositionMapHorizontal.get(nodeId) - displacement; + if (diff < constraint.gap) { + displacement -= constraint.gap - diff; + } + } else { + var diff = self.nodeToTempPositionMapHorizontal.get(nodeId) - self.nodeToTempPositionMapHorizontal.get(constraint.left) + displacement; + if (diff < constraint.gap) { + displacement += constraint.gap - diff; + } + } + }); + self.nodeToTempPositionMapHorizontal.set(nodeId, self.nodeToTempPositionMapHorizontal.get(nodeId) + displacement); + if (self.dummyToNodeForVerticalAlignment.has(nodeId)) { + self.dummyToNodeForVerticalAlignment.get(nodeId).forEach(function (nodeId) { + self.idToNodeMap.get(nodeId).displacementX = displacement; + }); + } else { + self.idToNodeMap.get(nodeId).displacementX = displacement; + } + } + }); + + this.nodesInRelativeVertical.forEach(function (nodeId) { + if (!self.fixedNodesOnHorizontal.has(nodeId)) { + var displacement = 0; + if (self.dummyToNodeForHorizontalAlignment.has(nodeId)) { + displacement = self.idToNodeMap.get(self.dummyToNodeForHorizontalAlignment.get(nodeId)[0]).displacementY; + } else { + displacement = self.idToNodeMap.get(nodeId).displacementY; + } + self.nodeToRelativeConstraintMapVertical.get(nodeId).forEach(function (constraint) { + if (constraint.bottom) { + var diff = self.nodeToTempPositionMapVertical.get(constraint.bottom) - self.nodeToTempPositionMapVertical.get(nodeId) - displacement; + if (diff < constraint.gap) { + displacement -= constraint.gap - diff; + } + } else { + var diff = self.nodeToTempPositionMapVertical.get(nodeId) - self.nodeToTempPositionMapVertical.get(constraint.top) + displacement; + if (diff < constraint.gap) { + displacement += constraint.gap - diff; + } + } + }); + self.nodeToTempPositionMapVertical.set(nodeId, self.nodeToTempPositionMapVertical.get(nodeId) + displacement); + if (self.dummyToNodeForHorizontalAlignment.has(nodeId)) { + self.dummyToNodeForHorizontalAlignment.get(nodeId).forEach(function (nodeId) { + self.idToNodeMap.get(nodeId).displacementY = displacement; + }); + } else { + self.idToNodeMap.get(nodeId).displacementY = displacement; + } + } + }); + } else { + for (var i = 0; i < this.componentsOnHorizontal.length; i++) { + var component = this.componentsOnHorizontal[i]; + if (this.fixedComponentsOnHorizontal[i]) { + for (var j = 0; j < component.length; j++) { + if (this.dummyToNodeForVerticalAlignment.has(component[j])) { + this.dummyToNodeForVerticalAlignment.get(component[j]).forEach(function (nodeId) { + self.idToNodeMap.get(nodeId).displacementX = 0; + }); + } else { + this.idToNodeMap.get(component[j]).displacementX = 0; + } + } + } else { + var sum = 0; + var count = 0; + for (var j = 0; j < component.length; j++) { + if (this.dummyToNodeForVerticalAlignment.has(component[j])) { + var actualNodes = this.dummyToNodeForVerticalAlignment.get(component[j]); + sum += actualNodes.length * this.idToNodeMap.get(actualNodes[0]).displacementX; + count += actualNodes.length; + } else { + sum += this.idToNodeMap.get(component[j]).displacementX; + count++; + } + } + var averageDisplacement = sum / count; + for (var j = 0; j < component.length; j++) { + if (this.dummyToNodeForVerticalAlignment.has(component[j])) { + this.dummyToNodeForVerticalAlignment.get(component[j]).forEach(function (nodeId) { + self.idToNodeMap.get(nodeId).displacementX = averageDisplacement; + }); + } else { + this.idToNodeMap.get(component[j]).displacementX = averageDisplacement; + } + } + } + } + + for (var i = 0; i < this.componentsOnVertical.length; i++) { + var component = this.componentsOnVertical[i]; + if (this.fixedComponentsOnVertical[i]) { + for (var j = 0; j < component.length; j++) { + if (this.dummyToNodeForHorizontalAlignment.has(component[j])) { + this.dummyToNodeForHorizontalAlignment.get(component[j]).forEach(function (nodeId) { + self.idToNodeMap.get(nodeId).displacementY = 0; + }); + } else { + this.idToNodeMap.get(component[j]).displacementY = 0; + } + } + } else { + var sum = 0; + var count = 0; + for (var j = 0; j < component.length; j++) { + if (this.dummyToNodeForHorizontalAlignment.has(component[j])) { + var actualNodes = this.dummyToNodeForHorizontalAlignment.get(component[j]); + sum += actualNodes.length * this.idToNodeMap.get(actualNodes[0]).displacementY; + count += actualNodes.length; + } else { + sum += this.idToNodeMap.get(component[j]).displacementY; + count++; + } + } + var averageDisplacement = sum / count; + for (var j = 0; j < component.length; j++) { + if (this.dummyToNodeForHorizontalAlignment.has(component[j])) { + this.dummyToNodeForHorizontalAlignment.get(component[j]).forEach(function (nodeId) { + self.idToNodeMap.get(nodeId).displacementY = averageDisplacement; + }); + } else { + this.idToNodeMap.get(component[j]).displacementY = averageDisplacement; + } + } + } + } + } + } +}; + +CoSELayout.prototype.calculateNodesToApplyGravitationTo = function () { + var nodeList = []; + var graph; + + var graphs = this.graphManager.getGraphs(); + var size = graphs.length; + var i; + for (i = 0; i < size; i++) { + graph = graphs[i]; + + graph.updateConnected(); + + if (!graph.isConnected) { + nodeList = nodeList.concat(graph.getNodes()); + } + } + + return nodeList; +}; + +CoSELayout.prototype.createBendpoints = function () { + var edges = []; + edges = edges.concat(this.graphManager.getAllEdges()); + var visited = new Set(); + var i; + for (i = 0; i < edges.length; i++) { + var edge = edges[i]; + + if (!visited.has(edge)) { + var source = edge.getSource(); + var target = edge.getTarget(); + + if (source == target) { + edge.getBendpoints().push(new PointD()); + edge.getBendpoints().push(new PointD()); + this.createDummyNodesForBendpoints(edge); + visited.add(edge); + } else { + var edgeList = []; + + edgeList = edgeList.concat(source.getEdgeListToNode(target)); + edgeList = edgeList.concat(target.getEdgeListToNode(source)); + + if (!visited.has(edgeList[0])) { + if (edgeList.length > 1) { + var k; + for (k = 0; k < edgeList.length; k++) { + var multiEdge = edgeList[k]; + multiEdge.getBendpoints().push(new PointD()); + this.createDummyNodesForBendpoints(multiEdge); + } + } + edgeList.forEach(function (edge) { + visited.add(edge); + }); + } + } + } + + if (visited.size == edges.length) { + break; + } + } +}; + +CoSELayout.prototype.positionNodesRadially = function (forest) { + // We tile the trees to a grid row by row; first tree starts at (0,0) + var currentStartingPoint = new Point(0, 0); + var numberOfColumns = Math.ceil(Math.sqrt(forest.length)); + var height = 0; + var currentY = 0; + var currentX = 0; + var point = new PointD(0, 0); + + for (var i = 0; i < forest.length; i++) { + if (i % numberOfColumns == 0) { + // Start of a new row, make the x coordinate 0, increment the + // y coordinate with the max height of the previous row + currentX = 0; + currentY = height; + + if (i != 0) { + currentY += CoSEConstants.DEFAULT_COMPONENT_SEPERATION; + } + + height = 0; + } + + var tree = forest[i]; + + // Find the center of the tree + var centerNode = Layout.findCenterOfTree(tree); + + // Set the staring point of the next tree + currentStartingPoint.x = currentX; + currentStartingPoint.y = currentY; + + // Do a radial layout starting with the center + point = CoSELayout.radialLayout(tree, centerNode, currentStartingPoint); + + if (point.y > height) { + height = Math.floor(point.y); + } + + currentX = Math.floor(point.x + CoSEConstants.DEFAULT_COMPONENT_SEPERATION); + } + + this.transform(new PointD(LayoutConstants.WORLD_CENTER_X - point.x / 2, LayoutConstants.WORLD_CENTER_Y - point.y / 2)); +}; + +CoSELayout.radialLayout = function (tree, centerNode, startingPoint) { + var radialSep = Math.max(this.maxDiagonalInTree(tree), CoSEConstants.DEFAULT_RADIAL_SEPARATION); + CoSELayout.branchRadialLayout(centerNode, null, 0, 359, 0, radialSep); + var bounds = LGraph.calculateBounds(tree); + + var transform = new Transform(); + transform.setDeviceOrgX(bounds.getMinX()); + transform.setDeviceOrgY(bounds.getMinY()); + transform.setWorldOrgX(startingPoint.x); + transform.setWorldOrgY(startingPoint.y); + + for (var i = 0; i < tree.length; i++) { + var node = tree[i]; + node.transform(transform); + } + + var bottomRight = new PointD(bounds.getMaxX(), bounds.getMaxY()); + + return transform.inverseTransformPoint(bottomRight); +}; + +CoSELayout.branchRadialLayout = function (node, parentOfNode, startAngle, endAngle, distance, radialSeparation) { + // First, position this node by finding its angle. + var halfInterval = (endAngle - startAngle + 1) / 2; + + if (halfInterval < 0) { + halfInterval += 180; + } + + var nodeAngle = (halfInterval + startAngle) % 360; + var teta = nodeAngle * IGeometry.TWO_PI / 360; + + // Make polar to java cordinate conversion. + var cos_teta = Math.cos(teta); + var x_ = distance * Math.cos(teta); + var y_ = distance * Math.sin(teta); + + node.setCenter(x_, y_); + + // Traverse all neighbors of this node and recursively call this + // function. + var neighborEdges = []; + neighborEdges = neighborEdges.concat(node.getEdges()); + var childCount = neighborEdges.length; + + if (parentOfNode != null) { + childCount--; + } + + var branchCount = 0; + + var incEdgesCount = neighborEdges.length; + var startIndex; + + var edges = node.getEdgesBetween(parentOfNode); + + // If there are multiple edges, prune them until there remains only one + // edge. + while (edges.length > 1) { + //neighborEdges.remove(edges.remove(0)); + var temp = edges[0]; + edges.splice(0, 1); + var index = neighborEdges.indexOf(temp); + if (index >= 0) { + neighborEdges.splice(index, 1); + } + incEdgesCount--; + childCount--; + } + + if (parentOfNode != null) { + //assert edges.length == 1; + startIndex = (neighborEdges.indexOf(edges[0]) + 1) % incEdgesCount; + } else { + startIndex = 0; + } + + var stepAngle = Math.abs(endAngle - startAngle) / childCount; + + for (var i = startIndex; branchCount != childCount; i = ++i % incEdgesCount) { + var currentNeighbor = neighborEdges[i].getOtherEnd(node); + + // Don't back traverse to root node in current tree. + if (currentNeighbor == parentOfNode) { + continue; + } + + var childStartAngle = (startAngle + branchCount * stepAngle) % 360; + var childEndAngle = (childStartAngle + stepAngle) % 360; + + CoSELayout.branchRadialLayout(currentNeighbor, node, childStartAngle, childEndAngle, distance + radialSeparation, radialSeparation); + + branchCount++; + } +}; + +CoSELayout.maxDiagonalInTree = function (tree) { + var maxDiagonal = Integer.MIN_VALUE; + + for (var i = 0; i < tree.length; i++) { + var node = tree[i]; + var diagonal = node.getDiagonal(); + + if (diagonal > maxDiagonal) { + maxDiagonal = diagonal; + } + } + + return maxDiagonal; +}; + +CoSELayout.prototype.calcRepulsionRange = function () { + // formula is 2 x (level + 1) x idealEdgeLength + return 2 * (this.level + 1) * this.idealEdgeLength; +}; + +// Tiling methods + +// Group zero degree members whose parents are not to be tiled, create dummy parents where needed and fill memberGroups by their dummp parent id's +CoSELayout.prototype.groupZeroDegreeMembers = function () { + var self = this; + // array of [parent_id x oneDegreeNode_id] + var tempMemberGroups = {}; // A temporary map of parent node and its zero degree members + this.memberGroups = {}; // A map of dummy parent node and its zero degree members whose parents are not to be tiled + this.idToDummyNode = {}; // A map of id to dummy node + + var zeroDegree = []; // List of zero degree nodes whose parents are not to be tiled + var allNodes = this.graphManager.getAllNodes(); + + // Fill zero degree list + for (var i = 0; i < allNodes.length; i++) { + var node = allNodes[i]; + var parent = node.getParent(); + // If a node has zero degree and its parent is not to be tiled if exists add that node to zeroDegres list + if (this.getNodeDegreeWithChildren(node) === 0 && (parent.id == undefined || !this.getToBeTiled(parent))) { + zeroDegree.push(node); + } + } + + // Create a map of parent node and its zero degree members + for (var i = 0; i < zeroDegree.length; i++) { + var node = zeroDegree[i]; // Zero degree node itself + var p_id = node.getParent().id; // Parent id + + if (typeof tempMemberGroups[p_id] === "undefined") tempMemberGroups[p_id] = []; + + tempMemberGroups[p_id] = tempMemberGroups[p_id].concat(node); // Push node to the list belongs to its parent in tempMemberGroups + } + + // If there are at least two nodes at a level, create a dummy compound for them + Object.keys(tempMemberGroups).forEach(function (p_id) { + if (tempMemberGroups[p_id].length > 1) { + var dummyCompoundId = "DummyCompound_" + p_id; // The id of dummy compound which will be created soon + self.memberGroups[dummyCompoundId] = tempMemberGroups[p_id]; // Add dummy compound to memberGroups + + var parent = tempMemberGroups[p_id][0].getParent(); // The parent of zero degree nodes will be the parent of new dummy compound + + // Create a dummy compound with calculated id + var dummyCompound = new CoSENode(self.graphManager); + dummyCompound.id = dummyCompoundId; + dummyCompound.paddingLeft = parent.paddingLeft || 0; + dummyCompound.paddingRight = parent.paddingRight || 0; + dummyCompound.paddingBottom = parent.paddingBottom || 0; + dummyCompound.paddingTop = parent.paddingTop || 0; + + self.idToDummyNode[dummyCompoundId] = dummyCompound; + + var dummyParentGraph = self.getGraphManager().add(self.newGraph(), dummyCompound); + var parentGraph = parent.getChild(); + + // Add dummy compound to parent the graph + parentGraph.add(dummyCompound); + + // For each zero degree node in this level remove it from its parent graph and add it to the graph of dummy parent + for (var i = 0; i < tempMemberGroups[p_id].length; i++) { + var node = tempMemberGroups[p_id][i]; + + parentGraph.remove(node); + dummyParentGraph.add(node); + } + } + }); +}; + +CoSELayout.prototype.clearCompounds = function () { + var childGraphMap = {}; + var idToNode = {}; + + // Get compound ordering by finding the inner one first + this.performDFSOnCompounds(); + + for (var i = 0; i < this.compoundOrder.length; i++) { + + idToNode[this.compoundOrder[i].id] = this.compoundOrder[i]; + childGraphMap[this.compoundOrder[i].id] = [].concat(this.compoundOrder[i].getChild().getNodes()); + + // Remove children of compounds + this.graphManager.remove(this.compoundOrder[i].getChild()); + this.compoundOrder[i].child = null; + } + + this.graphManager.resetAllNodes(); + + // Tile the removed children + this.tileCompoundMembers(childGraphMap, idToNode); +}; + +CoSELayout.prototype.clearZeroDegreeMembers = function () { + var self = this; + var tiledZeroDegreePack = this.tiledZeroDegreePack = []; + + Object.keys(this.memberGroups).forEach(function (id) { + var compoundNode = self.idToDummyNode[id]; // Get the dummy compound + + tiledZeroDegreePack[id] = self.tileNodes(self.memberGroups[id], compoundNode.paddingLeft + compoundNode.paddingRight); + + // Set the width and height of the dummy compound as calculated + compoundNode.rect.width = tiledZeroDegreePack[id].width; + compoundNode.rect.height = tiledZeroDegreePack[id].height; + compoundNode.setCenter(tiledZeroDegreePack[id].centerX, tiledZeroDegreePack[id].centerY); + + // compound left and top margings for labels + // when node labels are included, these values may be set to different values below and are used in tilingPostLayout, + // otherwise they stay as zero + compoundNode.labelMarginLeft = 0; + compoundNode.labelMarginTop = 0; + + // Update compound bounds considering its label properties and set label margins for left and top + if (CoSEConstants.NODE_DIMENSIONS_INCLUDE_LABELS) { + + var width = compoundNode.rect.width; + var height = compoundNode.rect.height; + + if (compoundNode.labelWidth) { + if (compoundNode.labelPosHorizontal == "left") { + compoundNode.rect.x -= compoundNode.labelWidth; + compoundNode.setWidth(width + compoundNode.labelWidth); + compoundNode.labelMarginLeft = compoundNode.labelWidth; + } else if (compoundNode.labelPosHorizontal == "center" && compoundNode.labelWidth > width) { + compoundNode.rect.x -= (compoundNode.labelWidth - width) / 2; + compoundNode.setWidth(compoundNode.labelWidth); + compoundNode.labelMarginLeft = (compoundNode.labelWidth - width) / 2; + } else if (compoundNode.labelPosHorizontal == "right") { + compoundNode.setWidth(width + compoundNode.labelWidth); + } + } + + if (compoundNode.labelHeight) { + if (compoundNode.labelPosVertical == "top") { + compoundNode.rect.y -= compoundNode.labelHeight; + compoundNode.setHeight(height + compoundNode.labelHeight); + compoundNode.labelMarginTop = compoundNode.labelHeight; + } else if (compoundNode.labelPosVertical == "center" && compoundNode.labelHeight > height) { + compoundNode.rect.y -= (compoundNode.labelHeight - height) / 2; + compoundNode.setHeight(compoundNode.labelHeight); + compoundNode.labelMarginTop = (compoundNode.labelHeight - height) / 2; + } else if (compoundNode.labelPosVertical == "bottom") { + compoundNode.setHeight(height + compoundNode.labelHeight); + } + } + } + }); +}; + +CoSELayout.prototype.repopulateCompounds = function () { + for (var i = this.compoundOrder.length - 1; i >= 0; i--) { + var lCompoundNode = this.compoundOrder[i]; + var id = lCompoundNode.id; + var horizontalMargin = lCompoundNode.paddingLeft; + var verticalMargin = lCompoundNode.paddingTop; + var labelMarginLeft = lCompoundNode.labelMarginLeft; + var labelMarginTop = lCompoundNode.labelMarginTop; + + this.adjustLocations(this.tiledMemberPack[id], lCompoundNode.rect.x, lCompoundNode.rect.y, horizontalMargin, verticalMargin, labelMarginLeft, labelMarginTop); + } +}; + +CoSELayout.prototype.repopulateZeroDegreeMembers = function () { + var self = this; + var tiledPack = this.tiledZeroDegreePack; + + Object.keys(tiledPack).forEach(function (id) { + var compoundNode = self.idToDummyNode[id]; // Get the dummy compound by its id + var horizontalMargin = compoundNode.paddingLeft; + var verticalMargin = compoundNode.paddingTop; + var labelMarginLeft = compoundNode.labelMarginLeft; + var labelMarginTop = compoundNode.labelMarginTop; + + // Adjust the positions of nodes wrt its compound + self.adjustLocations(tiledPack[id], compoundNode.rect.x, compoundNode.rect.y, horizontalMargin, verticalMargin, labelMarginLeft, labelMarginTop); + }); +}; + +CoSELayout.prototype.getToBeTiled = function (node) { + var id = node.id; + //firstly check the previous results + if (this.toBeTiled[id] != null) { + return this.toBeTiled[id]; + } + + //only compound nodes are to be tiled + var childGraph = node.getChild(); + if (childGraph == null) { + this.toBeTiled[id] = false; + return false; + } + + var children = childGraph.getNodes(); // Get the children nodes + + //a compound node is not to be tiled if all of its compound children are not to be tiled + for (var i = 0; i < children.length; i++) { + var theChild = children[i]; + + if (this.getNodeDegree(theChild) > 0) { + this.toBeTiled[id] = false; + return false; + } + + //pass the children not having the compound structure + if (theChild.getChild() == null) { + this.toBeTiled[theChild.id] = false; + continue; + } + + if (!this.getToBeTiled(theChild)) { + this.toBeTiled[id] = false; + return false; + } + } + this.toBeTiled[id] = true; + return true; +}; + +// Get degree of a node depending of its edges and independent of its children +CoSELayout.prototype.getNodeDegree = function (node) { + var id = node.id; + var edges = node.getEdges(); + var degree = 0; + + // For the edges connected + for (var i = 0; i < edges.length; i++) { + var edge = edges[i]; + if (edge.getSource().id !== edge.getTarget().id) { + degree = degree + 1; + } + } + return degree; +}; + +// Get degree of a node with its children +CoSELayout.prototype.getNodeDegreeWithChildren = function (node) { + var degree = this.getNodeDegree(node); + if (node.getChild() == null) { + return degree; + } + var children = node.getChild().getNodes(); + for (var i = 0; i < children.length; i++) { + var child = children[i]; + degree += this.getNodeDegreeWithChildren(child); + } + return degree; +}; + +CoSELayout.prototype.performDFSOnCompounds = function () { + this.compoundOrder = []; + this.fillCompexOrderByDFS(this.graphManager.getRoot().getNodes()); +}; + +CoSELayout.prototype.fillCompexOrderByDFS = function (children) { + for (var i = 0; i < children.length; i++) { + var child = children[i]; + if (child.getChild() != null) { + this.fillCompexOrderByDFS(child.getChild().getNodes()); + } + if (this.getToBeTiled(child)) { + this.compoundOrder.push(child); + } + } +}; + +/** +* This method places each zero degree member wrt given (x,y) coordinates (top left). +*/ +CoSELayout.prototype.adjustLocations = function (organization, x, y, compoundHorizontalMargin, compoundVerticalMargin, compoundLabelMarginLeft, compoundLabelMarginTop) { + x += compoundHorizontalMargin + compoundLabelMarginLeft; + y += compoundVerticalMargin + compoundLabelMarginTop; + + var left = x; + + for (var i = 0; i < organization.rows.length; i++) { + var row = organization.rows[i]; + x = left; + var maxHeight = 0; + + for (var j = 0; j < row.length; j++) { + var lnode = row[j]; + + lnode.rect.x = x; // + lnode.rect.width / 2; + lnode.rect.y = y; // + lnode.rect.height / 2; + + x += lnode.rect.width + organization.horizontalPadding; + + if (lnode.rect.height > maxHeight) maxHeight = lnode.rect.height; + } + + y += maxHeight + organization.verticalPadding; + } +}; + +CoSELayout.prototype.tileCompoundMembers = function (childGraphMap, idToNode) { + var self = this; + this.tiledMemberPack = []; + + Object.keys(childGraphMap).forEach(function (id) { + // Get the compound node + var compoundNode = idToNode[id]; + + self.tiledMemberPack[id] = self.tileNodes(childGraphMap[id], compoundNode.paddingLeft + compoundNode.paddingRight); + + compoundNode.rect.width = self.tiledMemberPack[id].width; + compoundNode.rect.height = self.tiledMemberPack[id].height; + compoundNode.setCenter(self.tiledMemberPack[id].centerX, self.tiledMemberPack[id].centerY); + + // compound left and top margings for labels + // when node labels are included, these values may be set to different values below and are used in tilingPostLayout, + // otherwise they stay as zero + compoundNode.labelMarginLeft = 0; + compoundNode.labelMarginTop = 0; + + // Update compound bounds considering its label properties and set label margins for left and top + if (CoSEConstants.NODE_DIMENSIONS_INCLUDE_LABELS) { + + var width = compoundNode.rect.width; + var height = compoundNode.rect.height; + + if (compoundNode.labelWidth) { + if (compoundNode.labelPosHorizontal == "left") { + compoundNode.rect.x -= compoundNode.labelWidth; + compoundNode.setWidth(width + compoundNode.labelWidth); + compoundNode.labelMarginLeft = compoundNode.labelWidth; + } else if (compoundNode.labelPosHorizontal == "center" && compoundNode.labelWidth > width) { + compoundNode.rect.x -= (compoundNode.labelWidth - width) / 2; + compoundNode.setWidth(compoundNode.labelWidth); + compoundNode.labelMarginLeft = (compoundNode.labelWidth - width) / 2; + } else if (compoundNode.labelPosHorizontal == "right") { + compoundNode.setWidth(width + compoundNode.labelWidth); + } + } + + if (compoundNode.labelHeight) { + if (compoundNode.labelPosVertical == "top") { + compoundNode.rect.y -= compoundNode.labelHeight; + compoundNode.setHeight(height + compoundNode.labelHeight); + compoundNode.labelMarginTop = compoundNode.labelHeight; + } else if (compoundNode.labelPosVertical == "center" && compoundNode.labelHeight > height) { + compoundNode.rect.y -= (compoundNode.labelHeight - height) / 2; + compoundNode.setHeight(compoundNode.labelHeight); + compoundNode.labelMarginTop = (compoundNode.labelHeight - height) / 2; + } else if (compoundNode.labelPosVertical == "bottom") { + compoundNode.setHeight(height + compoundNode.labelHeight); + } + } + } + }); +}; + +CoSELayout.prototype.tileNodes = function (nodes, minWidth) { + var horizontalOrg = this.tileNodesByFavoringDim(nodes, minWidth, true); + var verticalOrg = this.tileNodesByFavoringDim(nodes, minWidth, false); + + var horizontalRatio = this.getOrgRatio(horizontalOrg); + var verticalRatio = this.getOrgRatio(verticalOrg); + var bestOrg; + + // the best ratio is the one that is closer to 1 since the ratios are already normalized + // and the best organization is the one that has the best ratio + if (verticalRatio < horizontalRatio) { + bestOrg = verticalOrg; + } else { + bestOrg = horizontalOrg; + } + + return bestOrg; +}; + +// get the width/height ratio of the organization that is normalized so that it will not be less than 1 +CoSELayout.prototype.getOrgRatio = function (organization) { + // get dimensions and calculate the initial ratio + var width = organization.width; + var height = organization.height; + var ratio = width / height; + + // if the initial ratio is less then 1 then inverse it + if (ratio < 1) { + ratio = 1 / ratio; + } + + // return the normalized ratio + return ratio; +}; + +/* + * Calculates the ideal width for the rows. This method assumes that + * each node has the same sizes and calculates the ideal row width that + * approximates a square shaped complex accordingly. However, since nodes would + * have different sizes some rows would have different sizes and the resulting + * shape would not be an exact square. + */ +CoSELayout.prototype.calcIdealRowWidth = function (members, favorHorizontalDim) { + // To approximate a square shaped complex we need to make complex width equal to complex height. + // To achieve this we need to solve the following equation system for hc: + // (x + bx) * hc - bx = (y + by) * vc - by, hc * vc = n + // where x is the avarage width of the nodes, y is the avarage height of nodes + // bx and by are the buffer sizes in horizontal and vertical dimensions accordingly, + // hc and vc are the number of rows in horizontal and vertical dimensions + // n is number of members. + + var verticalPadding = CoSEConstants.TILING_PADDING_VERTICAL; + var horizontalPadding = CoSEConstants.TILING_PADDING_HORIZONTAL; + + // number of members + var membersSize = members.length; + + // sum of the width of all members + var totalWidth = 0; + + // sum of the height of all members + var totalHeight = 0; + + var maxWidth = 0; + + // traverse all members to calculate total width and total height and get the maximum members width + members.forEach(function (node) { + totalWidth += node.getWidth(); + totalHeight += node.getHeight(); + + if (node.getWidth() > maxWidth) { + maxWidth = node.getWidth(); + } + }); + + // average width of the members + var averageWidth = totalWidth / membersSize; + + // average height of the members + var averageHeight = totalHeight / membersSize; + + // solving the initial equation system for the hc yields the following second degree equation: + // hc^2 * (x+bx) + hc * (by - bx) - n * (y + by) = 0 + + // the delta value to solve the equation above for hc + var delta = Math.pow(verticalPadding - horizontalPadding, 2) + 4 * (averageWidth + horizontalPadding) * (averageHeight + verticalPadding) * membersSize; + + // solve the equation using delta value to calculate the horizontal count + // that represents the number of nodes in an ideal row + var horizontalCountDouble = (horizontalPadding - verticalPadding + Math.sqrt(delta)) / (2 * (averageWidth + horizontalPadding)); + // round the calculated horizontal count up or down according to the favored dimension + var horizontalCount; + + if (favorHorizontalDim) { + horizontalCount = Math.ceil(horizontalCountDouble); + // if horizontalCount count is not a float value then both of rounding to floor and ceil + // will yield the same values. Instead of repeating the same calculation try going up + // while favoring horizontal dimension in such cases + if (horizontalCount == horizontalCountDouble) { + horizontalCount++; + } + } else { + horizontalCount = Math.floor(horizontalCountDouble); + } + + // ideal width to be calculated + var idealWidth = horizontalCount * (averageWidth + horizontalPadding) - horizontalPadding; + + // if max width is bigger than calculated ideal width reset ideal width to it + if (maxWidth > idealWidth) { + idealWidth = maxWidth; + } + + // add the left-right margins to the ideal row width + idealWidth += horizontalPadding * 2; + + // return the ideal row width1 + return idealWidth; +}; + +CoSELayout.prototype.tileNodesByFavoringDim = function (nodes, minWidth, favorHorizontalDim) { + var verticalPadding = CoSEConstants.TILING_PADDING_VERTICAL; + var horizontalPadding = CoSEConstants.TILING_PADDING_HORIZONTAL; + var tilingCompareBy = CoSEConstants.TILING_COMPARE_BY; + var organization = { + rows: [], + rowWidth: [], + rowHeight: [], + width: 0, + height: minWidth, // assume minHeight equals to minWidth + verticalPadding: verticalPadding, + horizontalPadding: horizontalPadding, + centerX: 0, + centerY: 0 + }; + + if (tilingCompareBy) { + organization.idealRowWidth = this.calcIdealRowWidth(nodes, favorHorizontalDim); + } + + var getNodeArea = function getNodeArea(n) { + return n.rect.width * n.rect.height; + }; + + var areaCompareFcn = function areaCompareFcn(n1, n2) { + return getNodeArea(n2) - getNodeArea(n1); + }; + + // Sort the nodes in descending order of their areas + nodes.sort(function (n1, n2) { + var cmpBy = areaCompareFcn; + if (organization.idealRowWidth) { + cmpBy = tilingCompareBy; + return cmpBy(n1.id, n2.id); + } + return cmpBy(n1, n2); + }); + + // Create the organization -> calculate compound center + var sumCenterX = 0; + var sumCenterY = 0; + for (var i = 0; i < nodes.length; i++) { + var lNode = nodes[i]; + + sumCenterX += lNode.getCenterX(); + sumCenterY += lNode.getCenterY(); + } + + organization.centerX = sumCenterX / nodes.length; + organization.centerY = sumCenterY / nodes.length; + + // Create the organization -> tile members + for (var i = 0; i < nodes.length; i++) { + var lNode = nodes[i]; + + if (organization.rows.length == 0) { + this.insertNodeToRow(organization, lNode, 0, minWidth); + } else if (this.canAddHorizontal(organization, lNode.rect.width, lNode.rect.height)) { + var rowIndex = organization.rows.length - 1; + if (!organization.idealRowWidth) { + rowIndex = this.getShortestRowIndex(organization); + } + this.insertNodeToRow(organization, lNode, rowIndex, minWidth); + } else { + this.insertNodeToRow(organization, lNode, organization.rows.length, minWidth); + } + + this.shiftToLastRow(organization); + } + + return organization; +}; + +CoSELayout.prototype.insertNodeToRow = function (organization, node, rowIndex, minWidth) { + var minCompoundSize = minWidth; + + // Add new row if needed + if (rowIndex == organization.rows.length) { + var secondDimension = []; + + organization.rows.push(secondDimension); + organization.rowWidth.push(minCompoundSize); + organization.rowHeight.push(0); + } + + // Update row width + var w = organization.rowWidth[rowIndex] + node.rect.width; + + if (organization.rows[rowIndex].length > 0) { + w += organization.horizontalPadding; + } + + organization.rowWidth[rowIndex] = w; + // Update compound width + if (organization.width < w) { + organization.width = w; + } + + // Update height + var h = node.rect.height; + if (rowIndex > 0) h += organization.verticalPadding; + + var extraHeight = 0; + if (h > organization.rowHeight[rowIndex]) { + extraHeight = organization.rowHeight[rowIndex]; + organization.rowHeight[rowIndex] = h; + extraHeight = organization.rowHeight[rowIndex] - extraHeight; + } + + organization.height += extraHeight; + + // Insert node + organization.rows[rowIndex].push(node); +}; + +//Scans the rows of an organization and returns the one with the min width +CoSELayout.prototype.getShortestRowIndex = function (organization) { + var r = -1; + var min = Number.MAX_VALUE; + + for (var i = 0; i < organization.rows.length; i++) { + if (organization.rowWidth[i] < min) { + r = i; + min = organization.rowWidth[i]; + } + } + return r; +}; + +//Scans the rows of an organization and returns the one with the max width +CoSELayout.prototype.getLongestRowIndex = function (organization) { + var r = -1; + var max = Number.MIN_VALUE; + + for (var i = 0; i < organization.rows.length; i++) { + + if (organization.rowWidth[i] > max) { + r = i; + max = organization.rowWidth[i]; + } + } + + return r; +}; + +/** +* This method checks whether adding extra width to the organization violates +* the aspect ratio(1) or not. +*/ +CoSELayout.prototype.canAddHorizontal = function (organization, extraWidth, extraHeight) { + + // if there is an ideal row width specified use it instead of checking the aspect ratio + if (organization.idealRowWidth) { + var lastRowIndex = organization.rows.length - 1; + var lastRowWidth = organization.rowWidth[lastRowIndex]; + + // check and return if ideal row width will be exceed if the node is added to the row + return lastRowWidth + extraWidth + organization.horizontalPadding <= organization.idealRowWidth; + } + + var sri = this.getShortestRowIndex(organization); + + if (sri < 0) { + return true; + } + + var min = organization.rowWidth[sri]; + + if (min + organization.horizontalPadding + extraWidth <= organization.width) return true; + + var hDiff = 0; + + // Adding to an existing row + if (organization.rowHeight[sri] < extraHeight) { + if (sri > 0) hDiff = extraHeight + organization.verticalPadding - organization.rowHeight[sri]; + } + + var add_to_row_ratio; + if (organization.width - min >= extraWidth + organization.horizontalPadding) { + add_to_row_ratio = (organization.height + hDiff) / (min + extraWidth + organization.horizontalPadding); + } else { + add_to_row_ratio = (organization.height + hDiff) / organization.width; + } + + // Adding a new row for this node + hDiff = extraHeight + organization.verticalPadding; + var add_new_row_ratio; + if (organization.width < extraWidth) { + add_new_row_ratio = (organization.height + hDiff) / extraWidth; + } else { + add_new_row_ratio = (organization.height + hDiff) / organization.width; + } + + if (add_new_row_ratio < 1) add_new_row_ratio = 1 / add_new_row_ratio; + + if (add_to_row_ratio < 1) add_to_row_ratio = 1 / add_to_row_ratio; + + return add_to_row_ratio < add_new_row_ratio; +}; + +//If moving the last node from the longest row and adding it to the last +//row makes the bounding box smaller, do it. +CoSELayout.prototype.shiftToLastRow = function (organization) { + var longest = this.getLongestRowIndex(organization); + var last = organization.rowWidth.length - 1; + var row = organization.rows[longest]; + var node = row[row.length - 1]; + + var diff = node.width + organization.horizontalPadding; + + // Check if there is enough space on the last row + if (organization.width - organization.rowWidth[last] > diff && longest != last) { + // Remove the last element of the longest row + row.splice(-1, 1); + + // Push it to the last row + organization.rows[last].push(node); + + organization.rowWidth[longest] = organization.rowWidth[longest] - diff; + organization.rowWidth[last] = organization.rowWidth[last] + diff; + organization.width = organization.rowWidth[instance.getLongestRowIndex(organization)]; + + // Update heights of the organization + var maxHeight = Number.MIN_VALUE; + for (var i = 0; i < row.length; i++) { + if (row[i].height > maxHeight) maxHeight = row[i].height; + } + if (longest > 0) maxHeight += organization.verticalPadding; + + var prevTotal = organization.rowHeight[longest] + organization.rowHeight[last]; + + organization.rowHeight[longest] = maxHeight; + if (organization.rowHeight[last] < node.height + organization.verticalPadding) organization.rowHeight[last] = node.height + organization.verticalPadding; + + var finalTotal = organization.rowHeight[longest] + organization.rowHeight[last]; + organization.height += finalTotal - prevTotal; + + this.shiftToLastRow(organization); + } +}; + +CoSELayout.prototype.tilingPreLayout = function () { + if (CoSEConstants.TILE) { + // Find zero degree nodes and create a compound for each level + this.groupZeroDegreeMembers(); + // Tile and clear children of each compound + this.clearCompounds(); + // Separately tile and clear zero degree nodes for each level + this.clearZeroDegreeMembers(); + } +}; + +CoSELayout.prototype.tilingPostLayout = function () { + if (CoSEConstants.TILE) { + this.repopulateZeroDegreeMembers(); + this.repopulateCompounds(); + } +}; + +// ----------------------------------------------------------------------------- +// Section: Tree Reduction methods +// ----------------------------------------------------------------------------- +// Reduce trees +CoSELayout.prototype.reduceTrees = function () { + var prunedNodesAll = []; + var containsLeaf = true; + var node; + + while (containsLeaf) { + var allNodes = this.graphManager.getAllNodes(); + var prunedNodesInStepTemp = []; + containsLeaf = false; + + for (var i = 0; i < allNodes.length; i++) { + node = allNodes[i]; + if (node.getEdges().length == 1 && !node.getEdges()[0].isInterGraph && node.getChild() == null) { + if (CoSEConstants.PURE_INCREMENTAL) { + var otherEnd = node.getEdges()[0].getOtherEnd(node); + var relativePosition = new DimensionD(node.getCenterX() - otherEnd.getCenterX(), node.getCenterY() - otherEnd.getCenterY()); + prunedNodesInStepTemp.push([node, node.getEdges()[0], node.getOwner(), relativePosition]); + } else { + prunedNodesInStepTemp.push([node, node.getEdges()[0], node.getOwner()]); + } + containsLeaf = true; + } + } + if (containsLeaf == true) { + var prunedNodesInStep = []; + for (var j = 0; j < prunedNodesInStepTemp.length; j++) { + if (prunedNodesInStepTemp[j][0].getEdges().length == 1) { + prunedNodesInStep.push(prunedNodesInStepTemp[j]); + prunedNodesInStepTemp[j][0].getOwner().remove(prunedNodesInStepTemp[j][0]); + } + } + prunedNodesAll.push(prunedNodesInStep); + this.graphManager.resetAllNodes(); + this.graphManager.resetAllEdges(); + } + } + this.prunedNodesAll = prunedNodesAll; +}; + +// Grow tree one step +CoSELayout.prototype.growTree = function (prunedNodesAll) { + var lengthOfPrunedNodesInStep = prunedNodesAll.length; + var prunedNodesInStep = prunedNodesAll[lengthOfPrunedNodesInStep - 1]; + + var nodeData; + for (var i = 0; i < prunedNodesInStep.length; i++) { + nodeData = prunedNodesInStep[i]; + + this.findPlaceforPrunedNode(nodeData); + + nodeData[2].add(nodeData[0]); + nodeData[2].add(nodeData[1], nodeData[1].source, nodeData[1].target); + } + + prunedNodesAll.splice(prunedNodesAll.length - 1, 1); + this.graphManager.resetAllNodes(); + this.graphManager.resetAllEdges(); +}; + +// Find an appropriate position to replace pruned node, this method can be improved +CoSELayout.prototype.findPlaceforPrunedNode = function (nodeData) { + + var gridForPrunedNode; + var nodeToConnect; + var prunedNode = nodeData[0]; + if (prunedNode == nodeData[1].source) { + nodeToConnect = nodeData[1].target; + } else { + nodeToConnect = nodeData[1].source; + } + + if (CoSEConstants.PURE_INCREMENTAL) { + prunedNode.setCenter(nodeToConnect.getCenterX() + nodeData[3].getWidth(), nodeToConnect.getCenterY() + nodeData[3].getHeight()); + } else { + var startGridX = nodeToConnect.startX; + var finishGridX = nodeToConnect.finishX; + var startGridY = nodeToConnect.startY; + var finishGridY = nodeToConnect.finishY; + + var upNodeCount = 0; + var downNodeCount = 0; + var rightNodeCount = 0; + var leftNodeCount = 0; + var controlRegions = [upNodeCount, rightNodeCount, downNodeCount, leftNodeCount]; + + if (startGridY > 0) { + for (var i = startGridX; i <= finishGridX; i++) { + controlRegions[0] += this.grid[i][startGridY - 1].length + this.grid[i][startGridY].length - 1; + } + } + if (finishGridX < this.grid.length - 1) { + for (var i = startGridY; i <= finishGridY; i++) { + controlRegions[1] += this.grid[finishGridX + 1][i].length + this.grid[finishGridX][i].length - 1; + } + } + if (finishGridY < this.grid[0].length - 1) { + for (var i = startGridX; i <= finishGridX; i++) { + controlRegions[2] += this.grid[i][finishGridY + 1].length + this.grid[i][finishGridY].length - 1; + } + } + if (startGridX > 0) { + for (var i = startGridY; i <= finishGridY; i++) { + controlRegions[3] += this.grid[startGridX - 1][i].length + this.grid[startGridX][i].length - 1; + } + } + var min = Integer.MAX_VALUE; + var minCount; + var minIndex; + for (var j = 0; j < controlRegions.length; j++) { + if (controlRegions[j] < min) { + min = controlRegions[j]; + minCount = 1; + minIndex = j; + } else if (controlRegions[j] == min) { + minCount++; + } + } + + if (minCount == 3 && min == 0) { + if (controlRegions[0] == 0 && controlRegions[1] == 0 && controlRegions[2] == 0) { + gridForPrunedNode = 1; + } else if (controlRegions[0] == 0 && controlRegions[1] == 0 && controlRegions[3] == 0) { + gridForPrunedNode = 0; + } else if (controlRegions[0] == 0 && controlRegions[2] == 0 && controlRegions[3] == 0) { + gridForPrunedNode = 3; + } else if (controlRegions[1] == 0 && controlRegions[2] == 0 && controlRegions[3] == 0) { + gridForPrunedNode = 2; + } + } else if (minCount == 2 && min == 0) { + var random = Math.floor(Math.random() * 2); + if (controlRegions[0] == 0 && controlRegions[1] == 0) { + ; + if (random == 0) { + gridForPrunedNode = 0; + } else { + gridForPrunedNode = 1; + } + } else if (controlRegions[0] == 0 && controlRegions[2] == 0) { + if (random == 0) { + gridForPrunedNode = 0; + } else { + gridForPrunedNode = 2; + } + } else if (controlRegions[0] == 0 && controlRegions[3] == 0) { + if (random == 0) { + gridForPrunedNode = 0; + } else { + gridForPrunedNode = 3; + } + } else if (controlRegions[1] == 0 && controlRegions[2] == 0) { + if (random == 0) { + gridForPrunedNode = 1; + } else { + gridForPrunedNode = 2; + } + } else if (controlRegions[1] == 0 && controlRegions[3] == 0) { + if (random == 0) { + gridForPrunedNode = 1; + } else { + gridForPrunedNode = 3; + } + } else { + if (random == 0) { + gridForPrunedNode = 2; + } else { + gridForPrunedNode = 3; + } + } + } else if (minCount == 4 && min == 0) { + var random = Math.floor(Math.random() * 4); + gridForPrunedNode = random; + } else { + gridForPrunedNode = minIndex; + } + + if (gridForPrunedNode == 0) { + prunedNode.setCenter(nodeToConnect.getCenterX(), nodeToConnect.getCenterY() - nodeToConnect.getHeight() / 2 - FDLayoutConstants.DEFAULT_EDGE_LENGTH - prunedNode.getHeight() / 2); + } else if (gridForPrunedNode == 1) { + prunedNode.setCenter(nodeToConnect.getCenterX() + nodeToConnect.getWidth() / 2 + FDLayoutConstants.DEFAULT_EDGE_LENGTH + prunedNode.getWidth() / 2, nodeToConnect.getCenterY()); + } else if (gridForPrunedNode == 2) { + prunedNode.setCenter(nodeToConnect.getCenterX(), nodeToConnect.getCenterY() + nodeToConnect.getHeight() / 2 + FDLayoutConstants.DEFAULT_EDGE_LENGTH + prunedNode.getHeight() / 2); + } else { + prunedNode.setCenter(nodeToConnect.getCenterX() - nodeToConnect.getWidth() / 2 - FDLayoutConstants.DEFAULT_EDGE_LENGTH - prunedNode.getWidth() / 2, nodeToConnect.getCenterY()); + } + } +}; + +module.exports = CoSELayout; + +/***/ }), + +/***/ 991: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var FDLayoutNode = __webpack_require__(551).FDLayoutNode; +var IMath = __webpack_require__(551).IMath; + +function CoSENode(gm, loc, size, vNode) { + FDLayoutNode.call(this, gm, loc, size, vNode); +} + +CoSENode.prototype = Object.create(FDLayoutNode.prototype); +for (var prop in FDLayoutNode) { + CoSENode[prop] = FDLayoutNode[prop]; +} + +CoSENode.prototype.calculateDisplacement = function () { + var layout = this.graphManager.getLayout(); + // this check is for compound nodes that contain fixed nodes + if (this.getChild() != null && this.fixedNodeWeight) { + this.displacementX += layout.coolingFactor * (this.springForceX + this.repulsionForceX + this.gravitationForceX) / this.fixedNodeWeight; + this.displacementY += layout.coolingFactor * (this.springForceY + this.repulsionForceY + this.gravitationForceY) / this.fixedNodeWeight; + } else { + this.displacementX += layout.coolingFactor * (this.springForceX + this.repulsionForceX + this.gravitationForceX) / this.noOfChildren; + this.displacementY += layout.coolingFactor * (this.springForceY + this.repulsionForceY + this.gravitationForceY) / this.noOfChildren; + } + + if (Math.abs(this.displacementX) > layout.coolingFactor * layout.maxNodeDisplacement) { + this.displacementX = layout.coolingFactor * layout.maxNodeDisplacement * IMath.sign(this.displacementX); + } + + if (Math.abs(this.displacementY) > layout.coolingFactor * layout.maxNodeDisplacement) { + this.displacementY = layout.coolingFactor * layout.maxNodeDisplacement * IMath.sign(this.displacementY); + } + + // non-empty compound node, propogate movement to children as well + if (this.child && this.child.getNodes().length > 0) { + this.propogateDisplacementToChildren(this.displacementX, this.displacementY); + } +}; + +CoSENode.prototype.propogateDisplacementToChildren = function (dX, dY) { + var nodes = this.getChild().getNodes(); + var node; + for (var i = 0; i < nodes.length; i++) { + node = nodes[i]; + if (node.getChild() == null) { + node.displacementX += dX; + node.displacementY += dY; + } else { + node.propogateDisplacementToChildren(dX, dY); + } + } +}; + +CoSENode.prototype.move = function () { + var layout = this.graphManager.getLayout(); + + // a simple node or an empty compound node, move it + if (this.child == null || this.child.getNodes().length == 0) { + this.moveBy(this.displacementX, this.displacementY); + + layout.totalDisplacement += Math.abs(this.displacementX) + Math.abs(this.displacementY); + } + + this.springForceX = 0; + this.springForceY = 0; + this.repulsionForceX = 0; + this.repulsionForceY = 0; + this.gravitationForceX = 0; + this.gravitationForceY = 0; + this.displacementX = 0; + this.displacementY = 0; +}; + +CoSENode.prototype.setPred1 = function (pred1) { + this.pred1 = pred1; +}; + +CoSENode.prototype.getPred1 = function () { + return pred1; +}; + +CoSENode.prototype.getPred2 = function () { + return pred2; +}; + +CoSENode.prototype.setNext = function (next) { + this.next = next; +}; + +CoSENode.prototype.getNext = function () { + return next; +}; + +CoSENode.prototype.setProcessed = function (processed) { + this.processed = processed; +}; + +CoSENode.prototype.isProcessed = function () { + return processed; +}; + +module.exports = CoSENode; + +/***/ }), + +/***/ 902: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } + +var CoSEConstants = __webpack_require__(806); +var LinkedList = __webpack_require__(551).LinkedList; +var Matrix = __webpack_require__(551).Matrix; +var SVD = __webpack_require__(551).SVD; + +function ConstraintHandler() {} + +ConstraintHandler.handleConstraints = function (layout) { + // let layout = this.graphManager.getLayout(); + + // get constraints from layout + var constraints = {}; + constraints.fixedNodeConstraint = layout.constraints.fixedNodeConstraint; + constraints.alignmentConstraint = layout.constraints.alignmentConstraint; + constraints.relativePlacementConstraint = layout.constraints.relativePlacementConstraint; + + var idToNodeMap = new Map(); + var nodeIndexes = new Map(); + var xCoords = []; + var yCoords = []; + + var allNodes = layout.getAllNodes(); + var index = 0; + // fill index map and coordinates + for (var i = 0; i < allNodes.length; i++) { + var node = allNodes[i]; + if (node.getChild() == null) { + nodeIndexes.set(node.id, index++); + xCoords.push(node.getCenterX()); + yCoords.push(node.getCenterY()); + idToNodeMap.set(node.id, node); + } + } + + // if there exists relative placement constraint without gap value, set it to default + if (constraints.relativePlacementConstraint) { + constraints.relativePlacementConstraint.forEach(function (constraint) { + if (!constraint.gap && constraint.gap != 0) { + if (constraint.left) { + constraint.gap = CoSEConstants.DEFAULT_EDGE_LENGTH + idToNodeMap.get(constraint.left).getWidth() / 2 + idToNodeMap.get(constraint.right).getWidth() / 2; + } else { + constraint.gap = CoSEConstants.DEFAULT_EDGE_LENGTH + idToNodeMap.get(constraint.top).getHeight() / 2 + idToNodeMap.get(constraint.bottom).getHeight() / 2; + } + } + }); + } + + /* auxiliary functions */ + + // calculate difference between two position objects + var calculatePositionDiff = function calculatePositionDiff(pos1, pos2) { + return { x: pos1.x - pos2.x, y: pos1.y - pos2.y }; + }; + + // calculate average position of the nodes + var calculateAvgPosition = function calculateAvgPosition(nodeIdSet) { + var xPosSum = 0; + var yPosSum = 0; + nodeIdSet.forEach(function (nodeId) { + xPosSum += xCoords[nodeIndexes.get(nodeId)]; + yPosSum += yCoords[nodeIndexes.get(nodeId)]; + }); + + return { x: xPosSum / nodeIdSet.size, y: yPosSum / nodeIdSet.size }; + }; + + // find an appropriate positioning for the nodes in a given graph according to relative placement constraints + // this function also takes the fixed nodes and alignment constraints into account + // graph: dag to be evaluated, direction: "horizontal" or "vertical", + // fixedNodes: set of fixed nodes to consider during evaluation, dummyPositions: appropriate coordinates of the dummy nodes + var findAppropriatePositionForRelativePlacement = function findAppropriatePositionForRelativePlacement(graph, direction, fixedNodes, dummyPositions, componentSources) { + + // find union of two sets + function setUnion(setA, setB) { + var union = new Set(setA); + var _iteratorNormalCompletion = true; + var _didIteratorError = false; + var _iteratorError = undefined; + + try { + for (var _iterator = setB[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) { + var elem = _step.value; + + union.add(elem); + } + } catch (err) { + _didIteratorError = true; + _iteratorError = err; + } finally { + try { + if (!_iteratorNormalCompletion && _iterator.return) { + _iterator.return(); + } + } finally { + if (_didIteratorError) { + throw _iteratorError; + } + } + } + + return union; + } + + // find indegree count for each node + var inDegrees = new Map(); + + graph.forEach(function (value, key) { + inDegrees.set(key, 0); + }); + graph.forEach(function (value, key) { + value.forEach(function (adjacent) { + inDegrees.set(adjacent.id, inDegrees.get(adjacent.id) + 1); + }); + }); + + var positionMap = new Map(); // keeps the position for each node + var pastMap = new Map(); // keeps the predecessors(past) of a node + var queue = new LinkedList(); + inDegrees.forEach(function (value, key) { + if (value == 0) { + queue.push(key); + if (!fixedNodes) { + if (direction == "horizontal") { + positionMap.set(key, nodeIndexes.has(key) ? xCoords[nodeIndexes.get(key)] : dummyPositions.get(key)); + } else { + positionMap.set(key, nodeIndexes.has(key) ? yCoords[nodeIndexes.get(key)] : dummyPositions.get(key)); + } + } + } else { + positionMap.set(key, Number.NEGATIVE_INFINITY); + } + if (fixedNodes) { + pastMap.set(key, new Set([key])); + } + }); + + // align sources of each component in enforcement phase + if (fixedNodes) { + componentSources.forEach(function (component) { + var fixedIds = []; + component.forEach(function (nodeId) { + if (fixedNodes.has(nodeId)) { + fixedIds.push(nodeId); + } + }); + if (fixedIds.length > 0) { + var position = 0; + fixedIds.forEach(function (fixedId) { + if (direction == "horizontal") { + positionMap.set(fixedId, nodeIndexes.has(fixedId) ? xCoords[nodeIndexes.get(fixedId)] : dummyPositions.get(fixedId)); + position += positionMap.get(fixedId); + } else { + positionMap.set(fixedId, nodeIndexes.has(fixedId) ? yCoords[nodeIndexes.get(fixedId)] : dummyPositions.get(fixedId)); + position += positionMap.get(fixedId); + } + }); + position = position / fixedIds.length; + component.forEach(function (nodeId) { + if (!fixedNodes.has(nodeId)) { + positionMap.set(nodeId, position); + } + }); + } else { + var _position = 0; + component.forEach(function (nodeId) { + if (direction == "horizontal") { + _position += nodeIndexes.has(nodeId) ? xCoords[nodeIndexes.get(nodeId)] : dummyPositions.get(nodeId); + } else { + _position += nodeIndexes.has(nodeId) ? yCoords[nodeIndexes.get(nodeId)] : dummyPositions.get(nodeId); + } + }); + _position = _position / component.length; + component.forEach(function (nodeId) { + positionMap.set(nodeId, _position); + }); + } + }); + } + + // calculate positions of the nodes + + var _loop = function _loop() { + var currentNode = queue.shift(); + var neighbors = graph.get(currentNode); + neighbors.forEach(function (neighbor) { + if (positionMap.get(neighbor.id) < positionMap.get(currentNode) + neighbor.gap) { + if (fixedNodes && fixedNodes.has(neighbor.id)) { + var fixedPosition = void 0; + if (direction == "horizontal") { + fixedPosition = nodeIndexes.has(neighbor.id) ? xCoords[nodeIndexes.get(neighbor.id)] : dummyPositions.get(neighbor.id); + } else { + fixedPosition = nodeIndexes.has(neighbor.id) ? yCoords[nodeIndexes.get(neighbor.id)] : dummyPositions.get(neighbor.id); + } + positionMap.set(neighbor.id, fixedPosition); // TODO: may do unnecessary work + if (fixedPosition < positionMap.get(currentNode) + neighbor.gap) { + var diff = positionMap.get(currentNode) + neighbor.gap - fixedPosition; + pastMap.get(currentNode).forEach(function (nodeId) { + positionMap.set(nodeId, positionMap.get(nodeId) - diff); + }); + } + } else { + positionMap.set(neighbor.id, positionMap.get(currentNode) + neighbor.gap); + } + } + inDegrees.set(neighbor.id, inDegrees.get(neighbor.id) - 1); + if (inDegrees.get(neighbor.id) == 0) { + queue.push(neighbor.id); + } + if (fixedNodes) { + pastMap.set(neighbor.id, setUnion(pastMap.get(currentNode), pastMap.get(neighbor.id))); + } + }); + }; + + while (queue.length != 0) { + _loop(); + } + + // readjust position of the nodes after enforcement + if (fixedNodes) { + // find indegree count for each node + var sinkNodes = new Set(); + + graph.forEach(function (value, key) { + if (value.length == 0) { + sinkNodes.add(key); + } + }); + + var _components = []; + pastMap.forEach(function (value, key) { + if (sinkNodes.has(key)) { + var isFixedComponent = false; + var _iteratorNormalCompletion2 = true; + var _didIteratorError2 = false; + var _iteratorError2 = undefined; + + try { + for (var _iterator2 = value[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true) { + var nodeId = _step2.value; + + if (fixedNodes.has(nodeId)) { + isFixedComponent = true; + } + } + } catch (err) { + _didIteratorError2 = true; + _iteratorError2 = err; + } finally { + try { + if (!_iteratorNormalCompletion2 && _iterator2.return) { + _iterator2.return(); + } + } finally { + if (_didIteratorError2) { + throw _iteratorError2; + } + } + } + + if (!isFixedComponent) { + var isExist = false; + var existAt = void 0; + _components.forEach(function (component, index) { + if (component.has([].concat(_toConsumableArray(value))[0])) { + isExist = true; + existAt = index; + } + }); + if (!isExist) { + _components.push(new Set(value)); + } else { + value.forEach(function (ele) { + _components[existAt].add(ele); + }); + } + } + } + }); + + _components.forEach(function (component, index) { + var minBefore = Number.POSITIVE_INFINITY; + var minAfter = Number.POSITIVE_INFINITY; + var maxBefore = Number.NEGATIVE_INFINITY; + var maxAfter = Number.NEGATIVE_INFINITY; + + var _iteratorNormalCompletion3 = true; + var _didIteratorError3 = false; + var _iteratorError3 = undefined; + + try { + for (var _iterator3 = component[Symbol.iterator](), _step3; !(_iteratorNormalCompletion3 = (_step3 = _iterator3.next()).done); _iteratorNormalCompletion3 = true) { + var nodeId = _step3.value; + + var posBefore = void 0; + if (direction == "horizontal") { + posBefore = nodeIndexes.has(nodeId) ? xCoords[nodeIndexes.get(nodeId)] : dummyPositions.get(nodeId); + } else { + posBefore = nodeIndexes.has(nodeId) ? yCoords[nodeIndexes.get(nodeId)] : dummyPositions.get(nodeId); + } + var posAfter = positionMap.get(nodeId); + if (posBefore < minBefore) { + minBefore = posBefore; + } + if (posBefore > maxBefore) { + maxBefore = posBefore; + } + if (posAfter < minAfter) { + minAfter = posAfter; + } + if (posAfter > maxAfter) { + maxAfter = posAfter; + } + } + } catch (err) { + _didIteratorError3 = true; + _iteratorError3 = err; + } finally { + try { + if (!_iteratorNormalCompletion3 && _iterator3.return) { + _iterator3.return(); + } + } finally { + if (_didIteratorError3) { + throw _iteratorError3; + } + } + } + + var diff = (minBefore + maxBefore) / 2 - (minAfter + maxAfter) / 2; + + var _iteratorNormalCompletion4 = true; + var _didIteratorError4 = false; + var _iteratorError4 = undefined; + + try { + for (var _iterator4 = component[Symbol.iterator](), _step4; !(_iteratorNormalCompletion4 = (_step4 = _iterator4.next()).done); _iteratorNormalCompletion4 = true) { + var _nodeId = _step4.value; + + positionMap.set(_nodeId, positionMap.get(_nodeId) + diff); + } + } catch (err) { + _didIteratorError4 = true; + _iteratorError4 = err; + } finally { + try { + if (!_iteratorNormalCompletion4 && _iterator4.return) { + _iterator4.return(); + } + } finally { + if (_didIteratorError4) { + throw _iteratorError4; + } + } + } + }); + } + + return positionMap; + }; + + // find transformation based on rel. placement constraints if there are both alignment and rel. placement constraints + // or if there are only rel. placement contraints where the largest component isn't sufficiently large + var applyReflectionForRelativePlacement = function applyReflectionForRelativePlacement(relativePlacementConstraints) { + // variables to count votes + var reflectOnY = 0, + notReflectOnY = 0; + var reflectOnX = 0, + notReflectOnX = 0; + + relativePlacementConstraints.forEach(function (constraint) { + if (constraint.left) { + xCoords[nodeIndexes.get(constraint.left)] - xCoords[nodeIndexes.get(constraint.right)] >= 0 ? reflectOnY++ : notReflectOnY++; + } else { + yCoords[nodeIndexes.get(constraint.top)] - yCoords[nodeIndexes.get(constraint.bottom)] >= 0 ? reflectOnX++ : notReflectOnX++; + } + }); + + if (reflectOnY > notReflectOnY && reflectOnX > notReflectOnX) { + for (var _i = 0; _i < nodeIndexes.size; _i++) { + xCoords[_i] = -1 * xCoords[_i]; + yCoords[_i] = -1 * yCoords[_i]; + } + } else if (reflectOnY > notReflectOnY) { + for (var _i2 = 0; _i2 < nodeIndexes.size; _i2++) { + xCoords[_i2] = -1 * xCoords[_i2]; + } + } else if (reflectOnX > notReflectOnX) { + for (var _i3 = 0; _i3 < nodeIndexes.size; _i3++) { + yCoords[_i3] = -1 * yCoords[_i3]; + } + } + }; + + // find weakly connected components in undirected graph + var findComponents = function findComponents(graph) { + // find weakly connected components in dag + var components = []; + var queue = new LinkedList(); + var visited = new Set(); + var count = 0; + + graph.forEach(function (value, key) { + if (!visited.has(key)) { + components[count] = []; + var _currentNode = key; + queue.push(_currentNode); + visited.add(_currentNode); + components[count].push(_currentNode); + + while (queue.length != 0) { + _currentNode = queue.shift(); + var neighbors = graph.get(_currentNode); + neighbors.forEach(function (neighbor) { + if (!visited.has(neighbor.id)) { + queue.push(neighbor.id); + visited.add(neighbor.id); + components[count].push(neighbor.id); + } + }); + } + count++; + } + }); + return components; + }; + + // return undirected version of given dag + var dagToUndirected = function dagToUndirected(dag) { + var undirected = new Map(); + + dag.forEach(function (value, key) { + undirected.set(key, []); + }); + + dag.forEach(function (value, key) { + value.forEach(function (adjacent) { + undirected.get(key).push(adjacent); + undirected.get(adjacent.id).push({ id: key, gap: adjacent.gap, direction: adjacent.direction }); + }); + }); + + return undirected; + }; + + // return reversed (directions inverted) version of given dag + var dagToReversed = function dagToReversed(dag) { + var reversed = new Map(); + + dag.forEach(function (value, key) { + reversed.set(key, []); + }); + + dag.forEach(function (value, key) { + value.forEach(function (adjacent) { + reversed.get(adjacent.id).push({ id: key, gap: adjacent.gap, direction: adjacent.direction }); + }); + }); + + return reversed; + }; + + /**** apply transformation to the initial draft layout to better align with constrained nodes ****/ + // solve the Orthogonal Procrustean Problem to rotate and/or reflect initial draft layout + // here we follow the solution in Chapter 20.2 of Borg, I. & Groenen, P. (2005) Modern Multidimensional Scaling: Theory and Applications + + /* construct source and target configurations */ + + var targetMatrix = []; // A - target configuration + var sourceMatrix = []; // B - source configuration + var standardTransformation = false; // false for no transformation, true for standart (Procrustes) transformation (rotation and/or reflection) + var reflectionType = false; // false/true for reflection check, 'reflectOnX', 'reflectOnY' or 'reflectOnBoth' for reflection type if necessary + var fixedNodes = new Set(); + var dag = new Map(); // adjacency list to keep directed acyclic graph (dag) that consists of relative placement constraints + var dagUndirected = new Map(); // undirected version of the dag + var components = []; // weakly connected components + + // fill fixedNodes collection to use later + if (constraints.fixedNodeConstraint) { + constraints.fixedNodeConstraint.forEach(function (nodeData) { + fixedNodes.add(nodeData.nodeId); + }); + } + + // construct dag from relative placement constraints + if (constraints.relativePlacementConstraint) { + // construct both directed and undirected version of the dag + constraints.relativePlacementConstraint.forEach(function (constraint) { + if (constraint.left) { + if (dag.has(constraint.left)) { + dag.get(constraint.left).push({ id: constraint.right, gap: constraint.gap, direction: "horizontal" }); + } else { + dag.set(constraint.left, [{ id: constraint.right, gap: constraint.gap, direction: "horizontal" }]); + } + if (!dag.has(constraint.right)) { + dag.set(constraint.right, []); + } + } else { + if (dag.has(constraint.top)) { + dag.get(constraint.top).push({ id: constraint.bottom, gap: constraint.gap, direction: "vertical" }); + } else { + dag.set(constraint.top, [{ id: constraint.bottom, gap: constraint.gap, direction: "vertical" }]); + } + if (!dag.has(constraint.bottom)) { + dag.set(constraint.bottom, []); + } + } + }); + + dagUndirected = dagToUndirected(dag); + components = findComponents(dagUndirected); + } + + if (CoSEConstants.TRANSFORM_ON_CONSTRAINT_HANDLING) { + // first check fixed node constraint + if (constraints.fixedNodeConstraint && constraints.fixedNodeConstraint.length > 1) { + constraints.fixedNodeConstraint.forEach(function (nodeData, i) { + targetMatrix[i] = [nodeData.position.x, nodeData.position.y]; + sourceMatrix[i] = [xCoords[nodeIndexes.get(nodeData.nodeId)], yCoords[nodeIndexes.get(nodeData.nodeId)]]; + }); + standardTransformation = true; + } else if (constraints.alignmentConstraint) { + (function () { + // then check alignment constraint + var count = 0; + if (constraints.alignmentConstraint.vertical) { + var verticalAlign = constraints.alignmentConstraint.vertical; + + var _loop2 = function _loop2(_i4) { + var alignmentSet = new Set(); + verticalAlign[_i4].forEach(function (nodeId) { + alignmentSet.add(nodeId); + }); + var intersection = new Set([].concat(_toConsumableArray(alignmentSet)).filter(function (x) { + return fixedNodes.has(x); + })); + var xPos = void 0; + if (intersection.size > 0) xPos = xCoords[nodeIndexes.get(intersection.values().next().value)];else xPos = calculateAvgPosition(alignmentSet).x; + + verticalAlign[_i4].forEach(function (nodeId) { + targetMatrix[count] = [xPos, yCoords[nodeIndexes.get(nodeId)]]; + sourceMatrix[count] = [xCoords[nodeIndexes.get(nodeId)], yCoords[nodeIndexes.get(nodeId)]]; + count++; + }); + }; + + for (var _i4 = 0; _i4 < verticalAlign.length; _i4++) { + _loop2(_i4); + } + standardTransformation = true; + } + if (constraints.alignmentConstraint.horizontal) { + var horizontalAlign = constraints.alignmentConstraint.horizontal; + + var _loop3 = function _loop3(_i5) { + var alignmentSet = new Set(); + horizontalAlign[_i5].forEach(function (nodeId) { + alignmentSet.add(nodeId); + }); + var intersection = new Set([].concat(_toConsumableArray(alignmentSet)).filter(function (x) { + return fixedNodes.has(x); + })); + var yPos = void 0; + if (intersection.size > 0) yPos = xCoords[nodeIndexes.get(intersection.values().next().value)];else yPos = calculateAvgPosition(alignmentSet).y; + + horizontalAlign[_i5].forEach(function (nodeId) { + targetMatrix[count] = [xCoords[nodeIndexes.get(nodeId)], yPos]; + sourceMatrix[count] = [xCoords[nodeIndexes.get(nodeId)], yCoords[nodeIndexes.get(nodeId)]]; + count++; + }); + }; + + for (var _i5 = 0; _i5 < horizontalAlign.length; _i5++) { + _loop3(_i5); + } + standardTransformation = true; + } + if (constraints.relativePlacementConstraint) { + reflectionType = true; + } + })(); + } else if (constraints.relativePlacementConstraint) { + // finally check relative placement constraint + // find largest component in dag + var largestComponentSize = 0; + var largestComponentIndex = 0; + for (var _i6 = 0; _i6 < components.length; _i6++) { + if (components[_i6].length > largestComponentSize) { + largestComponentSize = components[_i6].length; + largestComponentIndex = _i6; + } + } + // if largest component isn't dominant, then take the votes for reflection + if (largestComponentSize < dagUndirected.size / 2) { + applyReflectionForRelativePlacement(constraints.relativePlacementConstraint); + standardTransformation = false; + reflectionType = false; + } else { + // use largest component for transformation + // construct horizontal and vertical subgraphs in the largest component + var subGraphOnHorizontal = new Map(); + var subGraphOnVertical = new Map(); + var constraintsInlargestComponent = []; + + components[largestComponentIndex].forEach(function (nodeId) { + dag.get(nodeId).forEach(function (adjacent) { + if (adjacent.direction == "horizontal") { + if (subGraphOnHorizontal.has(nodeId)) { + subGraphOnHorizontal.get(nodeId).push(adjacent); + } else { + subGraphOnHorizontal.set(nodeId, [adjacent]); + } + if (!subGraphOnHorizontal.has(adjacent.id)) { + subGraphOnHorizontal.set(adjacent.id, []); + } + constraintsInlargestComponent.push({ left: nodeId, right: adjacent.id }); + } else { + if (subGraphOnVertical.has(nodeId)) { + subGraphOnVertical.get(nodeId).push(adjacent); + } else { + subGraphOnVertical.set(nodeId, [adjacent]); + } + if (!subGraphOnVertical.has(adjacent.id)) { + subGraphOnVertical.set(adjacent.id, []); + } + constraintsInlargestComponent.push({ top: nodeId, bottom: adjacent.id }); + } + }); + }); + + applyReflectionForRelativePlacement(constraintsInlargestComponent); + reflectionType = false; + + // calculate appropriate positioning for subgraphs + var positionMapHorizontal = findAppropriatePositionForRelativePlacement(subGraphOnHorizontal, "horizontal"); + var positionMapVertical = findAppropriatePositionForRelativePlacement(subGraphOnVertical, "vertical"); + + // construct source and target configuration + components[largestComponentIndex].forEach(function (nodeId, i) { + sourceMatrix[i] = [xCoords[nodeIndexes.get(nodeId)], yCoords[nodeIndexes.get(nodeId)]]; + targetMatrix[i] = []; + if (positionMapHorizontal.has(nodeId)) { + targetMatrix[i][0] = positionMapHorizontal.get(nodeId); + } else { + targetMatrix[i][0] = xCoords[nodeIndexes.get(nodeId)]; + } + if (positionMapVertical.has(nodeId)) { + targetMatrix[i][1] = positionMapVertical.get(nodeId); + } else { + targetMatrix[i][1] = yCoords[nodeIndexes.get(nodeId)]; + } + }); + + standardTransformation = true; + } + } + + // if transformation is required, then calculate and apply transformation matrix + if (standardTransformation) { + /* calculate transformation matrix */ + var transformationMatrix = void 0; + var targetMatrixTranspose = Matrix.transpose(targetMatrix); // A' + var sourceMatrixTranspose = Matrix.transpose(sourceMatrix); // B' + + // centralize transpose matrices + for (var _i7 = 0; _i7 < targetMatrixTranspose.length; _i7++) { + targetMatrixTranspose[_i7] = Matrix.multGamma(targetMatrixTranspose[_i7]); + sourceMatrixTranspose[_i7] = Matrix.multGamma(sourceMatrixTranspose[_i7]); + } + + // do actual calculation for transformation matrix + var tempMatrix = Matrix.multMat(targetMatrixTranspose, Matrix.transpose(sourceMatrixTranspose)); // tempMatrix = A'B + var SVDResult = SVD.svd(tempMatrix); // SVD(A'B) = USV', svd function returns U, S and V + transformationMatrix = Matrix.multMat(SVDResult.V, Matrix.transpose(SVDResult.U)); // transformationMatrix = T = VU' + + /* apply found transformation matrix to obtain final draft layout */ + for (var _i8 = 0; _i8 < nodeIndexes.size; _i8++) { + var temp1 = [xCoords[_i8], yCoords[_i8]]; + var temp2 = [transformationMatrix[0][0], transformationMatrix[1][0]]; + var temp3 = [transformationMatrix[0][1], transformationMatrix[1][1]]; + xCoords[_i8] = Matrix.dotProduct(temp1, temp2); + yCoords[_i8] = Matrix.dotProduct(temp1, temp3); + } + + // applied only both alignment and rel. placement constraints exist + if (reflectionType) { + applyReflectionForRelativePlacement(constraints.relativePlacementConstraint); + } + } + } + + if (CoSEConstants.ENFORCE_CONSTRAINTS) { + /**** enforce constraints on the transformed draft layout ****/ + + /* first enforce fixed node constraint */ + + if (constraints.fixedNodeConstraint && constraints.fixedNodeConstraint.length > 0) { + var translationAmount = { x: 0, y: 0 }; + constraints.fixedNodeConstraint.forEach(function (nodeData, i) { + var posInTheory = { x: xCoords[nodeIndexes.get(nodeData.nodeId)], y: yCoords[nodeIndexes.get(nodeData.nodeId)] }; + var posDesired = nodeData.position; + var posDiff = calculatePositionDiff(posDesired, posInTheory); + translationAmount.x += posDiff.x; + translationAmount.y += posDiff.y; + }); + translationAmount.x /= constraints.fixedNodeConstraint.length; + translationAmount.y /= constraints.fixedNodeConstraint.length; + + xCoords.forEach(function (value, i) { + xCoords[i] += translationAmount.x; + }); + + yCoords.forEach(function (value, i) { + yCoords[i] += translationAmount.y; + }); + + constraints.fixedNodeConstraint.forEach(function (nodeData) { + xCoords[nodeIndexes.get(nodeData.nodeId)] = nodeData.position.x; + yCoords[nodeIndexes.get(nodeData.nodeId)] = nodeData.position.y; + }); + } + + /* then enforce alignment constraint */ + + if (constraints.alignmentConstraint) { + if (constraints.alignmentConstraint.vertical) { + var xAlign = constraints.alignmentConstraint.vertical; + + var _loop4 = function _loop4(_i9) { + var alignmentSet = new Set(); + xAlign[_i9].forEach(function (nodeId) { + alignmentSet.add(nodeId); + }); + var intersection = new Set([].concat(_toConsumableArray(alignmentSet)).filter(function (x) { + return fixedNodes.has(x); + })); + var xPos = void 0; + if (intersection.size > 0) xPos = xCoords[nodeIndexes.get(intersection.values().next().value)];else xPos = calculateAvgPosition(alignmentSet).x; + + alignmentSet.forEach(function (nodeId) { + if (!fixedNodes.has(nodeId)) xCoords[nodeIndexes.get(nodeId)] = xPos; + }); + }; + + for (var _i9 = 0; _i9 < xAlign.length; _i9++) { + _loop4(_i9); + } + } + if (constraints.alignmentConstraint.horizontal) { + var yAlign = constraints.alignmentConstraint.horizontal; + + var _loop5 = function _loop5(_i10) { + var alignmentSet = new Set(); + yAlign[_i10].forEach(function (nodeId) { + alignmentSet.add(nodeId); + }); + var intersection = new Set([].concat(_toConsumableArray(alignmentSet)).filter(function (x) { + return fixedNodes.has(x); + })); + var yPos = void 0; + if (intersection.size > 0) yPos = yCoords[nodeIndexes.get(intersection.values().next().value)];else yPos = calculateAvgPosition(alignmentSet).y; + + alignmentSet.forEach(function (nodeId) { + if (!fixedNodes.has(nodeId)) yCoords[nodeIndexes.get(nodeId)] = yPos; + }); + }; + + for (var _i10 = 0; _i10 < yAlign.length; _i10++) { + _loop5(_i10); + } + } + } + + /* finally enforce relative placement constraint */ + + if (constraints.relativePlacementConstraint) { + (function () { + var nodeToDummyForVerticalAlignment = new Map(); + var nodeToDummyForHorizontalAlignment = new Map(); + var dummyToNodeForVerticalAlignment = new Map(); + var dummyToNodeForHorizontalAlignment = new Map(); + var dummyPositionsForVerticalAlignment = new Map(); + var dummyPositionsForHorizontalAlignment = new Map(); + var fixedNodesOnHorizontal = new Set(); + var fixedNodesOnVertical = new Set(); + + // fill maps and sets + fixedNodes.forEach(function (nodeId) { + fixedNodesOnHorizontal.add(nodeId); + fixedNodesOnVertical.add(nodeId); + }); + + if (constraints.alignmentConstraint) { + if (constraints.alignmentConstraint.vertical) { + var verticalAlignment = constraints.alignmentConstraint.vertical; + + var _loop6 = function _loop6(_i11) { + dummyToNodeForVerticalAlignment.set("dummy" + _i11, []); + verticalAlignment[_i11].forEach(function (nodeId) { + nodeToDummyForVerticalAlignment.set(nodeId, "dummy" + _i11); + dummyToNodeForVerticalAlignment.get("dummy" + _i11).push(nodeId); + if (fixedNodes.has(nodeId)) { + fixedNodesOnHorizontal.add("dummy" + _i11); + } + }); + dummyPositionsForVerticalAlignment.set("dummy" + _i11, xCoords[nodeIndexes.get(verticalAlignment[_i11][0])]); + }; + + for (var _i11 = 0; _i11 < verticalAlignment.length; _i11++) { + _loop6(_i11); + } + } + if (constraints.alignmentConstraint.horizontal) { + var horizontalAlignment = constraints.alignmentConstraint.horizontal; + + var _loop7 = function _loop7(_i12) { + dummyToNodeForHorizontalAlignment.set("dummy" + _i12, []); + horizontalAlignment[_i12].forEach(function (nodeId) { + nodeToDummyForHorizontalAlignment.set(nodeId, "dummy" + _i12); + dummyToNodeForHorizontalAlignment.get("dummy" + _i12).push(nodeId); + if (fixedNodes.has(nodeId)) { + fixedNodesOnVertical.add("dummy" + _i12); + } + }); + dummyPositionsForHorizontalAlignment.set("dummy" + _i12, yCoords[nodeIndexes.get(horizontalAlignment[_i12][0])]); + }; + + for (var _i12 = 0; _i12 < horizontalAlignment.length; _i12++) { + _loop7(_i12); + } + } + } + + // construct horizontal and vertical dags (subgraphs) from overall dag + var dagOnHorizontal = new Map(); + var dagOnVertical = new Map(); + + var _loop8 = function _loop8(nodeId) { + dag.get(nodeId).forEach(function (adjacent) { + var sourceId = void 0; + var targetNode = void 0; + if (adjacent["direction"] == "horizontal") { + sourceId = nodeToDummyForVerticalAlignment.get(nodeId) ? nodeToDummyForVerticalAlignment.get(nodeId) : nodeId; + if (nodeToDummyForVerticalAlignment.get(adjacent.id)) { + targetNode = { id: nodeToDummyForVerticalAlignment.get(adjacent.id), gap: adjacent.gap, direction: adjacent.direction }; + } else { + targetNode = adjacent; + } + if (dagOnHorizontal.has(sourceId)) { + dagOnHorizontal.get(sourceId).push(targetNode); + } else { + dagOnHorizontal.set(sourceId, [targetNode]); + } + if (!dagOnHorizontal.has(targetNode.id)) { + dagOnHorizontal.set(targetNode.id, []); + } + } else { + sourceId = nodeToDummyForHorizontalAlignment.get(nodeId) ? nodeToDummyForHorizontalAlignment.get(nodeId) : nodeId; + if (nodeToDummyForHorizontalAlignment.get(adjacent.id)) { + targetNode = { id: nodeToDummyForHorizontalAlignment.get(adjacent.id), gap: adjacent.gap, direction: adjacent.direction }; + } else { + targetNode = adjacent; + } + if (dagOnVertical.has(sourceId)) { + dagOnVertical.get(sourceId).push(targetNode); + } else { + dagOnVertical.set(sourceId, [targetNode]); + } + if (!dagOnVertical.has(targetNode.id)) { + dagOnVertical.set(targetNode.id, []); + } + } + }); + }; + + var _iteratorNormalCompletion5 = true; + var _didIteratorError5 = false; + var _iteratorError5 = undefined; + + try { + for (var _iterator5 = dag.keys()[Symbol.iterator](), _step5; !(_iteratorNormalCompletion5 = (_step5 = _iterator5.next()).done); _iteratorNormalCompletion5 = true) { + var nodeId = _step5.value; + + _loop8(nodeId); + } + + // find source nodes of each component in horizontal and vertical dags + } catch (err) { + _didIteratorError5 = true; + _iteratorError5 = err; + } finally { + try { + if (!_iteratorNormalCompletion5 && _iterator5.return) { + _iterator5.return(); + } + } finally { + if (_didIteratorError5) { + throw _iteratorError5; + } + } + } + + var undirectedOnHorizontal = dagToUndirected(dagOnHorizontal); + var undirectedOnVertical = dagToUndirected(dagOnVertical); + var componentsOnHorizontal = findComponents(undirectedOnHorizontal); + var componentsOnVertical = findComponents(undirectedOnVertical); + var reversedDagOnHorizontal = dagToReversed(dagOnHorizontal); + var reversedDagOnVertical = dagToReversed(dagOnVertical); + var componentSourcesOnHorizontal = []; + var componentSourcesOnVertical = []; + + componentsOnHorizontal.forEach(function (component, index) { + componentSourcesOnHorizontal[index] = []; + component.forEach(function (nodeId) { + if (reversedDagOnHorizontal.get(nodeId).length == 0) { + componentSourcesOnHorizontal[index].push(nodeId); + } + }); + }); + + componentsOnVertical.forEach(function (component, index) { + componentSourcesOnVertical[index] = []; + component.forEach(function (nodeId) { + if (reversedDagOnVertical.get(nodeId).length == 0) { + componentSourcesOnVertical[index].push(nodeId); + } + }); + }); + + // calculate appropriate positioning for subgraphs + var positionMapHorizontal = findAppropriatePositionForRelativePlacement(dagOnHorizontal, "horizontal", fixedNodesOnHorizontal, dummyPositionsForVerticalAlignment, componentSourcesOnHorizontal); + var positionMapVertical = findAppropriatePositionForRelativePlacement(dagOnVertical, "vertical", fixedNodesOnVertical, dummyPositionsForHorizontalAlignment, componentSourcesOnVertical); + + // update positions of the nodes based on relative placement constraints + + var _loop9 = function _loop9(key) { + if (dummyToNodeForVerticalAlignment.get(key)) { + dummyToNodeForVerticalAlignment.get(key).forEach(function (nodeId) { + xCoords[nodeIndexes.get(nodeId)] = positionMapHorizontal.get(key); + }); + } else { + xCoords[nodeIndexes.get(key)] = positionMapHorizontal.get(key); + } + }; + + var _iteratorNormalCompletion6 = true; + var _didIteratorError6 = false; + var _iteratorError6 = undefined; + + try { + for (var _iterator6 = positionMapHorizontal.keys()[Symbol.iterator](), _step6; !(_iteratorNormalCompletion6 = (_step6 = _iterator6.next()).done); _iteratorNormalCompletion6 = true) { + var key = _step6.value; + + _loop9(key); + } + } catch (err) { + _didIteratorError6 = true; + _iteratorError6 = err; + } finally { + try { + if (!_iteratorNormalCompletion6 && _iterator6.return) { + _iterator6.return(); + } + } finally { + if (_didIteratorError6) { + throw _iteratorError6; + } + } + } + + var _loop10 = function _loop10(key) { + if (dummyToNodeForHorizontalAlignment.get(key)) { + dummyToNodeForHorizontalAlignment.get(key).forEach(function (nodeId) { + yCoords[nodeIndexes.get(nodeId)] = positionMapVertical.get(key); + }); + } else { + yCoords[nodeIndexes.get(key)] = positionMapVertical.get(key); + } + }; + + var _iteratorNormalCompletion7 = true; + var _didIteratorError7 = false; + var _iteratorError7 = undefined; + + try { + for (var _iterator7 = positionMapVertical.keys()[Symbol.iterator](), _step7; !(_iteratorNormalCompletion7 = (_step7 = _iterator7.next()).done); _iteratorNormalCompletion7 = true) { + var key = _step7.value; + + _loop10(key); + } + } catch (err) { + _didIteratorError7 = true; + _iteratorError7 = err; + } finally { + try { + if (!_iteratorNormalCompletion7 && _iterator7.return) { + _iterator7.return(); + } + } finally { + if (_didIteratorError7) { + throw _iteratorError7; + } + } + } + })(); + } + } + + // assign new coordinates to nodes after constraint handling + for (var _i13 = 0; _i13 < allNodes.length; _i13++) { + var _node = allNodes[_i13]; + if (_node.getChild() == null) { + _node.setCenter(xCoords[nodeIndexes.get(_node.id)], yCoords[nodeIndexes.get(_node.id)]); + } + } +}; + +module.exports = ConstraintHandler; + +/***/ }), + +/***/ 551: +/***/ ((module) => { + +module.exports = __WEBPACK_EXTERNAL_MODULE__551__; + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ var cachedModule = __webpack_module_cache__[moduleId]; +/******/ if (cachedModule !== undefined) { +/******/ return cachedModule.exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ +/******/ // startup +/******/ // Load entry module and return exports +/******/ // This entry module is referenced by other modules so it can't be inlined +/******/ var __webpack_exports__ = __webpack_require__(45); +/******/ +/******/ return __webpack_exports__; +/******/ })() +; +}); \ No newline at end of file diff --git a/tools/rirPrettyGraph/dependencies/cytoscape-autopan-on-drag.js b/tools/rirPrettyGraph/dependencies/cytoscape-autopan-on-drag.js new file mode 100644 index 000000000..cf1c3b2f8 --- /dev/null +++ b/tools/rirPrettyGraph/dependencies/cytoscape-autopan-on-drag.js @@ -0,0 +1,244 @@ +;(function(){ 'use strict'; + + // registers the extension on a cytoscape lib ref + var register = function( cytoscape ){ + + if( !cytoscape ){ return; } // can't register if cytoscape unspecified + + // Default options + var defaults = { + enabled: true, // Whether the extension is enabled on register + selector: 'node', // Which elements will be affected by this extension + speed: 1 // Speed of panning when elements exceed canvas bounds + }; + + // Merge default options with the ones coming from parameter + function extend(defaults, options) { + var obj = {}; + + for (var i in defaults) { + obj[i] = defaults[i]; + } + + for (var i in options) { + obj[i] = options[i]; + } + + return obj; + }; + + // Get scratch pad reserved for this extension on the given element or the core if 'name' parameter is not set, + // if the 'name' parameter is set then return the related property in the scratch instead of the whole scratchpad + function getScratch (eleOrCy, name) { + + if (eleOrCy.scratch("_autopanOnDrag") === undefined) { + eleOrCy.scratch("_autopanOnDrag", {}); + } + + var scratchPad = eleOrCy.scratch("_autopanOnDrag"); + + return ( name === undefined ) ? scratchPad : scratchPad[name]; + } + + // Set the a field (described by 'name' parameter) of scratchPad (that is reserved for this extension + // on an element or the core) to the given value (by 'val' parameter) + function setScratch (eleOrCy, name, val) { + + var scratchPad = getScratch(eleOrCy); + scratchPad[name] = val; + eleOrCy.scratch("_autopanOnDrag", scratchPad); + } + + function bindCyEvents (cy, options) { + + // check if the extension is enabled if it is return directly + var enabled = getScratch(cy, 'enabled'); + + if (enabled) { + return; + } + + // get eventFcns from the scratch pad, this object is empty + // or each property of it will be overridden inside this function + var eventFcns = getScratch(cy, 'eventFcns'); + + // get user options from the scratch pad + var options = getScratch(cy, 'options'); + + cy.on('tapstart', options.selector, eventFcns.tapstartFcn = function() { + var node = this; + + var renderedPosition = node.renderedPosition(); + var renderedWidth = node.renderedWidth(); + var renderedHeight = node.renderedHeight(); + + var maxRenderedX = cy.width(); + var maxRenderedY = cy.height(); + + var topLeftRenderedPosition = { + x: renderedPosition.x - renderedWidth / 2, + y: renderedPosition.y - renderedHeight / 2 + }; + + var bottomRightRenderedPosition = { + x: renderedPosition.x + renderedWidth / 2, + y: renderedPosition.y + renderedHeight / 2 + }; + + var exceed = false; + + if( ( bottomRightRenderedPosition.x >= maxRenderedX ) || ( topLeftRenderedPosition.x <= 0 ) + || ( bottomRightRenderedPosition.y >= maxRenderedY ) || ( topLeftRenderedPosition.y <= 0 ) ){ + exceed = true; + } + + if( !exceed ) { + // save the node who is currently being dragged to the scratch pad + setScratch(cy, 'currentNode', node); + } + + }); + + cy.on('tapdrag', eventFcns.tapdragFcn = function() { + + // get the node who is currently being dragged from scratch pad + var currentNode = getScratch(cy, 'currentNode'); + + if(currentNode === undefined) { + return; + } + + var newRenderedPosition = currentNode.renderedPosition(); + var renderedWidth = currentNode.renderedWidth(); + var renderedHeight = currentNode.renderedHeight(); + + var maxRenderedX = cy.width(); + var maxRenderedY = cy.height(); + + var topLeftRenderedPosition = { + x: newRenderedPosition.x - renderedWidth / 2, + y: newRenderedPosition.y - renderedHeight / 2 + }; + + var bottomRightRenderedPosition = { + x: newRenderedPosition.x + renderedWidth / 2, + y: newRenderedPosition.y + renderedHeight / 2 + }; + + var exceedX; + var exceedY; + + if(bottomRightRenderedPosition.x >= maxRenderedX) { + exceedX = -bottomRightRenderedPosition.x + maxRenderedX; + } + + if(topLeftRenderedPosition.x <= 0) { + exceedX = -topLeftRenderedPosition.x; + } + + if(bottomRightRenderedPosition.y >= maxRenderedY ) { + exceedY = -bottomRightRenderedPosition.y + maxRenderedY; + } + + if(topLeftRenderedPosition.y <= 0) { + exceedY = -topLeftRenderedPosition.y; + } + + if(exceedX) { + cy.panBy({x: exceedX * options.speed}); + } + + if(exceedY) { + cy.panBy({y: exceedY * options.speed}); + } + }); + + cy.on('tapend', eventFcns.tapendFcn = function() { + // unset the currently dragged node on scratch pad + setScratch(cy, 'currentNode', undefined); + }); + + // save the eventFcns on scratch pad + setScratch(cy, 'eventFcns', eventFcns); + + // mark that the extension is enabled now + setScratch(cy, 'enabled', true); + } + + function unbindCyEvents (cy) { + + // check if the extension is enabled if it is not return directly + var enabled = getScratch(cy, 'enabled'); + + if (!enabled) { + return; + } + + var eventFcns = getScratch(cy, 'eventFcns'); + var options = getScratch(cy, 'options'); + + cy.off('tapstart', options.selector, eventFcns.tapstartFcn); + cy.off('tapdrag', eventFcns.tapdragFcn); + cy.off('tapend', eventFcns.tapendFcn); + + // mark that the extension is disabled now + setScratch(cy, 'enabled', undefined); + } + + cytoscape( 'core', 'autopanOnDrag', function(opts){ + + var cy = this; + + // use the existing eventFcns if exists or create a new object for them + var eventFcns = getScratch(cy, 'eventFcns') || {}; + + // save eventFcns on scratch pad + setScratch(cy, 'eventFcns', eventFcns); + + if(opts !== 'get') { + // merge the options with existing ones + var options = extend(defaults, opts); + + // save options to the scratch pad + setScratch(cy, 'options', options); + + // if enabled option is set bind events for the cy instance + if(options.enabled) { + + // bind the events + bindCyEvents(cy); + + // mark that the extension is enabled + setScratch(cy, 'enabled', true); + } + } + + // return the extension api + return { + enable: function() { + bindCyEvents(cy); + }, + disable: function() { + unbindCyEvents(cy); + } + }; + + }); + + }; + + if( typeof module !== 'undefined' && module.exports ){ // expose as a commonjs module + module.exports = register; + } + + if( typeof define !== 'undefined' && define.amd ){ // expose as an amd/requirejs module + define('cytoscape-context-menus', function(){ + return register; + }); + } + + if( typeof cytoscape !== 'undefined' ){ // expose to global cytoscape (i.e. window.cytoscape) + register( cytoscape ); + } + +})(); diff --git a/tools/rirPrettyGraph/dependencies/cytoscape-fcose.js b/tools/rirPrettyGraph/dependencies/cytoscape-fcose.js new file mode 100644 index 000000000..9ad4f6ec3 --- /dev/null +++ b/tools/rirPrettyGraph/dependencies/cytoscape-fcose.js @@ -0,0 +1,1549 @@ +(function webpackUniversalModuleDefinition(root, factory) { + if(typeof exports === 'object' && typeof module === 'object') + module.exports = factory(require("cose-base")); + else if(typeof define === 'function' && define.amd) + define(["cose-base"], factory); + else if(typeof exports === 'object') + exports["cytoscapeFcose"] = factory(require("cose-base")); + else + root["cytoscapeFcose"] = factory(root["coseBase"]); +})(this, function(__WEBPACK_EXTERNAL_MODULE__140__) { +return /******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 658: +/***/ ((module) => { + + + +// Simple, internal Object.assign() polyfill for options objects etc. + +module.exports = Object.assign != null ? Object.assign.bind(Object) : function (tgt) { + for (var _len = arguments.length, srcs = Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) { + srcs[_key - 1] = arguments[_key]; + } + + srcs.forEach(function (src) { + Object.keys(src).forEach(function (k) { + return tgt[k] = src[k]; + }); + }); + + return tgt; +}; + +/***/ }), + +/***/ 548: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }(); + +/* + * Auxiliary functions + */ + +var LinkedList = __webpack_require__(140).layoutBase.LinkedList; + +var auxiliary = {}; + +// get the top most nodes +auxiliary.getTopMostNodes = function (nodes) { + var nodesMap = {}; + for (var i = 0; i < nodes.length; i++) { + nodesMap[nodes[i].id()] = true; + } + var roots = nodes.filter(function (ele, i) { + if (typeof ele === "number") { + ele = i; + } + var parent = ele.parent()[0]; + while (parent != null) { + if (nodesMap[parent.id()]) { + return false; + } + parent = parent.parent()[0]; + } + return true; + }); + + return roots; +}; + +// find disconnected components and create dummy nodes that connect them +auxiliary.connectComponents = function (cy, eles, topMostNodes, dummyNodes) { + var queue = new LinkedList(); + var visited = new Set(); + var visitedTopMostNodes = []; + var currentNeighbor = void 0; + var minDegreeNode = void 0; + var minDegree = void 0; + + var isConnected = false; + var count = 1; + var nodesConnectedToDummy = []; + var components = []; + + var _loop = function _loop() { + var cmpt = cy.collection(); + components.push(cmpt); + + var currentNode = topMostNodes[0]; + var childrenOfCurrentNode = cy.collection(); + childrenOfCurrentNode.merge(currentNode).merge(currentNode.descendants().intersection(eles)); + visitedTopMostNodes.push(currentNode); + + childrenOfCurrentNode.forEach(function (node) { + queue.push(node); + visited.add(node); + cmpt.merge(node); + }); + + var _loop2 = function _loop2() { + currentNode = queue.shift(); + + // Traverse all neighbors of this node + var neighborNodes = cy.collection(); + currentNode.neighborhood().nodes().forEach(function (node) { + if (eles.intersection(currentNode.edgesWith(node)).length > 0) { + neighborNodes.merge(node); + } + }); + + for (var i = 0; i < neighborNodes.length; i++) { + var neighborNode = neighborNodes[i]; + currentNeighbor = topMostNodes.intersection(neighborNode.union(neighborNode.ancestors())); + if (currentNeighbor != null && !visited.has(currentNeighbor[0])) { + var childrenOfNeighbor = currentNeighbor.union(currentNeighbor.descendants()); + + childrenOfNeighbor.forEach(function (node) { + queue.push(node); + visited.add(node); + cmpt.merge(node); + if (topMostNodes.has(node)) { + visitedTopMostNodes.push(node); + } + }); + } + } + }; + + while (queue.length != 0) { + _loop2(); + } + + cmpt.forEach(function (node) { + eles.intersection(node.connectedEdges()).forEach(function (e) { + // connectedEdges() usually cached + if (cmpt.has(e.source()) && cmpt.has(e.target())) { + // has() is cheap + cmpt.merge(e); + } + }); + }); + + if (visitedTopMostNodes.length == topMostNodes.length) { + isConnected = true; + } + + if (!isConnected || isConnected && count > 1) { + minDegreeNode = visitedTopMostNodes[0]; + minDegree = minDegreeNode.connectedEdges().length; + visitedTopMostNodes.forEach(function (node) { + if (node.connectedEdges().length < minDegree) { + minDegree = node.connectedEdges().length; + minDegreeNode = node; + } + }); + nodesConnectedToDummy.push(minDegreeNode.id()); + // TO DO: Check efficiency of this part + var temp = cy.collection(); + temp.merge(visitedTopMostNodes[0]); + visitedTopMostNodes.forEach(function (node) { + temp.merge(node); + }); + visitedTopMostNodes = []; + topMostNodes = topMostNodes.difference(temp); + count++; + } + }; + + do { + _loop(); + } while (!isConnected); + + if (dummyNodes) { + if (nodesConnectedToDummy.length > 0) { + dummyNodes.set('dummy' + (dummyNodes.size + 1), nodesConnectedToDummy); + } + } + return components; +}; + +// relocates componentResult to originalCenter if there is no fixedNodeConstraint +auxiliary.relocateComponent = function (originalCenter, componentResult, options) { + if (!options.fixedNodeConstraint) { + var minXCoord = Number.POSITIVE_INFINITY; + var maxXCoord = Number.NEGATIVE_INFINITY; + var minYCoord = Number.POSITIVE_INFINITY; + var maxYCoord = Number.NEGATIVE_INFINITY; + if (options.quality == "draft") { + // calculate current bounding box + var _iteratorNormalCompletion = true; + var _didIteratorError = false; + var _iteratorError = undefined; + + try { + for (var _iterator = componentResult.nodeIndexes[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) { + var _ref = _step.value; + + var _ref2 = _slicedToArray(_ref, 2); + + var key = _ref2[0]; + var value = _ref2[1]; + + var cyNode = options.cy.getElementById(key); + if (cyNode) { + var nodeBB = cyNode.boundingBox(); + var leftX = componentResult.xCoords[value] - nodeBB.w / 2; + var rightX = componentResult.xCoords[value] + nodeBB.w / 2; + var topY = componentResult.yCoords[value] - nodeBB.h / 2; + var bottomY = componentResult.yCoords[value] + nodeBB.h / 2; + + if (leftX < minXCoord) minXCoord = leftX; + if (rightX > maxXCoord) maxXCoord = rightX; + if (topY < minYCoord) minYCoord = topY; + if (bottomY > maxYCoord) maxYCoord = bottomY; + } + } + // find difference between current and original center + } catch (err) { + _didIteratorError = true; + _iteratorError = err; + } finally { + try { + if (!_iteratorNormalCompletion && _iterator.return) { + _iterator.return(); + } + } finally { + if (_didIteratorError) { + throw _iteratorError; + } + } + } + + var diffOnX = originalCenter.x - (maxXCoord + minXCoord) / 2; + var diffOnY = originalCenter.y - (maxYCoord + minYCoord) / 2; + // move component to original center + componentResult.xCoords = componentResult.xCoords.map(function (x) { + return x + diffOnX; + }); + componentResult.yCoords = componentResult.yCoords.map(function (y) { + return y + diffOnY; + }); + } else { + // calculate current bounding box + Object.keys(componentResult).forEach(function (item) { + var node = componentResult[item]; + var leftX = node.getRect().x; + var rightX = node.getRect().x + node.getRect().width; + var topY = node.getRect().y; + var bottomY = node.getRect().y + node.getRect().height; + + if (leftX < minXCoord) minXCoord = leftX; + if (rightX > maxXCoord) maxXCoord = rightX; + if (topY < minYCoord) minYCoord = topY; + if (bottomY > maxYCoord) maxYCoord = bottomY; + }); + // find difference between current and original center + var _diffOnX = originalCenter.x - (maxXCoord + minXCoord) / 2; + var _diffOnY = originalCenter.y - (maxYCoord + minYCoord) / 2; + // move component to original center + Object.keys(componentResult).forEach(function (item) { + var node = componentResult[item]; + node.setCenter(node.getCenterX() + _diffOnX, node.getCenterY() + _diffOnY); + }); + } + } +}; + +auxiliary.calcBoundingBox = function (parentNode, xCoords, yCoords, nodeIndexes) { + // calculate bounds + var left = Number.MAX_SAFE_INTEGER; + var right = Number.MIN_SAFE_INTEGER; + var top = Number.MAX_SAFE_INTEGER; + var bottom = Number.MIN_SAFE_INTEGER; + var nodeLeft = void 0; + var nodeRight = void 0; + var nodeTop = void 0; + var nodeBottom = void 0; + + var nodes = parentNode.descendants().not(":parent"); + var s = nodes.length; + for (var i = 0; i < s; i++) { + var node = nodes[i]; + + nodeLeft = xCoords[nodeIndexes.get(node.id())] - node.width() / 2; + nodeRight = xCoords[nodeIndexes.get(node.id())] + node.width() / 2; + nodeTop = yCoords[nodeIndexes.get(node.id())] - node.height() / 2; + nodeBottom = yCoords[nodeIndexes.get(node.id())] + node.height() / 2; + + if (left > nodeLeft) { + left = nodeLeft; + } + + if (right < nodeRight) { + right = nodeRight; + } + + if (top > nodeTop) { + top = nodeTop; + } + + if (bottom < nodeBottom) { + bottom = nodeBottom; + } + } + + var boundingBox = {}; + boundingBox.topLeftX = left; + boundingBox.topLeftY = top; + boundingBox.width = right - left; + boundingBox.height = bottom - top; + return boundingBox; +}; + +// This function finds and returns parent nodes whose all children are hidden +auxiliary.calcParentsWithoutChildren = function (cy, eles) { + var parentsWithoutChildren = cy.collection(); + eles.nodes(':parent').forEach(function (parent) { + var check = false; + parent.children().forEach(function (child) { + if (child.css('display') != 'none') { + check = true; + } + }); + if (!check) { + parentsWithoutChildren.merge(parent); + } + }); + + return parentsWithoutChildren; +}; + +module.exports = auxiliary; + +/***/ }), + +/***/ 816: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +/** + The implementation of the postprocessing part that applies CoSE layout over the spectral layout +*/ + +var aux = __webpack_require__(548); +var CoSELayout = __webpack_require__(140).CoSELayout; +var CoSENode = __webpack_require__(140).CoSENode; +var PointD = __webpack_require__(140).layoutBase.PointD; +var DimensionD = __webpack_require__(140).layoutBase.DimensionD; +var LayoutConstants = __webpack_require__(140).layoutBase.LayoutConstants; +var FDLayoutConstants = __webpack_require__(140).layoutBase.FDLayoutConstants; +var CoSEConstants = __webpack_require__(140).CoSEConstants; + +// main function that cose layout is processed +var coseLayout = function coseLayout(options, spectralResult) { + + var cy = options.cy; + var eles = options.eles; + var nodes = eles.nodes(); + var edges = eles.edges(); + + var nodeIndexes = void 0; + var xCoords = void 0; + var yCoords = void 0; + var idToLNode = {}; + + if (options.randomize) { + nodeIndexes = spectralResult["nodeIndexes"]; + xCoords = spectralResult["xCoords"]; + yCoords = spectralResult["yCoords"]; + } + + var isFn = function isFn(fn) { + return typeof fn === 'function'; + }; + + var optFn = function optFn(opt, ele) { + if (isFn(opt)) { + return opt(ele); + } else { + return opt; + } + }; + + /**** Postprocessing functions ****/ + + var parentsWithoutChildren = aux.calcParentsWithoutChildren(cy, eles); + + // transfer cytoscape nodes to cose nodes + var processChildrenList = function processChildrenList(parent, children, layout, options) { + var size = children.length; + for (var i = 0; i < size; i++) { + var theChild = children[i]; + var children_of_children = null; + if (theChild.intersection(parentsWithoutChildren).length == 0) { + children_of_children = theChild.children(); + } + var theNode = void 0; + + var dimensions = theChild.layoutDimensions({ + nodeDimensionsIncludeLabels: options.nodeDimensionsIncludeLabels + }); + + if (theChild.outerWidth() != null && theChild.outerHeight() != null) { + if (options.randomize) { + if (!theChild.isParent()) { + theNode = parent.add(new CoSENode(layout.graphManager, new PointD(xCoords[nodeIndexes.get(theChild.id())] - dimensions.w / 2, yCoords[nodeIndexes.get(theChild.id())] - dimensions.h / 2), new DimensionD(parseFloat(dimensions.w), parseFloat(dimensions.h)))); + } else { + var parentInfo = aux.calcBoundingBox(theChild, xCoords, yCoords, nodeIndexes); + if (theChild.intersection(parentsWithoutChildren).length == 0) { + theNode = parent.add(new CoSENode(layout.graphManager, new PointD(parentInfo.topLeftX, parentInfo.topLeftY), new DimensionD(parentInfo.width, parentInfo.height))); + } else { + // for the parentsWithoutChildren + theNode = parent.add(new CoSENode(layout.graphManager, new PointD(parentInfo.topLeftX, parentInfo.topLeftY), new DimensionD(parseFloat(dimensions.w), parseFloat(dimensions.h)))); + } + } + } else { + theNode = parent.add(new CoSENode(layout.graphManager, new PointD(theChild.position('x') - dimensions.w / 2, theChild.position('y') - dimensions.h / 2), new DimensionD(parseFloat(dimensions.w), parseFloat(dimensions.h)))); + } + } else { + theNode = parent.add(new CoSENode(this.graphManager)); + } + // Attach id to the layout node and repulsion value + theNode.id = theChild.data("id"); + theNode.nodeRepulsion = optFn(options.nodeRepulsion, theChild); + // Attach the paddings of cy node to layout node + theNode.paddingLeft = parseInt(theChild.css('padding')); + theNode.paddingTop = parseInt(theChild.css('padding')); + theNode.paddingRight = parseInt(theChild.css('padding')); + theNode.paddingBottom = parseInt(theChild.css('padding')); + + //Attach the label properties to both compound and simple nodes if labels will be included in node dimensions + //These properties will be used while updating bounds of compounds during iterations or tiling + //and will be used for simple nodes while transferring final positions to cytoscape + if (options.nodeDimensionsIncludeLabels) { + theNode.labelWidth = theChild.boundingBox({ includeLabels: true, includeNodes: false, includeOverlays: false }).w; + theNode.labelHeight = theChild.boundingBox({ includeLabels: true, includeNodes: false, includeOverlays: false }).h; + theNode.labelPosVertical = theChild.css("text-valign"); + theNode.labelPosHorizontal = theChild.css("text-halign"); + } + + // Map the layout node + idToLNode[theChild.data("id")] = theNode; + + if (isNaN(theNode.rect.x)) { + theNode.rect.x = 0; + } + + if (isNaN(theNode.rect.y)) { + theNode.rect.y = 0; + } + + if (children_of_children != null && children_of_children.length > 0) { + var theNewGraph = void 0; + theNewGraph = layout.getGraphManager().add(layout.newGraph(), theNode); + processChildrenList(theNewGraph, children_of_children, layout, options); + } + } + }; + + // transfer cytoscape edges to cose edges + var processEdges = function processEdges(layout, gm, edges) { + var idealLengthTotal = 0; + var edgeCount = 0; + for (var i = 0; i < edges.length; i++) { + var edge = edges[i]; + var sourceNode = idToLNode[edge.data("source")]; + var targetNode = idToLNode[edge.data("target")]; + if (sourceNode && targetNode && sourceNode !== targetNode && sourceNode.getEdgesBetween(targetNode).length == 0) { + var e1 = gm.add(layout.newEdge(), sourceNode, targetNode); + e1.id = edge.id(); + e1.idealLength = optFn(options.idealEdgeLength, edge); + e1.edgeElasticity = optFn(options.edgeElasticity, edge); + idealLengthTotal += e1.idealLength; + edgeCount++; + } + } + // we need to update the ideal edge length constant with the avg. ideal length value after processing edges + // in case there is no edge, use other options + if (options.idealEdgeLength != null) { + if (edgeCount > 0) CoSEConstants.DEFAULT_EDGE_LENGTH = FDLayoutConstants.DEFAULT_EDGE_LENGTH = idealLengthTotal / edgeCount;else if (!isFn(options.idealEdgeLength)) // in case there is no edge, but option gives a value to use + CoSEConstants.DEFAULT_EDGE_LENGTH = FDLayoutConstants.DEFAULT_EDGE_LENGTH = options.idealEdgeLength;else // in case there is no edge and we cannot get a value from option (because it's a function) + CoSEConstants.DEFAULT_EDGE_LENGTH = FDLayoutConstants.DEFAULT_EDGE_LENGTH = 50; + // we need to update these constant values based on the ideal edge length constant + CoSEConstants.MIN_REPULSION_DIST = FDLayoutConstants.MIN_REPULSION_DIST = FDLayoutConstants.DEFAULT_EDGE_LENGTH / 10.0; + CoSEConstants.DEFAULT_RADIAL_SEPARATION = FDLayoutConstants.DEFAULT_EDGE_LENGTH; + } + }; + + // transfer cytoscape constraints to cose layout + var processConstraints = function processConstraints(layout, options) { + // get nodes to be fixed + if (options.fixedNodeConstraint) { + layout.constraints["fixedNodeConstraint"] = options.fixedNodeConstraint; + } + // get nodes to be aligned + if (options.alignmentConstraint) { + layout.constraints["alignmentConstraint"] = options.alignmentConstraint; + } + // get nodes to be relatively placed + if (options.relativePlacementConstraint) { + layout.constraints["relativePlacementConstraint"] = options.relativePlacementConstraint; + } + }; + + /**** Apply postprocessing ****/ + if (options.nestingFactor != null) CoSEConstants.PER_LEVEL_IDEAL_EDGE_LENGTH_FACTOR = FDLayoutConstants.PER_LEVEL_IDEAL_EDGE_LENGTH_FACTOR = options.nestingFactor; + if (options.gravity != null) CoSEConstants.DEFAULT_GRAVITY_STRENGTH = FDLayoutConstants.DEFAULT_GRAVITY_STRENGTH = options.gravity; + if (options.numIter != null) CoSEConstants.MAX_ITERATIONS = FDLayoutConstants.MAX_ITERATIONS = options.numIter; + if (options.gravityRange != null) CoSEConstants.DEFAULT_GRAVITY_RANGE_FACTOR = FDLayoutConstants.DEFAULT_GRAVITY_RANGE_FACTOR = options.gravityRange; + if (options.gravityCompound != null) CoSEConstants.DEFAULT_COMPOUND_GRAVITY_STRENGTH = FDLayoutConstants.DEFAULT_COMPOUND_GRAVITY_STRENGTH = options.gravityCompound; + if (options.gravityRangeCompound != null) CoSEConstants.DEFAULT_COMPOUND_GRAVITY_RANGE_FACTOR = FDLayoutConstants.DEFAULT_COMPOUND_GRAVITY_RANGE_FACTOR = options.gravityRangeCompound; + if (options.initialEnergyOnIncremental != null) CoSEConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL = FDLayoutConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL = options.initialEnergyOnIncremental; + + if (options.tilingCompareBy != null) CoSEConstants.TILING_COMPARE_BY = options.tilingCompareBy; + + if (options.quality == 'proof') LayoutConstants.QUALITY = 2;else LayoutConstants.QUALITY = 0; + + CoSEConstants.NODE_DIMENSIONS_INCLUDE_LABELS = FDLayoutConstants.NODE_DIMENSIONS_INCLUDE_LABELS = LayoutConstants.NODE_DIMENSIONS_INCLUDE_LABELS = options.nodeDimensionsIncludeLabels; + CoSEConstants.DEFAULT_INCREMENTAL = FDLayoutConstants.DEFAULT_INCREMENTAL = LayoutConstants.DEFAULT_INCREMENTAL = !options.randomize; + CoSEConstants.ANIMATE = FDLayoutConstants.ANIMATE = LayoutConstants.ANIMATE = options.animate; + CoSEConstants.TILE = options.tile; + CoSEConstants.TILING_PADDING_VERTICAL = typeof options.tilingPaddingVertical === 'function' ? options.tilingPaddingVertical.call() : options.tilingPaddingVertical; + CoSEConstants.TILING_PADDING_HORIZONTAL = typeof options.tilingPaddingHorizontal === 'function' ? options.tilingPaddingHorizontal.call() : options.tilingPaddingHorizontal; + + CoSEConstants.DEFAULT_INCREMENTAL = FDLayoutConstants.DEFAULT_INCREMENTAL = LayoutConstants.DEFAULT_INCREMENTAL = true; + CoSEConstants.PURE_INCREMENTAL = !options.randomize; + LayoutConstants.DEFAULT_UNIFORM_LEAF_NODE_SIZES = options.uniformNodeDimensions; + + // This part is for debug/demo purpose + if (options.step == "transformed") { + CoSEConstants.TRANSFORM_ON_CONSTRAINT_HANDLING = true; + CoSEConstants.ENFORCE_CONSTRAINTS = false; + CoSEConstants.APPLY_LAYOUT = false; + } + if (options.step == "enforced") { + CoSEConstants.TRANSFORM_ON_CONSTRAINT_HANDLING = false; + CoSEConstants.ENFORCE_CONSTRAINTS = true; + CoSEConstants.APPLY_LAYOUT = false; + } + if (options.step == "cose") { + CoSEConstants.TRANSFORM_ON_CONSTRAINT_HANDLING = false; + CoSEConstants.ENFORCE_CONSTRAINTS = false; + CoSEConstants.APPLY_LAYOUT = true; + } + if (options.step == "all") { + if (options.randomize) CoSEConstants.TRANSFORM_ON_CONSTRAINT_HANDLING = true;else CoSEConstants.TRANSFORM_ON_CONSTRAINT_HANDLING = false; + CoSEConstants.ENFORCE_CONSTRAINTS = true; + CoSEConstants.APPLY_LAYOUT = true; + } + + if (options.fixedNodeConstraint || options.alignmentConstraint || options.relativePlacementConstraint) { + CoSEConstants.TREE_REDUCTION_ON_INCREMENTAL = false; + } else { + CoSEConstants.TREE_REDUCTION_ON_INCREMENTAL = true; + } + + var coseLayout = new CoSELayout(); + var gm = coseLayout.newGraphManager(); + + processChildrenList(gm.addRoot(), aux.getTopMostNodes(nodes), coseLayout, options); + processEdges(coseLayout, gm, edges); + processConstraints(coseLayout, options); + + coseLayout.runLayout(); + + return idToLNode; +}; + +module.exports = { coseLayout: coseLayout }; + +/***/ }), + +/***/ 212: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +/** + The implementation of the fcose layout algorithm +*/ + +var assign = __webpack_require__(658); +var aux = __webpack_require__(548); + +var _require = __webpack_require__(657), + spectralLayout = _require.spectralLayout; + +var _require2 = __webpack_require__(816), + coseLayout = _require2.coseLayout; + +var defaults = Object.freeze({ + + // 'draft', 'default' or 'proof' + // - 'draft' only applies spectral layout + // - 'default' improves the quality with subsequent CoSE layout (fast cooling rate) + // - 'proof' improves the quality with subsequent CoSE layout (slow cooling rate) + quality: "default", + // Use random node positions at beginning of layout + // if this is set to false, then quality option must be "proof" + randomize: true, + // Whether or not to animate the layout + animate: true, + // Duration of animation in ms, if enabled + animationDuration: 1000, + // Easing of animation, if enabled + animationEasing: undefined, + // Fit the viewport to the repositioned nodes + fit: true, + // Padding around layout + padding: 30, + // Whether to include labels in node dimensions. Valid in "proof" quality + nodeDimensionsIncludeLabels: false, + // Whether or not simple nodes (non-compound nodes) are of uniform dimensions + uniformNodeDimensions: false, + // Whether to pack disconnected components - valid only if randomize: true + packComponents: true, + // Layout step - all, transformed, enforced, cose - for debug purpose only + step: "all", + + /* spectral layout options */ + + // False for random, true for greedy + samplingType: true, + // Sample size to construct distance matrix + sampleSize: 25, + // Separation amount between nodes + nodeSeparation: 75, + // Power iteration tolerance + piTol: 0.0000001, + + /* CoSE layout options */ + + // Node repulsion (non overlapping) multiplier + nodeRepulsion: function nodeRepulsion(node) { + return 4500; + }, + // Ideal edge (non nested) length + idealEdgeLength: function idealEdgeLength(edge) { + return 50; + }, + // Divisor to compute edge forces + edgeElasticity: function edgeElasticity(edge) { + return 0.45; + }, + // Nesting factor (multiplier) to compute ideal edge length for nested edges + nestingFactor: 0.1, + // Gravity force (constant) + gravity: 0.25, + // Maximum number of iterations to perform + numIter: 2500, + // For enabling tiling + tile: true, + // The function that specifies the criteria for comparing nodes while sorting them during tiling operation. + // Takes the node id as a parameter and the default tiling operation is perfomed when this option is not set. + tilingCompareBy: undefined, + // Represents the amount of the vertical space to put between the zero degree members during the tiling operation(can also be a function) + tilingPaddingVertical: 10, + // Represents the amount of the horizontal space to put between the zero degree members during the tiling operation(can also be a function) + tilingPaddingHorizontal: 10, + // Gravity range (constant) for compounds + gravityRangeCompound: 1.5, + // Gravity force (constant) for compounds + gravityCompound: 1.0, + // Gravity range (constant) + gravityRange: 3.8, + // Initial cooling factor for incremental layout + initialEnergyOnIncremental: 0.3, + + /* constraint options */ + + // Fix required nodes to predefined positions + // [{nodeId: 'n1', position: {x: 100, y: 200}, {...}] + fixedNodeConstraint: undefined, + // Align required nodes in vertical/horizontal direction + // {vertical: [['n1', 'n2')], ['n3', 'n4']], horizontal: ['n2', 'n4']} + alignmentConstraint: undefined, + // Place two nodes relatively in vertical/horizontal direction + // [{top: 'n1', bottom: 'n2', gap: 100}, {left: 'n3', right: 'n4', gap: 75}] + relativePlacementConstraint: undefined, + + /* layout event callbacks */ + ready: function ready() {}, // on layoutready + stop: function stop() {} // on layoutstop +}); + +var Layout = function () { + function Layout(options) { + _classCallCheck(this, Layout); + + this.options = assign({}, defaults, options); + } + + _createClass(Layout, [{ + key: 'run', + value: function run() { + var layout = this; + var options = this.options; + var cy = options.cy; + var eles = options.eles; + + var spectralResult = []; + var xCoords = void 0; + var yCoords = void 0; + var coseResult = []; + var components = void 0; + var componentCenters = []; + + // basic validity check for constraint inputs + if (options.fixedNodeConstraint && (!Array.isArray(options.fixedNodeConstraint) || options.fixedNodeConstraint.length == 0)) { + options.fixedNodeConstraint = undefined; + } + + if (options.alignmentConstraint) { + if (options.alignmentConstraint.vertical && (!Array.isArray(options.alignmentConstraint.vertical) || options.alignmentConstraint.vertical.length == 0)) { + options.alignmentConstraint.vertical = undefined; + } + if (options.alignmentConstraint.horizontal && (!Array.isArray(options.alignmentConstraint.horizontal) || options.alignmentConstraint.horizontal.length == 0)) { + options.alignmentConstraint.horizontal = undefined; + } + } + + if (options.relativePlacementConstraint && (!Array.isArray(options.relativePlacementConstraint) || options.relativePlacementConstraint.length == 0)) { + options.relativePlacementConstraint = undefined; + } + + // if any constraint exists, set some options + var constraintExist = options.fixedNodeConstraint || options.alignmentConstraint || options.relativePlacementConstraint; + if (constraintExist) { + // constraints work with these options + options.tile = false; + options.packComponents = false; + } + + // decide component packing is enabled or not + var layUtil = void 0; + var packingEnabled = false; + if (cy.layoutUtilities && options.packComponents) { + layUtil = cy.layoutUtilities("get"); + if (!layUtil) layUtil = cy.layoutUtilities(); + packingEnabled = true; + } + + if (eles.nodes().length > 0) { + // if packing is not enabled, perform layout on the whole graph + if (!packingEnabled) { + // store component center + var boundingBox = options.eles.boundingBox(); + componentCenters.push({ x: boundingBox.x1 + boundingBox.w / 2, y: boundingBox.y1 + boundingBox.h / 2 }); + // apply spectral layout + if (options.randomize) { + var result = spectralLayout(options); + spectralResult.push(result); + } + // apply cose layout as postprocessing + if (options.quality == "default" || options.quality == "proof") { + coseResult.push(coseLayout(options, spectralResult[0])); + aux.relocateComponent(componentCenters[0], coseResult[0], options); // relocate center to original position + } else { + aux.relocateComponent(componentCenters[0], spectralResult[0], options); // relocate center to original position + } + } else { + // packing is enabled + var topMostNodes = aux.getTopMostNodes(options.eles.nodes()); + components = aux.connectComponents(cy, options.eles, topMostNodes); + // store component centers + components.forEach(function (component) { + var boundingBox = component.boundingBox(); + componentCenters.push({ x: boundingBox.x1 + boundingBox.w / 2, y: boundingBox.y1 + boundingBox.h / 2 }); + }); + + //send each component to spectral layout if randomized + if (options.randomize) { + components.forEach(function (component) { + options.eles = component; + spectralResult.push(spectralLayout(options)); + }); + } + + if (options.quality == "default" || options.quality == "proof") { + var toBeTiledNodes = cy.collection(); + if (options.tile) { + // behave nodes to be tiled as one component + var nodeIndexes = new Map(); + var _xCoords = []; + var _yCoords = []; + var count = 0; + var tempSpectralResult = { nodeIndexes: nodeIndexes, xCoords: _xCoords, yCoords: _yCoords }; + var indexesToBeDeleted = []; + components.forEach(function (component, index) { + if (component.edges().length == 0) { + component.nodes().forEach(function (node, i) { + toBeTiledNodes.merge(component.nodes()[i]); + if (!node.isParent()) { + tempSpectralResult.nodeIndexes.set(component.nodes()[i].id(), count++); + tempSpectralResult.xCoords.push(component.nodes()[0].position().x); + tempSpectralResult.yCoords.push(component.nodes()[0].position().y); + } + }); + indexesToBeDeleted.push(index); + } + }); + if (toBeTiledNodes.length > 1) { + var _boundingBox = toBeTiledNodes.boundingBox(); + componentCenters.push({ x: _boundingBox.x1 + _boundingBox.w / 2, y: _boundingBox.y1 + _boundingBox.h / 2 }); + components.push(toBeTiledNodes); + spectralResult.push(tempSpectralResult); + for (var i = indexesToBeDeleted.length - 1; i >= 0; i--) { + components.splice(indexesToBeDeleted[i], 1); + spectralResult.splice(indexesToBeDeleted[i], 1); + componentCenters.splice(indexesToBeDeleted[i], 1); + }; + } + } + components.forEach(function (component, index) { + // send each component to cose layout + options.eles = component; + coseResult.push(coseLayout(options, spectralResult[index])); + aux.relocateComponent(componentCenters[index], coseResult[index], options); // relocate center to original position + }); + } else { + components.forEach(function (component, index) { + aux.relocateComponent(componentCenters[index], spectralResult[index], options); // relocate center to original position + }); + } + + // packing + var componentsEvaluated = new Set(); + if (components.length > 1) { + var subgraphs = []; + var hiddenEles = eles.filter(function (ele) { + return ele.css('display') == 'none'; + }); + components.forEach(function (component, index) { + var nodeIndexes = void 0; + if (options.quality == "draft") { + nodeIndexes = spectralResult[index].nodeIndexes; + } + + if (component.nodes().not(hiddenEles).length > 0) { + var subgraph = {}; + subgraph.edges = []; + subgraph.nodes = []; + var nodeIndex = void 0; + component.nodes().not(hiddenEles).forEach(function (node) { + if (options.quality == "draft") { + if (!node.isParent()) { + nodeIndex = nodeIndexes.get(node.id()); + subgraph.nodes.push({ x: spectralResult[index].xCoords[nodeIndex] - node.boundingbox().w / 2, y: spectralResult[index].yCoords[nodeIndex] - node.boundingbox().h / 2, width: node.boundingbox().w, height: node.boundingbox().h }); + } else { + var parentInfo = aux.calcBoundingBox(node, spectralResult[index].xCoords, spectralResult[index].yCoords, nodeIndexes); + subgraph.nodes.push({ x: parentInfo.topLeftX, y: parentInfo.topLeftY, width: parentInfo.width, height: parentInfo.height }); + } + } else { + if (coseResult[index][node.id()]) { + subgraph.nodes.push({ x: coseResult[index][node.id()].getLeft(), y: coseResult[index][node.id()].getTop(), width: coseResult[index][node.id()].getWidth(), height: coseResult[index][node.id()].getHeight() }); + } + } + }); + component.edges().forEach(function (edge) { + var source = edge.source(); + var target = edge.target(); + if (source.css("display") != "none" && target.css("display") != "none") { + if (options.quality == "draft") { + var sourceNodeIndex = nodeIndexes.get(source.id()); + var targetNodeIndex = nodeIndexes.get(target.id()); + var sourceCenter = []; + var targetCenter = []; + if (source.isParent()) { + var parentInfo = aux.calcBoundingBox(source, spectralResult[index].xCoords, spectralResult[index].yCoords, nodeIndexes); + sourceCenter.push(parentInfo.topLeftX + parentInfo.width / 2); + sourceCenter.push(parentInfo.topLeftY + parentInfo.height / 2); + } else { + sourceCenter.push(spectralResult[index].xCoords[sourceNodeIndex]); + sourceCenter.push(spectralResult[index].yCoords[sourceNodeIndex]); + } + if (target.isParent()) { + var _parentInfo = aux.calcBoundingBox(target, spectralResult[index].xCoords, spectralResult[index].yCoords, nodeIndexes); + targetCenter.push(_parentInfo.topLeftX + _parentInfo.width / 2); + targetCenter.push(_parentInfo.topLeftY + _parentInfo.height / 2); + } else { + targetCenter.push(spectralResult[index].xCoords[targetNodeIndex]); + targetCenter.push(spectralResult[index].yCoords[targetNodeIndex]); + } + subgraph.edges.push({ startX: sourceCenter[0], startY: sourceCenter[1], endX: targetCenter[0], endY: targetCenter[1] }); + } else { + if (coseResult[index][source.id()] && coseResult[index][target.id()]) { + subgraph.edges.push({ startX: coseResult[index][source.id()].getCenterX(), startY: coseResult[index][source.id()].getCenterY(), endX: coseResult[index][target.id()].getCenterX(), endY: coseResult[index][target.id()].getCenterY() }); + } + } + } + }); + if (subgraph.nodes.length > 0) { + subgraphs.push(subgraph); + componentsEvaluated.add(index); + } + } + }); + var shiftResult = layUtil.packComponents(subgraphs, options.randomize).shifts; + if (options.quality == "draft") { + spectralResult.forEach(function (result, index) { + var newXCoords = result.xCoords.map(function (x) { + return x + shiftResult[index].dx; + }); + var newYCoords = result.yCoords.map(function (y) { + return y + shiftResult[index].dy; + }); + result.xCoords = newXCoords; + result.yCoords = newYCoords; + }); + } else { + var _count = 0; + componentsEvaluated.forEach(function (index) { + Object.keys(coseResult[index]).forEach(function (item) { + var nodeRectangle = coseResult[index][item]; + nodeRectangle.setCenter(nodeRectangle.getCenterX() + shiftResult[_count].dx, nodeRectangle.getCenterY() + shiftResult[_count].dy); + }); + _count++; + }); + } + } + } + } + + // get each element's calculated position + var getPositions = function getPositions(ele, i) { + if (options.quality == "default" || options.quality == "proof") { + if (typeof ele === "number") { + ele = i; + } + var pos = void 0; + var node = void 0; + var theId = ele.data('id'); + coseResult.forEach(function (result) { + if (theId in result) { + pos = { x: result[theId].getRect().getCenterX(), y: result[theId].getRect().getCenterY() }; + node = result[theId]; + } + }); + if (options.nodeDimensionsIncludeLabels) { + if (node.labelWidth) { + if (node.labelPosHorizontal == "left") { + pos.x += node.labelWidth / 2; + } else if (node.labelPosHorizontal == "right") { + pos.x -= node.labelWidth / 2; + } + } + if (node.labelHeight) { + if (node.labelPosVertical == "top") { + pos.y += node.labelHeight / 2; + } else if (node.labelPosVertical == "bottom") { + pos.y -= node.labelHeight / 2; + } + } + } + if (pos == undefined) pos = { x: ele.position("x"), y: ele.position("y") }; + return { + x: pos.x, + y: pos.y + }; + } else { + var _pos = void 0; + spectralResult.forEach(function (result) { + var index = result.nodeIndexes.get(ele.id()); + if (index != undefined) { + _pos = { x: result.xCoords[index], y: result.yCoords[index] }; + } + }); + if (_pos == undefined) _pos = { x: ele.position("x"), y: ele.position("y") }; + return { + x: _pos.x, + y: _pos.y + }; + } + }; + + // quality = "draft" and randomize = false are contradictive so in that case positions don't change + if (options.quality == "default" || options.quality == "proof" || options.randomize) { + // transfer calculated positions to nodes (positions of only simple nodes are evaluated, compounds are positioned automatically) + var parentsWithoutChildren = aux.calcParentsWithoutChildren(cy, eles); + var _hiddenEles = eles.filter(function (ele) { + return ele.css('display') == 'none'; + }); + options.eles = eles.not(_hiddenEles); + + eles.nodes().not(":parent").not(_hiddenEles).layoutPositions(layout, options, getPositions); + + if (parentsWithoutChildren.length > 0) { + parentsWithoutChildren.forEach(function (ele) { + ele.position(getPositions(ele)); + }); + } + } else { + console.log("If randomize option is set to false, then quality option must be 'default' or 'proof'."); + } + } + }]); + + return Layout; +}(); + +module.exports = Layout; + +/***/ }), + +/***/ 657: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +/** + The implementation of the spectral layout that is the first part of the fcose layout algorithm +*/ + +var aux = __webpack_require__(548); +var Matrix = __webpack_require__(140).layoutBase.Matrix; +var SVD = __webpack_require__(140).layoutBase.SVD; + +// main function that spectral layout is processed +var spectralLayout = function spectralLayout(options) { + + var cy = options.cy; + var eles = options.eles; + var nodes = eles.nodes(); + var parentNodes = eles.nodes(":parent"); + + var dummyNodes = new Map(); // map to keep dummy nodes and their neighbors + var nodeIndexes = new Map(); // map to keep indexes to nodes + var parentChildMap = new Map(); // mapping btw. compound and its representative node + var allNodesNeighborhood = []; // array to keep neighborhood of all nodes + var xCoords = []; + var yCoords = []; + + var samplesColumn = []; // sampled vertices + var minDistancesColumn = []; + var C = []; // column sampling matrix + var PHI = []; // intersection of column and row sampling matrices + var INV = []; // inverse of PHI + + var firstSample = void 0; // the first sampled node + var nodeSize = void 0; + + var infinity = 100000000; + var small = 0.000000001; + + var piTol = options.piTol; + var samplingType = options.samplingType; // false for random, true for greedy + var nodeSeparation = options.nodeSeparation; + var sampleSize = void 0; + + /**** Spectral-preprocessing functions ****/ + + /**** Spectral layout functions ****/ + + // determine which columns to be sampled + var randomSampleCR = function randomSampleCR() { + var sample = 0; + var count = 0; + var flag = false; + + while (count < sampleSize) { + sample = Math.floor(Math.random() * nodeSize); + + flag = false; + for (var i = 0; i < count; i++) { + if (samplesColumn[i] == sample) { + flag = true; + break; + } + } + + if (!flag) { + samplesColumn[count] = sample; + count++; + } else { + continue; + } + } + }; + + // takes the index of the node(pivot) to initiate BFS as a parameter + var BFS = function BFS(pivot, index, samplingMethod) { + var path = []; // the front of the path + var front = 0; // the back of the path + var back = 0; + var current = 0; + var temp = void 0; + var distance = []; + + var max_dist = 0; // the furthest node to be returned + var max_ind = 1; + + for (var i = 0; i < nodeSize; i++) { + distance[i] = infinity; + } + + path[back] = pivot; + distance[pivot] = 0; + + while (back >= front) { + current = path[front++]; + var neighbors = allNodesNeighborhood[current]; + for (var _i = 0; _i < neighbors.length; _i++) { + temp = nodeIndexes.get(neighbors[_i]); + if (distance[temp] == infinity) { + distance[temp] = distance[current] + 1; + path[++back] = temp; + } + } + C[current][index] = distance[current] * nodeSeparation; + } + + if (samplingMethod) { + for (var _i2 = 0; _i2 < nodeSize; _i2++) { + if (C[_i2][index] < minDistancesColumn[_i2]) minDistancesColumn[_i2] = C[_i2][index]; + } + + for (var _i3 = 0; _i3 < nodeSize; _i3++) { + if (minDistancesColumn[_i3] > max_dist) { + max_dist = minDistancesColumn[_i3]; + max_ind = _i3; + } + } + } + return max_ind; + }; + + // apply BFS to all nodes or selected samples + var allBFS = function allBFS(samplingMethod) { + + var sample = void 0; + + if (!samplingMethod) { + randomSampleCR(); + + // call BFS + for (var i = 0; i < sampleSize; i++) { + BFS(samplesColumn[i], i, samplingMethod, false); + } + } else { + sample = Math.floor(Math.random() * nodeSize); + firstSample = sample; + + for (var _i4 = 0; _i4 < nodeSize; _i4++) { + minDistancesColumn[_i4] = infinity; + } + + for (var _i5 = 0; _i5 < sampleSize; _i5++) { + samplesColumn[_i5] = sample; + sample = BFS(sample, _i5, samplingMethod); + } + } + + // form the squared distances for C + for (var _i6 = 0; _i6 < nodeSize; _i6++) { + for (var j = 0; j < sampleSize; j++) { + C[_i6][j] *= C[_i6][j]; + } + } + + // form PHI + for (var _i7 = 0; _i7 < sampleSize; _i7++) { + PHI[_i7] = []; + } + + for (var _i8 = 0; _i8 < sampleSize; _i8++) { + for (var _j = 0; _j < sampleSize; _j++) { + PHI[_i8][_j] = C[samplesColumn[_j]][_i8]; + } + } + }; + + // perform the SVD algorithm and apply a regularization step + var sample = function sample() { + + var SVDResult = SVD.svd(PHI); + + var a_q = SVDResult.S; + var a_u = SVDResult.U; + var a_v = SVDResult.V; + + var max_s = a_q[0] * a_q[0] * a_q[0]; + + var a_Sig = []; + + // regularization + for (var i = 0; i < sampleSize; i++) { + a_Sig[i] = []; + for (var j = 0; j < sampleSize; j++) { + a_Sig[i][j] = 0; + if (i == j) { + a_Sig[i][j] = a_q[i] / (a_q[i] * a_q[i] + max_s / (a_q[i] * a_q[i])); + } + } + } + + INV = Matrix.multMat(Matrix.multMat(a_v, a_Sig), Matrix.transpose(a_u)); + }; + + // calculate final coordinates + var powerIteration = function powerIteration() { + // two largest eigenvalues + var theta1 = void 0; + var theta2 = void 0; + + // initial guesses for eigenvectors + var Y1 = []; + var Y2 = []; + + var V1 = []; + var V2 = []; + + for (var i = 0; i < nodeSize; i++) { + Y1[i] = Math.random(); + Y2[i] = Math.random(); + } + + Y1 = Matrix.normalize(Y1); + Y2 = Matrix.normalize(Y2); + + var count = 0; + // to keep track of the improvement ratio in power iteration + var current = small; + var previous = small; + + var temp = void 0; + + while (true) { + count++; + + for (var _i9 = 0; _i9 < nodeSize; _i9++) { + V1[_i9] = Y1[_i9]; + } + + Y1 = Matrix.multGamma(Matrix.multL(Matrix.multGamma(V1), C, INV)); + theta1 = Matrix.dotProduct(V1, Y1); + Y1 = Matrix.normalize(Y1); + + current = Matrix.dotProduct(V1, Y1); + + temp = Math.abs(current / previous); + + if (temp <= 1 + piTol && temp >= 1) { + break; + } + + previous = current; + } + + for (var _i10 = 0; _i10 < nodeSize; _i10++) { + V1[_i10] = Y1[_i10]; + } + + count = 0; + previous = small; + while (true) { + count++; + + for (var _i11 = 0; _i11 < nodeSize; _i11++) { + V2[_i11] = Y2[_i11]; + } + + V2 = Matrix.minusOp(V2, Matrix.multCons(V1, Matrix.dotProduct(V1, V2))); + Y2 = Matrix.multGamma(Matrix.multL(Matrix.multGamma(V2), C, INV)); + theta2 = Matrix.dotProduct(V2, Y2); + Y2 = Matrix.normalize(Y2); + + current = Matrix.dotProduct(V2, Y2); + + temp = Math.abs(current / previous); + + if (temp <= 1 + piTol && temp >= 1) { + break; + } + + previous = current; + } + + for (var _i12 = 0; _i12 < nodeSize; _i12++) { + V2[_i12] = Y2[_i12]; + } + + // theta1 now contains dominant eigenvalue + // theta2 now contains the second-largest eigenvalue + // V1 now contains theta1's eigenvector + // V2 now contains theta2's eigenvector + + //populate the two vectors + xCoords = Matrix.multCons(V1, Math.sqrt(Math.abs(theta1))); + yCoords = Matrix.multCons(V2, Math.sqrt(Math.abs(theta2))); + }; + + /**** Preparation for spectral layout (Preprocessing) ****/ + + // connect disconnected components (first top level, then inside of each compound node) + aux.connectComponents(cy, eles, aux.getTopMostNodes(nodes), dummyNodes); + + parentNodes.forEach(function (ele) { + aux.connectComponents(cy, eles, aux.getTopMostNodes(ele.descendants().intersection(eles)), dummyNodes); + }); + + // assign indexes to nodes (first real, then dummy nodes) + var index = 0; + for (var i = 0; i < nodes.length; i++) { + if (!nodes[i].isParent()) { + nodeIndexes.set(nodes[i].id(), index++); + } + } + + var _iteratorNormalCompletion = true; + var _didIteratorError = false; + var _iteratorError = undefined; + + try { + for (var _iterator = dummyNodes.keys()[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) { + var key = _step.value; + + nodeIndexes.set(key, index++); + } + + // instantiate the neighborhood matrix + } catch (err) { + _didIteratorError = true; + _iteratorError = err; + } finally { + try { + if (!_iteratorNormalCompletion && _iterator.return) { + _iterator.return(); + } + } finally { + if (_didIteratorError) { + throw _iteratorError; + } + } + } + + for (var _i13 = 0; _i13 < nodeIndexes.size; _i13++) { + allNodesNeighborhood[_i13] = []; + } + + // form a parent-child map to keep representative node of each compound node + parentNodes.forEach(function (ele) { + var children = ele.children().intersection(eles); + + // let random = 0; + while (children.nodes(":childless").length == 0) { + // random = Math.floor(Math.random() * children.nodes().length); // if all children are compound then proceed randomly + children = children.nodes()[0].children().intersection(eles); + } + // select the representative node - we can apply different methods here + // random = Math.floor(Math.random() * children.nodes(":childless").length); + var index = 0; + var min = children.nodes(":childless")[0].connectedEdges().length; + children.nodes(":childless").forEach(function (ele2, i) { + if (ele2.connectedEdges().length < min) { + min = ele2.connectedEdges().length; + index = i; + } + }); + parentChildMap.set(ele.id(), children.nodes(":childless")[index].id()); + }); + + // add neighborhood relations (first real, then dummy nodes) + nodes.forEach(function (ele) { + var eleIndex = void 0; + + if (ele.isParent()) eleIndex = nodeIndexes.get(parentChildMap.get(ele.id()));else eleIndex = nodeIndexes.get(ele.id()); + + ele.neighborhood().nodes().forEach(function (node) { + if (eles.intersection(ele.edgesWith(node)).length > 0) { + if (node.isParent()) allNodesNeighborhood[eleIndex].push(parentChildMap.get(node.id()));else allNodesNeighborhood[eleIndex].push(node.id()); + } + }); + }); + + var _loop = function _loop(_key) { + var eleIndex = nodeIndexes.get(_key); + var disconnectedId = void 0; + dummyNodes.get(_key).forEach(function (id) { + if (cy.getElementById(id).isParent()) disconnectedId = parentChildMap.get(id);else disconnectedId = id; + + allNodesNeighborhood[eleIndex].push(disconnectedId); + allNodesNeighborhood[nodeIndexes.get(disconnectedId)].push(_key); + }); + }; + + var _iteratorNormalCompletion2 = true; + var _didIteratorError2 = false; + var _iteratorError2 = undefined; + + try { + for (var _iterator2 = dummyNodes.keys()[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true) { + var _key = _step2.value; + + _loop(_key); + } + + // nodeSize now only considers the size of transformed graph + } catch (err) { + _didIteratorError2 = true; + _iteratorError2 = err; + } finally { + try { + if (!_iteratorNormalCompletion2 && _iterator2.return) { + _iterator2.return(); + } + } finally { + if (_didIteratorError2) { + throw _iteratorError2; + } + } + } + + nodeSize = nodeIndexes.size; + + var spectralResult = void 0; + + // If number of nodes in transformed graph is 1 or 2, either SVD or powerIteration causes problem + // So skip spectral and layout the graph with cose + if (nodeSize > 2) { + // if # of nodes in transformed graph is smaller than sample size, + // then use # of nodes as sample size + sampleSize = nodeSize < options.sampleSize ? nodeSize : options.sampleSize; + + // instantiates the partial matrices that will be used in spectral layout + for (var _i14 = 0; _i14 < nodeSize; _i14++) { + C[_i14] = []; + } + for (var _i15 = 0; _i15 < sampleSize; _i15++) { + INV[_i15] = []; + } + + /**** Apply spectral layout ****/ + + if (options.quality == "draft" || options.step == "all") { + allBFS(samplingType); + sample(); + powerIteration(); + + spectralResult = { nodeIndexes: nodeIndexes, xCoords: xCoords, yCoords: yCoords }; + } else { + nodeIndexes.forEach(function (value, key) { + xCoords.push(cy.getElementById(key).position("x")); + yCoords.push(cy.getElementById(key).position("y")); + }); + spectralResult = { nodeIndexes: nodeIndexes, xCoords: xCoords, yCoords: yCoords }; + } + return spectralResult; + } else { + var iterator = nodeIndexes.keys(); + var firstNode = cy.getElementById(iterator.next().value); + var firstNodePos = firstNode.position(); + var firstNodeWidth = firstNode.outerWidth(); + xCoords.push(firstNodePos.x); + yCoords.push(firstNodePos.y); + if (nodeSize == 2) { + var secondNode = cy.getElementById(iterator.next().value); + var secondNodeWidth = secondNode.outerWidth(); + xCoords.push(firstNodePos.x + firstNodeWidth / 2 + secondNodeWidth / 2 + options.idealEdgeLength); + yCoords.push(firstNodePos.y); + } + + spectralResult = { nodeIndexes: nodeIndexes, xCoords: xCoords, yCoords: yCoords }; + return spectralResult; + } +}; + +module.exports = { spectralLayout: spectralLayout }; + +/***/ }), + +/***/ 579: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + + +var impl = __webpack_require__(212); + +// registers the extension on a cytoscape lib ref +var register = function register(cytoscape) { + if (!cytoscape) { + return; + } // can't register if cytoscape unspecified + + cytoscape('layout', 'fcose', impl); // register with cytoscape.js +}; + +if (typeof cytoscape !== 'undefined') { + // expose to global cytoscape (i.e. window.cytoscape) + register(cytoscape); +} + +module.exports = register; + +/***/ }), + +/***/ 140: +/***/ ((module) => { + +module.exports = __WEBPACK_EXTERNAL_MODULE__140__; + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ var cachedModule = __webpack_module_cache__[moduleId]; +/******/ if (cachedModule !== undefined) { +/******/ return cachedModule.exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ +/******/ // startup +/******/ // Load entry module and return exports +/******/ // This entry module is referenced by other modules so it can't be inlined +/******/ var __webpack_exports__ = __webpack_require__(579); +/******/ +/******/ return __webpack_exports__; +/******/ })() +; +}); \ No newline at end of file diff --git a/tools/rirPrettyGraph/dependencies/cytoscape-lasso.min.js b/tools/rirPrettyGraph/dependencies/cytoscape-lasso.min.js new file mode 100644 index 000000000..9cab0355c --- /dev/null +++ b/tools/rirPrettyGraph/dependencies/cytoscape-lasso.min.js @@ -0,0 +1,2 @@ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):(t="undefined"!=typeof globalThis?globalThis:t||self).CytoscapeLasso=e()}(this,(function(){"use strict";var t=function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")};function e(t,e){for(var n=0;n=t.length?{done:!0}:{done:!1,value:t[i++]}},e:function(t){throw t},f:o}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,s=!0,c=!1;return{s:function(){n=t[Symbol.iterator]()},n:function(){var t=n.next();return s=t.done,t},e:function(t){c=!0,a=t},f:function(){try{s||null==n.return||n.return()}finally{if(c)throw a}}}}function r(t,e){(null==e||e>t.length)&&(e=t.length);for(var n=0,i=new Array(e);n=this.cy.renderer().desktopTapThreshold2&&(this.activated=!0)}}},{key:"finish",value:function(t){if(this.activated){var e=this.getGraphPolygon(this.polygon),n=this.cy.nodes().filter((function(t){var n=t.position();return function(t,e){for(var n=t[0],i=t[1],o=!1,r=0,a=e.length-1;ri!=l>i&&n<(h-s)*(i-c)/(l-c)+s&&(o=!o)}return o}([n.x,n.y],e)}));a(t)||"additive"===this.cy.selectionType()||this.cy.$(s).unmerge(n).unselect(),n.emit("box").stdFilter(c).select().emit("boxselect"),this.activated=!1}}},{key:"render",value:function(){if(this.ctx.clearRect(0,0,this.canvas.width,this.canvas.height),this.activated){var t=this.cy.style(),e=t.core("selection-box-color").value,n=t.core("selection-box-border-color").value,i=t.core("selection-box-border-width").value,r=t.core("selection-box-opacity").value,a=this.canvas.width/this.canvas.clientWidth;this.ctx.scale(a,a);var s=this.getCanvasPolygon(this.polygon);this.ctx.beginPath(),this.ctx.moveTo(s[0],s[1]);var c,h=o(s);try{for(h.s();!(c=h.n()).done;){var l=c.value;this.ctx.lineTo(l[0],l[1])}}catch(t){h.e(t)}finally{h.f()}i>0&&(this.ctx.lineWidth=i,this.ctx.strokeStyle="rgba(".concat(n[0],", ").concat(n[1],", ").concat(n[2],", ").concat(r,")"),this.ctx.stroke()),this.ctx.closePath(),this.ctx.fillStyle="rgba(".concat(e[0],", ").concat(e[1],", ").concat(e[2],", ").concat(r,")"),this.ctx.fill(),this.ctx.setTransform(1,0,0,1,0,0)}}},{key:"getCanvasPosition",value:function(t){var e=this.cy.renderer().findContainerClientCoords();return[t[0]-e[0],t[1]-e[1]]}},{key:"getGraphPosition",value:function(t){return this.cy.renderer().projectIntoViewport(t[0],t[1])}},{key:"getCanvasPolygon",value:function(t){var e=this;return t.map((function(t){return e.getCanvasPosition(t)}))}},{key:"getGraphPolygon",value:function(t){var e=this;return t.map((function(t){return e.getGraphPosition(t)}))}}]),e}();function l(t){t&&t("core","lassoSelectionEnabled",(function(t){return void 0===t?this._private.lassoSelectionEnabled:(this._private.lassoSelectionEnabled=!!t,t&&!this._private.lassoHandler?this._private.lassoHandler=new h(this):!t&&this._private.lassoHandler&&(this._private.lassoHandler.destroy(),this._private.lassoHandler=void 0),this)}))}return void 0!==window.cytoscape&&l(window.cytoscape),l})); +//# sourceMappingURL=cytoscape-lasso.min.js.map diff --git a/tools/rirPrettyGraph/dependencies/cytoscape.min.js b/tools/rirPrettyGraph/dependencies/cytoscape.min.js new file mode 100644 index 000000000..d3ec635c4 --- /dev/null +++ b/tools/rirPrettyGraph/dependencies/cytoscape.min.js @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2016-2023, The Cytoscape Consortium. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the “Software”), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):(e="undefined"!=typeof globalThis?globalThis:e||self).cytoscape=t()}(this,(function(){"use strict";function e(t){return(e="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e})(t)}function t(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function n(e,t){for(var n=0;ne.length)&&(t=e.length);for(var n=0,r=new Array(t);nt?1:0},I=null!=Object.assign?Object.assign.bind(Object):function(e){for(var t=arguments,n=1;n255)return;t.push(Math.floor(a))}var o=r[1]||r[2]||r[3],s=r[1]&&r[2]&&r[3];if(o&&!s)return;var l=n[4];if(void 0!==l){if((l=parseFloat(l))<0||l>1)return;t.push(l)}}return t}(e)||function(e){var t,n,r,i,a,o,s,l;function u(e,t,n){return n<0&&(n+=1),n>1&&(n-=1),n<1/6?e+6*(t-e)*n:n<.5?t:n<2/3?e+(t-e)*(2/3-n)*6:e}var c=new RegExp("^hsl[a]?\\(((?:[-+]?(?:(?:\\d+|\\d*\\.\\d+)(?:[Ee][+-]?\\d+)?)))\\s*,\\s*((?:[-+]?(?:(?:\\d+|\\d*\\.\\d+)(?:[Ee][+-]?\\d+)?))[%])\\s*,\\s*((?:[-+]?(?:(?:\\d+|\\d*\\.\\d+)(?:[Ee][+-]?\\d+)?))[%])(?:\\s*,\\s*((?:[-+]?(?:(?:\\d+|\\d*\\.\\d+)(?:[Ee][+-]?\\d+)?))))?\\)$").exec(e);if(c){if((n=parseInt(c[1]))<0?n=(360- -1*n%360)%360:n>360&&(n%=360),n/=360,(r=parseFloat(c[2]))<0||r>100)return;if(r/=100,(i=parseFloat(c[3]))<0||i>100)return;if(i/=100,void 0!==(a=c[4])&&((a=parseFloat(a))<0||a>1))return;if(0===r)o=s=l=Math.round(255*i);else{var d=i<.5?i*(1+r):i+r-i*r,h=2*i-d;o=Math.round(255*u(h,d,n+1/3)),s=Math.round(255*u(h,d,n)),l=Math.round(255*u(h,d,n-1/3))}t=[o,s,l,a]}return t}(e)},L={transparent:[0,0,0,0],aliceblue:[240,248,255],antiquewhite:[250,235,215],aqua:[0,255,255],aquamarine:[127,255,212],azure:[240,255,255],beige:[245,245,220],bisque:[255,228,196],black:[0,0,0],blanchedalmond:[255,235,205],blue:[0,0,255],blueviolet:[138,43,226],brown:[165,42,42],burlywood:[222,184,135],cadetblue:[95,158,160],chartreuse:[127,255,0],chocolate:[210,105,30],coral:[255,127,80],cornflowerblue:[100,149,237],cornsilk:[255,248,220],crimson:[220,20,60],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgoldenrod:[184,134,11],darkgray:[169,169,169],darkgreen:[0,100,0],darkgrey:[169,169,169],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkseagreen:[143,188,143],darkslateblue:[72,61,139],darkslategray:[47,79,79],darkslategrey:[47,79,79],darkturquoise:[0,206,209],darkviolet:[148,0,211],deeppink:[255,20,147],deepskyblue:[0,191,255],dimgray:[105,105,105],dimgrey:[105,105,105],dodgerblue:[30,144,255],firebrick:[178,34,34],floralwhite:[255,250,240],forestgreen:[34,139,34],fuchsia:[255,0,255],gainsboro:[220,220,220],ghostwhite:[248,248,255],gold:[255,215,0],goldenrod:[218,165,32],gray:[128,128,128],grey:[128,128,128],green:[0,128,0],greenyellow:[173,255,47],honeydew:[240,255,240],hotpink:[255,105,180],indianred:[205,92,92],indigo:[75,0,130],ivory:[255,255,240],khaki:[240,230,140],lavender:[230,230,250],lavenderblush:[255,240,245],lawngreen:[124,252,0],lemonchiffon:[255,250,205],lightblue:[173,216,230],lightcoral:[240,128,128],lightcyan:[224,255,255],lightgoldenrodyellow:[250,250,210],lightgray:[211,211,211],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightsalmon:[255,160,122],lightseagreen:[32,178,170],lightskyblue:[135,206,250],lightslategray:[119,136,153],lightslategrey:[119,136,153],lightsteelblue:[176,196,222],lightyellow:[255,255,224],lime:[0,255,0],limegreen:[50,205,50],linen:[250,240,230],magenta:[255,0,255],maroon:[128,0,0],mediumaquamarine:[102,205,170],mediumblue:[0,0,205],mediumorchid:[186,85,211],mediumpurple:[147,112,219],mediumseagreen:[60,179,113],mediumslateblue:[123,104,238],mediumspringgreen:[0,250,154],mediumturquoise:[72,209,204],mediumvioletred:[199,21,133],midnightblue:[25,25,112],mintcream:[245,255,250],mistyrose:[255,228,225],moccasin:[255,228,181],navajowhite:[255,222,173],navy:[0,0,128],oldlace:[253,245,230],olive:[128,128,0],olivedrab:[107,142,35],orange:[255,165,0],orangered:[255,69,0],orchid:[218,112,214],palegoldenrod:[238,232,170],palegreen:[152,251,152],paleturquoise:[175,238,238],palevioletred:[219,112,147],papayawhip:[255,239,213],peachpuff:[255,218,185],peru:[205,133,63],pink:[255,192,203],plum:[221,160,221],powderblue:[176,224,230],purple:[128,0,128],red:[255,0,0],rosybrown:[188,143,143],royalblue:[65,105,225],saddlebrown:[139,69,19],salmon:[250,128,114],sandybrown:[244,164,96],seagreen:[46,139,87],seashell:[255,245,238],sienna:[160,82,45],silver:[192,192,192],skyblue:[135,206,235],slateblue:[106,90,205],slategray:[112,128,144],slategrey:[112,128,144],snow:[255,250,250],springgreen:[0,255,127],steelblue:[70,130,180],tan:[210,180,140],teal:[0,128,128],thistle:[216,191,216],tomato:[255,99,71],turquoise:[64,224,208],violet:[238,130,238],wheat:[245,222,179],white:[255,255,255],whitesmoke:[245,245,245],yellow:[255,255,0],yellowgreen:[154,205,50]},O=function(e){for(var t=e.map,n=e.keys,r=n.length,i=0;i=t||n<0||d&&e-u>=a}function v(){var e=X();if(g(e))return y(e);s=setTimeout(v,function(e){var n=t-(e-l);return d?pe(n,a-(e-u)):n}(e))}function y(e){return s=void 0,h&&r?p(e):(r=i=void 0,o)}function m(){var e=X(),n=g(e);if(r=arguments,i=this,l=e,n){if(void 0===s)return f(l);if(d)return clearTimeout(s),s=setTimeout(v,t),p(l)}return void 0===s&&(s=setTimeout(v,t)),o}return t=de(t)||0,V(n)&&(c=!!n.leading,a=(d="maxWait"in n)?he(de(n.maxWait)||0,t):a,h="trailing"in n?!!n.trailing:h),m.cancel=function(){void 0!==s&&clearTimeout(s),u=0,r=l=i=s=void 0},m.flush=function(){return void 0===s?o:y(X())},m},ge=s?s.performance:null,ve=ge&&ge.now?function(){return ge.now()}:function(){return Date.now()},ye=function(){if(s){if(s.requestAnimationFrame)return function(e){s.requestAnimationFrame(e)};if(s.mozRequestAnimationFrame)return function(e){s.mozRequestAnimationFrame(e)};if(s.webkitRequestAnimationFrame)return function(e){s.webkitRequestAnimationFrame(e)};if(s.msRequestAnimationFrame)return function(e){s.msRequestAnimationFrame(e)}}return function(e){e&&setTimeout((function(){e(ve())}),1e3/60)}}(),me=function(e){return ye(e)},be=ve,xe=65599,we=function(e){for(var t,n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:9261,r=n;!(t=e.next()).done;)r=r*xe+t.value|0;return r},Ee=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:9261;return t*xe+e|0},ke=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:5381;return(t<<5)+t+e|0},Ce=function(e){return 2097152*e[0]+e[1]},Se=function(e,t){return[Ee(e[0],t[0]),ke(e[1],t[1])]},Pe=function(e,t){var n={value:0,done:!1},r=0,i=e.length;return we({next:function(){return r=0&&(e[r]!==t||(e.splice(r,1),!n));r--);},He=function(e){e.splice(0,e.length)},Ke=function(e,t,n){return n&&(t=M(n,t)),e[t]},Ge=function(e,t,n,r){n&&(t=M(n,t)),e[t]=r},Ue="undefined"!=typeof Map?Map:function(){function e(){t(this,e),this._obj={}}return r(e,[{key:"set",value:function(e,t){return this._obj[e]=t,this}},{key:"delete",value:function(e){return this._obj[e]=void 0,this}},{key:"clear",value:function(){this._obj={}}},{key:"has",value:function(e){return void 0!==this._obj[e]}},{key:"get",value:function(e){return this._obj[e]}}]),e}(),Ze=function(){function e(n){if(t(this,e),this._obj=Object.create(null),this.size=0,null!=n){var r;r=null!=n.instanceString&&n.instanceString()===this.instanceString()?n.toArray():n;for(var i=0;i2&&void 0!==arguments[2])||arguments[2];if(void 0!==e&&void 0!==t&&k(e)){var r=t.group;if(null==r&&(r=t.data&&null!=t.data.source&&null!=t.data.target?"edges":"nodes"),"nodes"===r||"edges"===r){this.length=1,this[0]=this;var i=this._private={cy:e,single:!0,data:t.data||{},position:t.position||{x:0,y:0},autoWidth:void 0,autoHeight:void 0,autoPadding:void 0,compoundBoundsClean:!1,listeners:[],group:r,style:{},rstyle:{},styleCxts:[],styleKeys:{},removed:!0,selected:!!t.selected,selectable:void 0===t.selectable||!!t.selectable,locked:!!t.locked,grabbed:!1,grabbable:void 0===t.grabbable||!!t.grabbable,pannable:void 0===t.pannable?"edges"===r:!!t.pannable,active:!1,classes:new $e,animation:{current:[],queue:[]},rscratch:{},scratch:t.scratch||{},edges:[],children:[],parent:t.parent&&t.parent.isNode()?t.parent:null,traversalCache:{},backgrounding:!1,bbCache:null,bbCacheShift:{x:0,y:0},bodyBounds:null,overlayBounds:null,labelBounds:{all:null,source:null,target:null,main:null},arrowBounds:{source:null,target:null,"mid-source":null,"mid-target":null}};if(null==i.position.x&&(i.position.x=0),null==i.position.y&&(i.position.y=0),t.renderedPosition){var a=t.renderedPosition,o=e.pan(),s=e.zoom();i.position={x:(a.x-o.x)/s,y:(a.y-o.y)/s}}var l=[];v(t.classes)?l=t.classes:f(t.classes)&&(l=t.classes.split(/\s+/));for(var u=0,c=l.length;ut?1:0},u=function(e,t,i,a,o){var s;if(null==i&&(i=0),null==o&&(o=n),i<0)throw new Error("lo must be non-negative");for(null==a&&(a=e.length);in;0<=n?t++:t--)u.push(t);return u}.apply(this).reverse()).length;ag;0<=g?++h:--h)v.push(a(e,r));return v},f=function(e,t,r,i){var a,o,s;for(null==i&&(i=n),a=e[r];r>t&&i(a,o=e[s=r-1>>1])<0;)e[r]=o,r=s;return e[r]=a},g=function(e,t,r){var i,a,o,s,l;for(null==r&&(r=n),a=e.length,l=t,o=e[t],i=2*t+1;i0;){var k=b.pop(),C=v(k),S=k.id();if(d[S]=C,C!==1/0)for(var P=k.neighborhood().intersect(p),D=0;D0)for(n.unshift(t);c[i];){var a=c[i];n.unshift(a.edge),n.unshift(a.node),i=(r=a.node).id()}return o.spawn(n)}}}},it={kruskal:function(e){e=e||function(e){return 1};for(var t=this.byGroup(),n=t.nodes,r=t.edges,i=n.length,a=new Array(i),o=n,s=function(e){for(var t=0;t0;){if(l=g.pop(),u=l.id(),v.delete(u),w++,u===d){for(var E=[],k=i,C=d,S=m[C];E.unshift(k),null!=S&&E.unshift(S),null!=(k=y[C]);)S=m[C=k.id()];return{found:!0,distance:h[u],path:this.spawn(E),steps:w}}f[u]=!0;for(var P=l._private.edges,D=0;DD&&(p[P]=D,m[P]=S,b[P]=w),!i){var T=S*u+C;!i&&p[T]>D&&(p[T]=D,m[T]=C,b[T]=w)}}}for(var _=0;_1&&void 0!==arguments[1]?arguments[1]:a,r=b(e),i=[],o=r;;){if(null==o)return t.spawn();var l=m(o),u=l.edge,c=l.pred;if(i.unshift(o[0]),o.same(n)&&i.length>0)break;null!=u&&i.unshift(u),o=c}return s.spawn(i)},hasNegativeWeightCycle:g,negativeWeightCycles:v}}},dt=Math.sqrt(2),ht=function(e,t,n){0===n.length&&Oe("Karger-Stein must be run on a connected (sub)graph");for(var r=n[e],i=r[1],a=r[2],o=t[i],s=t[a],l=n,u=l.length-1;u>=0;u--){var c=l[u],d=c[1],h=c[2];(t[d]===o&&t[h]===s||t[d]===s&&t[h]===o)&&l.splice(u,1)}for(var p=0;pr;){var i=Math.floor(Math.random()*t.length);t=ht(i,e,t),n--}return t},ft={kargerStein:function(){var e=this,t=this.byGroup(),n=t.nodes,r=t.edges;r.unmergeBy((function(e){return e.isLoop()}));var i=n.length,a=r.length,o=Math.ceil(Math.pow(Math.log(i)/Math.LN2,2)),s=Math.floor(i/dt);if(!(i<2)){for(var l=[],u=0;u0?1:e<0?-1:0},wt=function(e,t){return Math.sqrt(Et(e,t))},Et=function(e,t){var n=t.x-e.x,r=t.y-e.y;return n*n+r*r},kt=function(e){for(var t=e.length,n=0,r=0;r=e.x1&&e.y2>=e.y1)return{x1:e.x1,y1:e.y1,x2:e.x2,y2:e.y2,w:e.x2-e.x1,h:e.y2-e.y1};if(null!=e.w&&null!=e.h&&e.w>=0&&e.h>=0)return{x1:e.x1,y1:e.y1,x2:e.x1+e.w,y2:e.y1+e.h,w:e.w,h:e.h}}},Tt=function(e,t,n){e.x1=Math.min(e.x1,t),e.x2=Math.max(e.x2,t),e.w=e.x2-e.x1,e.y1=Math.min(e.y1,n),e.y2=Math.max(e.y2,n),e.h=e.y2-e.y1},_t=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0;return e.x1-=t,e.x2+=t,e.y1-=t,e.y2+=t,e.w=e.x2-e.x1,e.h=e.y2-e.y1,e},Mt=function(e){var t,n,r,i,o=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[0];if(1===o.length)t=n=r=i=o[0];else if(2===o.length)t=r=o[0],i=n=o[1];else if(4===o.length){var s=a(o,4);t=s[0],n=s[1],r=s[2],i=s[3]}return e.x1-=i,e.x2+=n,e.y1-=t,e.y2+=r,e.w=e.x2-e.x1,e.h=e.y2-e.y1,e},Bt=function(e,t){e.x1=t.x1,e.y1=t.y1,e.x2=t.x2,e.y2=t.y2,e.w=e.x2-e.x1,e.h=e.y2-e.y1},Nt=function(e,t){return!(e.x1>t.x2)&&(!(t.x1>e.x2)&&(!(e.x2t.y2)&&!(t.y1>e.y2)))))))},zt=function(e,t,n){return e.x1<=t&&t<=e.x2&&e.y1<=n&&n<=e.y2},It=function(e,t){return zt(e,t.x1,t.y1)&&zt(e,t.x2,t.y2)},At=function(e,t,n,r,i,a,o){var s,l=Jt(i,a),u=i/2,c=a/2,d=r-c-o;if((s=Kt(e,t,n,r,n-u+l-o,d,n+u-l+o,d,!1)).length>0)return s;var h=n+u+o;if((s=Kt(e,t,n,r,h,r-c+l-o,h,r+c-l+o,!1)).length>0)return s;var p=r+c+o;if((s=Kt(e,t,n,r,n-u+l-o,p,n+u-l+o,p,!1)).length>0)return s;var f,g=n-u-o;if((s=Kt(e,t,n,r,g,r-c+l-o,g,r+c-l+o,!1)).length>0)return s;var v=n-u+l,y=r-c+l;if((f=Wt(e,t,n,r,v,y,l+o)).length>0&&f[0]<=v&&f[1]<=y)return[f[0],f[1]];var m=n+u-l,b=r-c+l;if((f=Wt(e,t,n,r,m,b,l+o)).length>0&&f[0]>=m&&f[1]<=b)return[f[0],f[1]];var x=n+u-l,w=r+c-l;if((f=Wt(e,t,n,r,x,w,l+o)).length>0&&f[0]>=x&&f[1]>=w)return[f[0],f[1]];var E=n-u+l,k=r+c-l;return(f=Wt(e,t,n,r,E,k,l+o)).length>0&&f[0]<=E&&f[1]>=k?[f[0],f[1]]:[]},Lt=function(e,t,n,r,i,a,o){var s=o,l=Math.min(n,i),u=Math.max(n,i),c=Math.min(r,a),d=Math.max(r,a);return l-s<=e&&e<=u+s&&c-s<=t&&t<=d+s},Ot=function(e,t,n,r,i,a,o,s,l){var u=Math.min(n,o,i)-l,c=Math.max(n,o,i)+l,d=Math.min(r,s,a)-l,h=Math.max(r,s,a)+l;return!(ec||th)},Rt=function(e,t,n,r,i,a,o,s){var l=[];!function(e,t,n,r,i){var a,o,s,l,u,c,d,h;0===e&&(e=1e-5),s=-27*(r/=e)+(t/=e)*(9*(n/=e)-t*t*2),a=(o=(3*n-t*t)/9)*o*o+(s/=54)*s,i[1]=0,d=t/3,a>0?(u=(u=s+Math.sqrt(a))<0?-Math.pow(-u,1/3):Math.pow(u,1/3),c=(c=s-Math.sqrt(a))<0?-Math.pow(-c,1/3):Math.pow(c,1/3),i[0]=-d+u+c,d+=(u+c)/2,i[4]=i[2]=-d,d=Math.sqrt(3)*(-c+u)/2,i[3]=d,i[5]=-d):(i[5]=i[3]=0,0===a?(h=s<0?-Math.pow(-s,1/3):Math.pow(s,1/3),i[0]=2*h-d,i[4]=i[2]=-(h+d)):(l=(o=-o)*o*o,l=Math.acos(s/Math.sqrt(l)),h=2*Math.sqrt(o),i[0]=-d+h*Math.cos(l/3),i[2]=-d+h*Math.cos((l+2*Math.PI)/3),i[4]=-d+h*Math.cos((l+4*Math.PI)/3)))}(1*n*n-4*n*i+2*n*o+4*i*i-4*i*o+o*o+r*r-4*r*a+2*r*s+4*a*a-4*a*s+s*s,9*n*i-3*n*n-3*n*o-6*i*i+3*i*o+9*r*a-3*r*r-3*r*s-6*a*a+3*a*s,3*n*n-6*n*i+n*o-n*e+2*i*i+2*i*e-o*e+3*r*r-6*r*a+r*s-r*t+2*a*a+2*a*t-s*t,1*n*i-n*n+n*e-i*e+r*a-r*r+r*t-a*t,l);for(var u=[],c=0;c<6;c+=2)Math.abs(l[c+1])<1e-7&&l[c]>=0&&l[c]<=1&&u.push(l[c]);u.push(1),u.push(0);for(var d,h,p,f=-1,g=0;g=0?pl?(e-i)*(e-i)+(t-a)*(t-a):u-d},Ft=function(e,t,n){for(var r,i,a,o,s=0,l=0;l=e&&e>=a||r<=e&&e<=a))continue;(e-r)/(a-r)*(o-i)+i>t&&s++}return s%2!=0},jt=function(e,t,n,r,i,a,o,s,l){var u,c=new Array(n.length);null!=s[0]?(u=Math.atan(s[1]/s[0]),s[0]<0?u+=Math.PI/2:u=-u-Math.PI/2):u=s;for(var d,h=Math.cos(-u),p=Math.sin(-u),f=0;f0){var g=Yt(c,-l);d=qt(g)}else d=c;return Ft(e,t,d)},qt=function(e){for(var t,n,r,i,a,o,s,l,u=new Array(e.length/2),c=0;c=0&&f<=1&&v.push(f),g>=0&&g<=1&&v.push(g),0===v.length)return[];var y=v[0]*s[0]+e,m=v[0]*s[1]+t;return v.length>1?v[0]==v[1]?[y,m]:[y,m,v[1]*s[0]+e,v[1]*s[1]+t]:[y,m]},Ht=function(e,t,n){return t<=e&&e<=n||n<=e&&e<=t?e:e<=t&&t<=n||n<=t&&t<=e?t:n},Kt=function(e,t,n,r,i,a,o,s,l){var u=e-i,c=n-e,d=o-i,h=t-a,p=r-t,f=s-a,g=d*h-f*u,v=c*h-p*u,y=f*c-d*p;if(0!==y){var m=g/y,b=v/y;return-.001<=m&&m<=1.001&&-.001<=b&&b<=1.001||l?[e+m*c,t+m*p]:[]}return 0===g||0===v?Ht(e,n,o)===o?[o,s]:Ht(e,n,i)===i?[i,a]:Ht(i,o,n)===n?[n,r]:[]:[]},Gt=function(e,t,n,r,i,a,o,s){var l,u,c,d,h,p,f=[],g=new Array(n.length),v=!0;if(null==a&&(v=!1),v){for(var y=0;y0){var m=Yt(g,-s);u=qt(m)}else u=g}else u=n;for(var b=0;bu&&(u=t)},d=function(e){return l[e]},h=0;h0?b.edgesTo(m)[0]:m.edgesTo(b)[0];var w=r(x);m=m.id(),h[m]>h[v]+w&&(h[m]=h[v]+w,p.nodes.indexOf(m)<0?p.push(m):p.updateItem(m),u[m]=0,l[m]=[]),h[m]==h[v]+w&&(u[m]=u[m]+u[v],l[m].push(v))}else for(var E=0;E0;){for(var P=n.pop(),D=0;D0&&o.push(n[s]);0!==o.length&&i.push(r.collection(o))}return i}(c,l,t,r);return b=function(e){for(var t=0;t5&&void 0!==arguments[5]?arguments[5]:wn,o=r,s=0;s=2?Dn(e,t,n,0,Cn,Sn):Dn(e,t,n,0,kn)},squaredEuclidean:function(e,t,n){return Dn(e,t,n,0,Cn)},manhattan:function(e,t,n){return Dn(e,t,n,0,kn)},max:function(e,t,n){return Dn(e,t,n,-1/0,Pn)}};function _n(e,t,n,r,i,a){var o;return o=g(e)?e:Tn[e]||Tn.euclidean,0===t&&g(e)?o(i,a):o(t,n,r,i,a)}Tn["squared-euclidean"]=Tn.squaredEuclidean,Tn.squaredeuclidean=Tn.squaredEuclidean;var Mn=Xe({k:2,m:2,sensitivityThreshold:1e-4,distance:"euclidean",maxIterations:10,attributes:[],testMode:!1,testCentroids:null}),Bn=function(e){return Mn(e)},Nn=function(e,t,n,r,i){var a="kMedoids"!==i?function(e){return n[e]}:function(e){return r[e](n)},o=n,s=t;return _n(e,r.length,a,(function(e){return r[e](t)}),o,s)},zn=function(e,t,n){for(var r=n.length,i=new Array(r),a=new Array(r),o=new Array(t),s=null,l=0;ln)return!1}return!0},On=function(e,t,n){for(var r=0;ri&&(i=t[l][u],a=u);o[a].push(e[l])}for(var c=0;c=i.threshold||"dendrogram"===i.mode&&1===e.length)return!1;var p,f=t[o],g=t[r[o]];p="dendrogram"===i.mode?{left:f,right:g,key:f.key}:{value:f.value.concat(g.value),key:f.key},e[f.index]=p,e.splice(g.index,1),t[f.key]=p;for(var v=0;vn[g.key][y.key]&&(a=n[g.key][y.key])):"max"===i.linkage?(a=n[f.key][y.key],n[f.key][y.key]1&&void 0!==arguments[1]?arguments[1]:0,n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:e.length,r=!(arguments.length>3&&void 0!==arguments[3])||arguments[3],i=!(arguments.length>4&&void 0!==arguments[4])||arguments[4],a=!(arguments.length>5&&void 0!==arguments[5])||arguments[5];r?e=e.slice(t,n):(n0&&e.splice(0,t));for(var o=0,s=e.length-1;s>=0;s--){var l=e[s];a?isFinite(l)||(e[s]=-1/0,o++):e.splice(s,1)}i&&e.sort((function(e,t){return e-t}));var u=e.length,c=Math.floor(u/2);return u%2!=0?e[c+1+o]:(e[c-1+o]+e[c+o])/2}(e):"mean"===t?function(e){for(var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0,n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:e.length,r=0,i=0,a=t;a1&&void 0!==arguments[1]?arguments[1]:0,n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:e.length,r=1/0,i=t;i1&&void 0!==arguments[1]?arguments[1]:0,n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:e.length,r=-1/0,i=t;io&&(a=l,o=t[i*e+l])}a>0&&r.push(a)}for(var u=0;u=D?(T=D,D=M,_=B):M>T&&(T=M);for(var N=0;N0?1:0;C[k%u.minIterations*t+R]=V,O+=V}if(O>0&&(k>=u.minIterations-1||k==u.maxIterations-1)){for(var F=0,j=0;j0&&r.push(i);return r}(t,a,o),X=function(e,t,n){for(var r=Jn(e,t,n),i=0;il&&(s=u,l=c)}n[i]=a[s]}return r=Jn(e,t,n)}(t,r,Y),W={},H=0;H1)}}));var l=Object.keys(t).filter((function(e){return t[e].cutVertex})).map((function(t){return e.getElementById(t)}));return{cut:e.spawn(l),components:i}},ir=function(){var e=this,t={},n=0,r=[],i=[],a=e.spawn(e);return e.forEach((function(o){if(o.isNode()){var s=o.id();s in t||function o(s){if(i.push(s),t[s]={index:n,low:n++,explored:!1},e.getElementById(s).connectedEdges().intersection(e).forEach((function(e){var n=e.target().id();n!==s&&(n in t||o(n),t[n].explored||(t[s].low=Math.min(t[s].low,t[n].low)))})),t[s].index===t[s].low){for(var l=e.spawn();;){var u=i.pop();if(l.merge(e.getElementById(u)),t[u].low=t[s].index,t[u].explored=!0,u===s)break}var c=l.edgesWith(l),d=l.merge(c);r.push(d),a=a.difference(d)}}(s)}})),{cut:a,components:r}},ar={};[et,rt,it,ot,lt,ct,ft,rn,on,ln,cn,xn,Yn,Un,tr,{hierholzer:function(e){if(!y(e)){var t=arguments;e={root:t[0],directed:t[1]}}var n,r,i,a=nr(e),o=a.root,s=a.directed,l=this,u=!1;o&&(i=f(o)?this.filter(o)[0].id():o[0].id());var c={},d={};s?l.forEach((function(e){var t=e.id();if(e.isNode()){var i=e.indegree(!0),a=e.outdegree(!0),o=i-a,s=a-i;1==o?n?u=!0:n=t:1==s?r?u=!0:r=t:(s>1||o>1)&&(u=!0),c[t]=[],e.outgoers().forEach((function(e){e.isEdge()&&c[t].push(e.id())}))}else d[t]=[void 0,e.target().id()]})):l.forEach((function(e){var t=e.id();e.isNode()?(e.degree(!0)%2&&(n?r?u=!0:r=t:n=t),c[t]=[],e.connectedEdges().forEach((function(e){return c[t].push(e.id())}))):d[t]=[e.source().id(),e.target().id()]}));var h={found:!1,trail:void 0};if(u)return h;if(r&&n)if(s){if(i&&r!=i)return h;i=r}else{if(i&&r!=i&&n!=i)return h;i||(i=r)}else i||(i=l[0].id());var p=function(e){for(var t,n,r,i=e,a=[e];c[i].length;)t=c[i].shift(),n=d[t][0],i!=(r=d[t][1])?(c[r]=c[r].filter((function(e){return e!=t})),i=r):s||i==n||(c[n]=c[n].filter((function(e){return e!=t})),i=n),a.unshift(t),a.unshift(i);return a},g=[],v=[];for(v=p(i);1!=v.length;)0==c[v[0]].length?(g.unshift(l.getElementById(v.shift())),g.unshift(l.getElementById(v.shift()))):v=p(v.shift()).concat(v);for(var m in g.unshift(l.getElementById(v.shift())),c)if(c[m].length)return h;return h.found=!0,h.trail=this.spawn(g,!0),h}},{hopcroftTarjanBiconnected:rr,htbc:rr,htb:rr,hopcroftTarjanBiconnectedComponents:rr},{tarjanStronglyConnected:ir,tsc:ir,tscc:ir,tarjanStronglyConnectedComponents:ir}].forEach((function(e){I(ar,e)})); + /*! + Embeddable Minimum Strictly-Compliant Promises/A+ 1.1.1 Thenable + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com) + Licensed under The MIT License (http://opensource.org/licenses/MIT) + */ + var or=function e(t){if(!(this instanceof e))return new e(t);this.id="Thenable/1.0.7",this.state=0,this.fulfillValue=void 0,this.rejectReason=void 0,this.onFulfilled=[],this.onRejected=[],this.proxy={then:this.then.bind(this)},"function"==typeof t&&t.call(this,this.fulfill.bind(this),this.reject.bind(this))};or.prototype={fulfill:function(e){return sr(this,1,"fulfillValue",e)},reject:function(e){return sr(this,2,"rejectReason",e)},then:function(e,t){var n=new or;return this.onFulfilled.push(cr(e,n,"fulfill")),this.onRejected.push(cr(t,n,"reject")),lr(this),n.proxy}};var sr=function(e,t,n,r){return 0===e.state&&(e.state=t,e[n]=r,lr(e)),e},lr=function(e){1===e.state?ur(e,"onFulfilled",e.fulfillValue):2===e.state&&ur(e,"onRejected",e.rejectReason)},ur=function(e,t,n){if(0!==e[t].length){var r=e[t];e[t]=[];var i=function(){for(var e=0;e0:void 0}},clearQueue:function(){return function(){var e=void 0!==this.length?this:[this];if(!(this._private.cy||this).styleEnabled())return this;for(var t=0;t-1};var Jr=function(e,t){var n=this.__data__,r=Gr(n,e);return r<0?(++this.size,n.push([e,t])):n[r][1]=t,this};function ei(e){var t=-1,n=null==e?0:e.length;for(this.clear();++t-1&&e%1==0&&e0&&this.spawn(n).updateStyle().emit("class"),this},addClass:function(e){return this.toggleClass(e,!0)},hasClass:function(e){var t=this[0];return null!=t&&t._private.classes.has(e)},toggleClass:function(e,t){v(e)||(e=e.match(/\S+/g)||[]);for(var n=void 0===t,r=[],i=0,a=this.length;i0&&this.spawn(r).updateStyle().emit("class"),this},removeClass:function(e){return this.toggleClass(e,!1)},flashClass:function(e,t){var n=this;if(null==t)t=250;else if(0===t)return n;return n.addClass(e),setTimeout((function(){n.removeClass(e)}),t),n}};Ri.className=Ri.classNames=Ri.classes;var Vi={metaChar:"[\\!\\\"\\#\\$\\%\\&\\'\\(\\)\\*\\+\\,\\.\\/\\:\\;\\<\\=\\>\\?\\@\\[\\]\\^\\`\\{\\|\\}\\~]",comparatorOp:"=|\\!=|>|>=|<|<=|\\$=|\\^=|\\*=",boolOp:"\\?|\\!|\\^",string:"\"(?:\\\\\"|[^\"])*\"|'(?:\\\\'|[^'])*'",number:N,meta:"degree|indegree|outdegree",separator:"\\s*,\\s*",descendant:"\\s+",child:"\\s+>\\s+",subject:"\\$",group:"node|edge|\\*",directedEdge:"\\s+->\\s+",undirectedEdge:"\\s+<->\\s+"};Vi.variable="(?:[\\w-.]|(?:\\\\"+Vi.metaChar+"))+",Vi.className="(?:[\\w-]|(?:\\\\"+Vi.metaChar+"))+",Vi.value=Vi.string+"|"+Vi.number,Vi.id=Vi.variable,function(){var e,t,n;for(e=Vi.comparatorOp.split("|"),n=0;n=0||"="!==t&&(Vi.comparatorOp+="|\\!"+t)}();var Fi=0,ji=1,qi=2,Yi=3,Xi=4,Wi=5,Hi=6,Ki=7,Gi=8,Ui=9,Zi=10,$i=11,Qi=12,Ji=13,ea=14,ta=15,na=16,ra=17,ia=18,aa=19,oa=20,sa=[{selector:":selected",matches:function(e){return e.selected()}},{selector:":unselected",matches:function(e){return!e.selected()}},{selector:":selectable",matches:function(e){return e.selectable()}},{selector:":unselectable",matches:function(e){return!e.selectable()}},{selector:":locked",matches:function(e){return e.locked()}},{selector:":unlocked",matches:function(e){return!e.locked()}},{selector:":visible",matches:function(e){return e.visible()}},{selector:":hidden",matches:function(e){return!e.visible()}},{selector:":transparent",matches:function(e){return e.transparent()}},{selector:":grabbed",matches:function(e){return e.grabbed()}},{selector:":free",matches:function(e){return!e.grabbed()}},{selector:":removed",matches:function(e){return e.removed()}},{selector:":inside",matches:function(e){return!e.removed()}},{selector:":grabbable",matches:function(e){return e.grabbable()}},{selector:":ungrabbable",matches:function(e){return!e.grabbable()}},{selector:":animated",matches:function(e){return e.animated()}},{selector:":unanimated",matches:function(e){return!e.animated()}},{selector:":parent",matches:function(e){return e.isParent()}},{selector:":childless",matches:function(e){return e.isChildless()}},{selector:":child",matches:function(e){return e.isChild()}},{selector:":orphan",matches:function(e){return e.isOrphan()}},{selector:":nonorphan",matches:function(e){return e.isChild()}},{selector:":compound",matches:function(e){return e.isNode()?e.isParent():e.source().isParent()||e.target().isParent()}},{selector:":loop",matches:function(e){return e.isLoop()}},{selector:":simple",matches:function(e){return e.isSimple()}},{selector:":active",matches:function(e){return e.active()}},{selector:":inactive",matches:function(e){return!e.active()}},{selector:":backgrounding",matches:function(e){return e.backgrounding()}},{selector:":nonbackgrounding",matches:function(e){return!e.backgrounding()}}].sort((function(e,t){return function(e,t){return-1*z(e,t)}(e.selector,t.selector)})),la=function(){for(var e,t={},n=0;n0&&l.edgeCount>0)return Ve("The selector `"+e+"` is invalid because it uses both a compound selector and an edge selector"),!1;if(l.edgeCount>1)return Ve("The selector `"+e+"` is invalid because it uses multiple edge selectors"),!1;1===l.edgeCount&&Ve("The selector `"+e+"` is deprecated. Edge selectors do not take effect on changes to source and target nodes after an edge is added, for performance reasons. Use a class or data selector on edges instead, updating the class or data of an edge when your app detects a change in source or target nodes.")}return!0},toString:function(){if(null!=this.toStringCache)return this.toStringCache;for(var e=function(e){return null==e?"":e},t=function(t){return f(t)?'"'+t+'"':e(t)},n=function(e){return" "+e+" "},r=function(r,a){var o=r.type,s=r.value;switch(o){case Fi:var l=e(s);return l.substring(0,l.length-1);case Yi:var u=r.field,c=r.operator;return"["+u+n(e(c))+t(s)+"]";case Wi:var d=r.operator,h=r.field;return"["+e(d)+h+"]";case Xi:return"["+r.field+"]";case Hi:var p=r.operator;return"[["+r.field+n(e(p))+t(s)+"]]";case Ki:return s;case Gi:return"#"+s;case Ui:return"."+s;case ra:case ta:return i(r.parent,a)+n(">")+i(r.child,a);case ia:case na:return i(r.ancestor,a)+" "+i(r.descendant,a);case aa:var f=i(r.left,a),g=i(r.subject,a),v=i(r.right,a);return f+(f.length>0?" ":"")+g+v;case oa:return""}},i=function(e,t){return e.checks.reduce((function(n,i,a){return n+(t===e&&0===a?"$":"")+r(i,t)}),"")},a="",o=0;o1&&o=0&&(t=t.replace("!",""),c=!0),t.indexOf("@")>=0&&(t=t.replace("@",""),u=!0),(o||l||u)&&(i=o||s?""+e:"",a=""+n),u&&(e=i=i.toLowerCase(),n=a=a.toLowerCase()),t){case"*=":r=i.indexOf(a)>=0;break;case"$=":r=i.indexOf(a,i.length-a.length)>=0;break;case"^=":r=0===i.indexOf(a);break;case"=":r=e===n;break;case">":d=!0,r=e>n;break;case">=":d=!0,r=e>=n;break;case"<":d=!0,r=e0;){var u=i.shift();t(u),a.add(u.id()),o&&r(i,a,u)}return e}function Da(e,t,n){if(n.isParent())for(var r=n._private.children,i=0;i1&&void 0!==arguments[1])||arguments[1];return Pa(this,e,t,Da)},Sa.forEachUp=function(e){var t=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];return Pa(this,e,t,Ta)},Sa.forEachUpAndDown=function(e){var t=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];return Pa(this,e,t,_a)},Sa.ancestors=Sa.parents,(Ea=ka={data:Li.data({field:"data",bindingEvent:"data",allowBinding:!0,allowSetting:!0,settingEvent:"data",settingTriggersEvent:!0,triggerFnName:"trigger",allowGetting:!0,immutableKeys:{id:!0,source:!0,target:!0,parent:!0},updateStyle:!0}),removeData:Li.removeData({field:"data",event:"data",triggerFnName:"trigger",triggerEvent:!0,immutableKeys:{id:!0,source:!0,target:!0,parent:!0},updateStyle:!0}),scratch:Li.data({field:"scratch",bindingEvent:"scratch",allowBinding:!0,allowSetting:!0,settingEvent:"scratch",settingTriggersEvent:!0,triggerFnName:"trigger",allowGetting:!0,updateStyle:!0}),removeScratch:Li.removeData({field:"scratch",event:"scratch",triggerFnName:"trigger",triggerEvent:!0,updateStyle:!0}),rscratch:Li.data({field:"rscratch",allowBinding:!1,allowSetting:!0,settingTriggersEvent:!1,allowGetting:!0}),removeRscratch:Li.removeData({field:"rscratch",triggerEvent:!1}),id:function(){var e=this[0];if(e)return e._private.data.id}}).attr=Ea.data,Ea.removeAttr=Ea.removeData;var Ma,Ba,Na=ka,za={};function Ia(e){return function(t){if(void 0===t&&(t=!0),0!==this.length&&this.isNode()&&!this.removed()){for(var n=0,r=this[0],i=r._private.edges,a=0;at})),minIndegree:Aa("indegree",(function(e,t){return et})),minOutdegree:Aa("outdegree",(function(e,t){return et}))}),I(za,{totalDegree:function(e){for(var t=0,n=this.nodes(),r=0;r0,c=u;u&&(l=l[0]);var d=c?l.position():{x:0,y:0};return i={x:s.x-d.x,y:s.y-d.y},void 0===e?i:i[e]}for(var h=0;h0,m=v;v&&(g=g[0]);var b=m?g.position():{x:0,y:0};void 0!==t?p.position(e,t+b[e]):void 0!==i&&p.position({x:i.x+b.x,y:i.y+b.y})}}else if(!a)return;return this}}).modelPosition=Ma.point=Ma.position,Ma.modelPositions=Ma.points=Ma.positions,Ma.renderedPoint=Ma.renderedPosition,Ma.relativePoint=Ma.relativePosition;var Ra,Va,Fa=Ba;Ra=Va={},Va.renderedBoundingBox=function(e){var t=this.boundingBox(e),n=this.cy(),r=n.zoom(),i=n.pan(),a=t.x1*r+i.x,o=t.x2*r+i.x,s=t.y1*r+i.y,l=t.y2*r+i.y;return{x1:a,x2:o,y1:s,y2:l,w:o-a,h:l-s}},Va.dirtyCompoundBoundsCache=function(){var e=arguments.length>0&&void 0!==arguments[0]&&arguments[0],t=this.cy();return t.styleEnabled()&&t.hasCompoundNodes()?(this.forEachUp((function(t){if(t.isParent()){var n=t._private;n.compoundBoundsClean=!1,n.bbCache=null,e||t.emitAndNotify("bounds")}})),this):this},Va.updateCompoundBounds=function(){var e=arguments.length>0&&void 0!==arguments[0]&&arguments[0],t=this.cy();if(!t.styleEnabled()||!t.hasCompoundNodes())return this;if(!e&&t.batching())return this;function n(e){if(e.isParent()){var t=e._private,n=e.children(),r="include"===e.pstyle("compound-sizing-wrt-labels").value,i={width:{val:e.pstyle("min-width").pfValue,left:e.pstyle("min-width-bias-left"),right:e.pstyle("min-width-bias-right")},height:{val:e.pstyle("min-height").pfValue,top:e.pstyle("min-height-bias-top"),bottom:e.pstyle("min-height-bias-bottom")}},a=n.boundingBox({includeLabels:r,includeOverlays:!1,useCache:!1}),o=t.position;0!==a.w&&0!==a.h||((a={w:e.pstyle("width").pfValue,h:e.pstyle("height").pfValue}).x1=o.x-a.w/2,a.x2=o.x+a.w/2,a.y1=o.y-a.h/2,a.y2=o.y+a.h/2);var s=i.width.left.value;"px"===i.width.left.units&&i.width.val>0&&(s=100*s/i.width.val);var l=i.width.right.value;"px"===i.width.right.units&&i.width.val>0&&(l=100*l/i.width.val);var u=i.height.top.value;"px"===i.height.top.units&&i.height.val>0&&(u=100*u/i.height.val);var c=i.height.bottom.value;"px"===i.height.bottom.units&&i.height.val>0&&(c=100*c/i.height.val);var d=y(i.width.val-a.w,s,l),h=d.biasDiff,p=d.biasComplementDiff,f=y(i.height.val-a.h,u,c),g=f.biasDiff,v=f.biasComplementDiff;t.autoPadding=function(e,t,n,r){if("%"!==n.units)return"px"===n.units?n.pfValue:0;switch(r){case"width":return e>0?n.pfValue*e:0;case"height":return t>0?n.pfValue*t:0;case"average":return e>0&&t>0?n.pfValue*(e+t)/2:0;case"min":return e>0&&t>0?e>t?n.pfValue*t:n.pfValue*e:0;case"max":return e>0&&t>0?e>t?n.pfValue*e:n.pfValue*t:0;default:return 0}}(a.w,a.h,e.pstyle("padding"),e.pstyle("padding-relative-to").value),t.autoWidth=Math.max(a.w,i.width.val),o.x=(-h+a.x1+a.x2+p)/2,t.autoHeight=Math.max(a.h,i.height.val),o.y=(-g+a.y1+a.y2+v)/2}function y(e,t,n){var r=0,i=0,a=t+n;return e>0&&a>0&&(r=t/a*e,i=n/a*e),{biasDiff:r,biasComplementDiff:i}}}for(var r=0;re.x2?r:e.x2,e.y1=ne.y2?i:e.y2,e.w=e.x2-e.x1,e.h=e.y2-e.y1)},Ya=function(e,t){return null==t?e:qa(e,t.x1,t.y1,t.x2,t.y2)},Xa=function(e,t,n){return Ke(e,t,n)},Wa=function(e,t,n){if(!t.cy().headless()){var r,i,a=t._private,o=a.rstyle,s=o.arrowWidth/2;if("none"!==t.pstyle(n+"-arrow-shape").value){"source"===n?(r=o.srcX,i=o.srcY):"target"===n?(r=o.tgtX,i=o.tgtY):(r=o.midX,i=o.midY);var l=a.arrowBounds=a.arrowBounds||{},u=l[n]=l[n]||{};u.x1=r-s,u.y1=i-s,u.x2=r+s,u.y2=i+s,u.w=u.x2-u.x1,u.h=u.y2-u.y1,_t(u,1),qa(e,u.x1,u.y1,u.x2,u.y2)}}},Ha=function(e,t,n){if(!t.cy().headless()){var r;r=n?n+"-":"";var i=t._private,a=i.rstyle;if(t.pstyle(r+"label").strValue){var o,s,l,u,c=t.pstyle("text-halign"),d=t.pstyle("text-valign"),h=Xa(a,"labelWidth",n),p=Xa(a,"labelHeight",n),f=Xa(a,"labelX",n),g=Xa(a,"labelY",n),v=t.pstyle(r+"text-margin-x").pfValue,y=t.pstyle(r+"text-margin-y").pfValue,m=t.isEdge(),b=t.pstyle(r+"text-rotation"),x=t.pstyle("text-outline-width").pfValue,w=t.pstyle("text-border-width").pfValue/2,E=t.pstyle("text-background-padding").pfValue,k=p,C=h,S=C/2,P=k/2;if(m)o=f-S,s=f+S,l=g-P,u=g+P;else{switch(c.value){case"left":o=f-C,s=f;break;case"center":o=f-S,s=f+S;break;case"right":o=f,s=f+C}switch(d.value){case"top":l=g-k,u=g;break;case"center":l=g-P,u=g+P;break;case"bottom":l=g,u=g+k}}o+=v-Math.max(x,w)-E-2,s+=v+Math.max(x,w)+E+2,l+=y-Math.max(x,w)-E-2,u+=y+Math.max(x,w)+E+2;var D=n||"main",T=i.labelBounds,_=T[D]=T[D]||{};_.x1=o,_.y1=l,_.x2=s,_.y2=u,_.w=s-o,_.h=u-l;var M=m&&"autorotate"===b.strValue,B=null!=b.pfValue&&0!==b.pfValue;if(M||B){var N=M?Xa(i.rstyle,"labelAngle",n):b.pfValue,z=Math.cos(N),I=Math.sin(N),A=(o+s)/2,L=(l+u)/2;if(!m){switch(c.value){case"left":A=s;break;case"right":A=o}switch(d.value){case"top":L=u;break;case"bottom":L=l}}var O=function(e,t){return{x:(e-=A)*z-(t-=L)*I+A,y:e*I+t*z+L}},R=O(o,l),V=O(o,u),F=O(s,l),j=O(s,u);o=Math.min(R.x,V.x,F.x,j.x),s=Math.max(R.x,V.x,F.x,j.x),l=Math.min(R.y,V.y,F.y,j.y),u=Math.max(R.y,V.y,F.y,j.y)}var q=D+"Rot",Y=T[q]=T[q]||{};Y.x1=o,Y.y1=l,Y.x2=s,Y.y2=u,Y.w=s-o,Y.h=u-l,qa(e,o,l,s,u),qa(i.labelBounds.all,o,l,s,u)}return e}},Ka=function(e){var t=0,n=function(e){return(e?1:0)<(r=T[1].x)){var _=n;n=r,r=_}if(i>(a=T[1].y)){var M=i;i=a,a=M}qa(h,n-k,i-k,r+k,a+k)}}else if("bezier"===D||"unbundled-bezier"===D||"segments"===D||"taxi"===D){var B;switch(D){case"bezier":case"unbundled-bezier":B=v.bezierPts;break;case"segments":case"taxi":B=v.linePts}if(null!=B)for(var N=0;N(r=A.x)){var L=n;n=r,r=L}if((i=I.y)>(a=A.y)){var O=i;i=a,a=O}qa(h,n-=k,i-=k,r+=k,a+=k)}if(c&&t.includeEdges&&g&&(Wa(h,e,"mid-source"),Wa(h,e,"mid-target"),Wa(h,e,"source"),Wa(h,e,"target")),c)if("yes"===e.pstyle("ghost").value){var R=e.pstyle("ghost-offset-x").pfValue,V=e.pstyle("ghost-offset-y").pfValue;qa(h,h.x1+R,h.y1+V,h.x2+R,h.y2+V)}var F=p.bodyBounds=p.bodyBounds||{};Bt(F,h),Mt(F,y),_t(F,1),c&&(n=h.x1,r=h.x2,i=h.y1,a=h.y2,qa(h,n-E,i-E,r+E,a+E));var j=p.overlayBounds=p.overlayBounds||{};Bt(j,h),Mt(j,y),_t(j,1);var q=p.labelBounds=p.labelBounds||{};null!=q.all?((l=q.all).x1=1/0,l.y1=1/0,l.x2=-1/0,l.y2=-1/0,l.w=0,l.h=0):q.all=Dt(),c&&t.includeLabels&&(t.includeMainLabels&&Ha(h,e,null),g&&(t.includeSourceLabels&&Ha(h,e,"source"),t.includeTargetLabels&&Ha(h,e,"target")))}return h.x1=ja(h.x1),h.y1=ja(h.y1),h.x2=ja(h.x2),h.y2=ja(h.y2),h.w=ja(h.x2-h.x1),h.h=ja(h.y2-h.y1),h.w>0&&h.h>0&&b&&(Mt(h,y),_t(h,1)),h}(e,Za),r.bbCache=n,r.bbCachePosKey=o):n=r.bbCache,!a){var c=e.isNode();n=Dt(),(t.includeNodes&&c||t.includeEdges&&!c)&&(t.includeOverlays?Ya(n,r.overlayBounds):Ya(n,r.bodyBounds)),t.includeLabels&&(t.includeMainLabels&&(!i||t.includeSourceLabels&&t.includeTargetLabels)?Ya(n,r.labelBounds.all):(t.includeMainLabels&&Ya(n,r.labelBounds.mainRot),t.includeSourceLabels&&Ya(n,r.labelBounds.sourceRot),t.includeTargetLabels&&Ya(n,r.labelBounds.targetRot))),n.w=n.x2-n.x1,n.h=n.y2-n.y1}return n},Za={includeNodes:!0,includeEdges:!0,includeLabels:!0,includeMainLabels:!0,includeSourceLabels:!0,includeTargetLabels:!0,includeOverlays:!0,includeUnderlays:!0,useCache:!0},$a=Ka(Za),Qa=Xe(Za);Va.boundingBox=function(e){var t;if(1!==this.length||null==this[0]._private.bbCache||this[0]._private.styleDirty||void 0!==e&&void 0!==e.useCache&&!0!==e.useCache){t=Dt();var n=Qa(e=e||Za);if(this.cy().styleEnabled())for(var r=0;r0&&void 0!==arguments[0]?arguments[0]:fo,t=arguments.length>1?arguments[1]:void 0,n=0;n=0;s--)o(s);return this},vo.removeAllListeners=function(){return this.removeListener("*")},vo.emit=vo.trigger=function(e,t,n){var r=this.listeners,i=r.length;return this.emitting++,v(t)||(t=[t]),bo(this,(function(e,a){null!=n&&(r=[{event:a.event,type:a.type,namespace:a.namespace,callback:n}],i=r.length);for(var o=function(n){var i=r[n];if(i.type===a.type&&(!i.namespace||i.namespace===a.namespace||".*"===i.namespace)&&e.eventMatches(e.context,i,a)){var o=[a];null!=t&&function(e,t){for(var n=0;n1&&!r){var i=this.length-1,a=this[i],o=a._private.data.id;this[i]=void 0,this[e]=a,n.set(o,{ele:a,index:e})}return this.length--,this},unmergeOne:function(e){e=e[0];var t=this._private,n=e._private.data.id,r=t.map.get(n);if(!r)return this;var i=r.index;return this.unmergeAt(i),this},unmerge:function(e){var t=this._private.cy;if(!e)return this;if(e&&f(e)){var n=e;e=t.mutableElements().filter(n)}for(var r=0;r=0;t--){e(this[t])&&this.unmergeAt(t)}return this},map:function(e,t){for(var n=[],r=0;rr&&(r=o,n=a)}return{value:r,ele:n}},min:function(e,t){for(var n,r=1/0,i=0;i=0&&i1&&void 0!==arguments[1])||arguments[1],n=this[0],r=n.cy();if(r.styleEnabled()&&n){this.cleanStyle();var i=n._private.style[e];return null!=i?i:t?r.style().getDefaultProperty(e):null}},numericStyle:function(e){var t=this[0];if(t.cy().styleEnabled()&&t){var n=t.pstyle(e);return void 0!==n.pfValue?n.pfValue:n.value}},numericStyleUnits:function(e){var t=this[0];if(t.cy().styleEnabled())return t?t.pstyle(e).units:void 0},renderedStyle:function(e){var t=this.cy();if(!t.styleEnabled())return this;var n=this[0];return n?t.style().getRenderedStyle(n,e):void 0},style:function(e,t){var n=this.cy();if(!n.styleEnabled())return this;var r=n.style();if(y(e)){var i=e;r.applyBypass(this,i,!1),this.emitAndNotify("style")}else if(f(e)){if(void 0===t){var a=this[0];return a?r.getStylePropertyValue(a,e):void 0}r.applyBypass(this,e,t,!1),this.emitAndNotify("style")}else if(void 0===e){var o=this[0];return o?r.getRawStyle(o):void 0}return this},removeStyle:function(e){var t=this.cy();if(!t.styleEnabled())return this;var n=t.style();if(void 0===e)for(var r=0;r0&&t.push(c[0]),t.push(s[0])}return this.spawn(t,!0).filter(e)}),"neighborhood"),closedNeighborhood:function(e){return this.neighborhood().add(this).filter(e)},openNeighborhood:function(e){return this.neighborhood(e)}}),Yo.neighbourhood=Yo.neighborhood,Yo.closedNeighbourhood=Yo.closedNeighborhood,Yo.openNeighbourhood=Yo.openNeighborhood,I(Yo,{source:Ca((function(e){var t,n=this[0];return n&&(t=n._private.source||n.cy().collection()),t&&e?t.filter(e):t}),"source"),target:Ca((function(e){var t,n=this[0];return n&&(t=n._private.target||n.cy().collection()),t&&e?t.filter(e):t}),"target"),sources:Ko({attr:"source"}),targets:Ko({attr:"target"})}),I(Yo,{edgesWith:Ca(Go(),"edgesWith"),edgesTo:Ca(Go({thisIsSrc:!0}),"edgesTo")}),I(Yo,{connectedEdges:Ca((function(e){for(var t=[],n=0;n0);return a},component:function(){var e=this[0];return e.cy().mutableElements().components(e)[0]}}),Yo.componentsOf=Yo.components;var Zo=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]&&arguments[2],r=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(void 0!==e){var i=new Ue,a=!1;if(t){if(t.length>0&&y(t[0])&&!w(t[0])){a=!0;for(var o=[],s=new $e,l=0,u=t.length;l0&&void 0!==arguments[0])||arguments[0],r=!(arguments.length>1&&void 0!==arguments[1])||arguments[1],i=this,a=i.cy(),o=a._private,s=[],l=[],u=0,c=i.length;u0){for(var R=e.length===i.length?i:new Zo(a,e),V=0;V0&&void 0!==arguments[0])||arguments[0],t=!(arguments.length>1&&void 0!==arguments[1])||arguments[1],n=this,r=[],i={},a=n._private.cy;function o(e){for(var t=e._private.edges,n=0;n0&&(e?D.emitAndNotify("remove"):t&&D.emit("remove"));for(var T=0;T1e-4&&Math.abs(s.v)>1e-4;);return a?function(e){return u[e*(u.length-1)|0]}:c}}(),es=function(e,t,n,r){var i=function(e,t,n,r){var i=4,a=.001,o=1e-7,s=10,l=11,u=1/(l-1),c="undefined"!=typeof Float32Array;if(4!==arguments.length)return!1;for(var d=0;d<4;++d)if("number"!=typeof arguments[d]||isNaN(arguments[d])||!isFinite(arguments[d]))return!1;e=Math.min(e,1),n=Math.min(n,1),e=Math.max(e,0),n=Math.max(n,0);var h=c?new Float32Array(l):new Array(l);function p(e,t){return 1-3*t+3*e}function f(e,t){return 3*t-6*e}function g(e){return 3*e}function v(e,t,n){return((p(t,n)*e+f(t,n))*e+g(t))*e}function y(e,t,n){return 3*p(t,n)*e*e+2*f(t,n)*e+g(t)}function m(t,r){for(var a=0;a0?i=l:r=l}while(Math.abs(a)>o&&++u=a?m(t,s):0===c?s:x(t,r,r+u)}var E=!1;function k(){E=!0,e===t&&n===r||b()}var C=function(i){return E||k(),e===t&&n===r?i:0===i?0:1===i?1:v(w(i),t,r)};C.getControlPoints=function(){return[{x:e,y:t},{x:n,y:r}]};var S="generateBezier("+[e,t,n,r]+")";return C.toString=function(){return S},C}(e,t,n,r);return function(e,t,n){return e+(t-e)*i(n)}},ts={linear:function(e,t,n){return e+(t-e)*n},ease:es(.25,.1,.25,1),"ease-in":es(.42,0,1,1),"ease-out":es(0,0,.58,1),"ease-in-out":es(.42,0,.58,1),"ease-in-sine":es(.47,0,.745,.715),"ease-out-sine":es(.39,.575,.565,1),"ease-in-out-sine":es(.445,.05,.55,.95),"ease-in-quad":es(.55,.085,.68,.53),"ease-out-quad":es(.25,.46,.45,.94),"ease-in-out-quad":es(.455,.03,.515,.955),"ease-in-cubic":es(.55,.055,.675,.19),"ease-out-cubic":es(.215,.61,.355,1),"ease-in-out-cubic":es(.645,.045,.355,1),"ease-in-quart":es(.895,.03,.685,.22),"ease-out-quart":es(.165,.84,.44,1),"ease-in-out-quart":es(.77,0,.175,1),"ease-in-quint":es(.755,.05,.855,.06),"ease-out-quint":es(.23,1,.32,1),"ease-in-out-quint":es(.86,0,.07,1),"ease-in-expo":es(.95,.05,.795,.035),"ease-out-expo":es(.19,1,.22,1),"ease-in-out-expo":es(1,0,0,1),"ease-in-circ":es(.6,.04,.98,.335),"ease-out-circ":es(.075,.82,.165,1),"ease-in-out-circ":es(.785,.135,.15,.86),spring:function(e,t,n){if(0===n)return ts.linear;var r=Jo(e,t,n);return function(e,t,n){return e+(t-e)*r(n)}},"cubic-bezier":es};function ns(e,t,n,r,i){if(1===r)return n;if(t===n)return n;var a=i(t,n,r);return null==e||((e.roundValue||e.color)&&(a=Math.round(a)),void 0!==e.min&&(a=Math.max(a,e.min)),void 0!==e.max&&(a=Math.min(a,e.max))),a}function rs(e,t){return null!=e.pfValue||null!=e.value?null==e.pfValue||null!=t&&"%"===t.type.units?e.value:e.pfValue:e}function is(e,t,n,r,i){var a=null!=i?i.type:null;n<0?n=0:n>1&&(n=1);var o=rs(e,i),s=rs(t,i);if(m(o)&&m(s))return ns(a,o,s,n,r);if(v(o)&&v(s)){for(var l=[],u=0;u0?("spring"===d&&h.push(o.duration),o.easingImpl=ts[d].apply(null,h)):o.easingImpl=ts[d]}var p,g=o.easingImpl;if(p=0===o.duration?1:(n-l)/o.duration,o.applying&&(p=o.progress),p<0?p=0:p>1&&(p=1),null==o.delay){var v=o.startPosition,y=o.position;if(y&&i&&!e.locked()){var m={};os(v.x,y.x)&&(m.x=is(v.x,y.x,p,g)),os(v.y,y.y)&&(m.y=is(v.y,y.y,p,g)),e.position(m)}var b=o.startPan,x=o.pan,w=a.pan,E=null!=x&&r;E&&(os(b.x,x.x)&&(w.x=is(b.x,x.x,p,g)),os(b.y,x.y)&&(w.y=is(b.y,x.y,p,g)),e.emit("pan"));var k=o.startZoom,C=o.zoom,S=null!=C&&r;S&&(os(k,C)&&(a.zoom=Pt(a.minZoom,is(k,C,p,g),a.maxZoom)),e.emit("zoom")),(E||S)&&e.emit("viewport");var P=o.style;if(P&&P.length>0&&i){for(var D=0;D=0;t--){(0,e[t])()}e.splice(0,e.length)},c=a.length-1;c>=0;c--){var d=a[c],h=d._private;h.stopped?(a.splice(c,1),h.hooked=!1,h.playing=!1,h.started=!1,u(h.frames)):(h.playing||h.applying)&&(h.playing&&h.applying&&(h.applying=!1),h.started||ss(0,d,e),as(t,d,e,n),h.applying&&(h.applying=!1),u(h.frames),null!=h.step&&h.step(e),d.completed()&&(a.splice(c,1),h.hooked=!1,h.playing=!1,h.started=!1,u(h.completes)),s=!0)}return n||0!==a.length||0!==o.length||r.push(t),s}for(var a=!1,o=0;o0?t.notify("draw",n):t.notify("draw")),n.unmerge(r),t.emit("step")}var us={animate:Li.animate(),animation:Li.animation(),animated:Li.animated(),clearQueue:Li.clearQueue(),delay:Li.delay(),delayAnimation:Li.delayAnimation(),stop:Li.stop(),addToAnimationPool:function(e){this.styleEnabled()&&this._private.aniEles.merge(e)},stopAnimationLoop:function(){this._private.animationsRunning=!1},startAnimationLoop:function(){var e=this;if(e._private.animationsRunning=!0,e.styleEnabled()){var t=e.renderer();t&&t.beforeRender?t.beforeRender((function(t,n){ls(n,e)}),t.beforeRenderPriorities.animations):function t(){e._private.animationsRunning&&me((function(n){ls(n,e),t()}))}()}}},cs={qualifierCompare:function(e,t){return null==e||null==t?null==e&&null==t:e.sameText(t)},eventMatches:function(e,t,n){var r=t.qualifier;return null==r||e!==n.target&&w(n.target)&&r.matches(n.target)},addEventFields:function(e,t){t.cy=e,t.target=e},callbackContext:function(e,t,n){return null!=t.qualifier?n.target:e}},ds=function(e){return f(e)?new ba(e):e},hs={createEmitter:function(){var e=this._private;return e.emitter||(e.emitter=new go(cs,this)),this},emitter:function(){return this._private.emitter},on:function(e,t,n){return this.emitter().on(e,ds(t),n),this},removeListener:function(e,t,n){return this.emitter().removeListener(e,ds(t),n),this},removeAllListeners:function(){return this.emitter().removeAllListeners(),this},one:function(e,t,n){return this.emitter().one(e,ds(t),n),this},once:function(e,t,n){return this.emitter().one(e,ds(t),n),this},emit:function(e,t){return this.emitter().emit(e,t),this},emitAndNotify:function(e,t){return this.emit(e),this.notify(e,t),this}};Li.eventAliasesOn(hs);var ps={png:function(e){return e=e||{},this._private.renderer.png(e)},jpg:function(e){var t=this._private.renderer;return(e=e||{}).bg=e.bg||"#fff",t.jpg(e)}};ps.jpeg=ps.jpg;var fs={layout:function(e){if(null!=e)if(null!=e.name){var t=e.name,n=this.extension("layout",t);if(null!=n){var r;r=f(e.eles)?this.$(e.eles):null!=e.eles?e.eles:this.$();var i=new n(I({},e,{cy:this,eles:r}));return i}Oe("No such layout `"+t+"` found. Did you forget to import it and `cytoscape.use()` it?")}else Oe("A `name` must be specified to make a layout");else Oe("Layout options must be specified to make a layout")}};fs.createLayout=fs.makeLayout=fs.layout;var gs={notify:function(e,t){var n=this._private;if(this.batching()){n.batchNotifications=n.batchNotifications||{};var r=n.batchNotifications[e]=n.batchNotifications[e]||this.collection();null!=t&&r.merge(t)}else if(n.notificationsEnabled){var i=this.renderer();!this.destroyed()&&i&&i.notify(e,t)}},notifications:function(e){var t=this._private;return void 0===e?t.notificationsEnabled:(t.notificationsEnabled=!!e,this)},noNotifications:function(e){this.notifications(!1),e(),this.notifications(!0)},batching:function(){return this._private.batchCount>0},startBatch:function(){var e=this._private;return null==e.batchCount&&(e.batchCount=0),0===e.batchCount&&(e.batchStyleEles=this.collection(),e.batchNotifications={}),e.batchCount++,this},endBatch:function(){var e=this._private;if(0===e.batchCount)return this;if(e.batchCount--,0===e.batchCount){e.batchStyleEles.updateStyle();var t=this.renderer();Object.keys(e.batchNotifications).forEach((function(n){var r=e.batchNotifications[n];r.empty()?t.notify(n):t.notify(n,r)}))}return this},batch:function(e){return this.startBatch(),e(),this.endBatch(),this},batchData:function(e){var t=this;return this.batch((function(){for(var n=Object.keys(e),r=0;r0;)e.removeChild(e.childNodes[0]);this._private.renderer=null,this.mutableElements().forEach((function(e){var t=e._private;t.rscratch={},t.rstyle={},t.animation.current=[],t.animation.queue=[]}))},onRender:function(e){return this.on("render",e)},offRender:function(e){return this.off("render",e)}};ys.invalidateDimensions=ys.resize;var ms={collection:function(e,t){return f(e)?this.$(e):x(e)?e.collection():v(e)?(t||(t={}),new Zo(this,e,t.unique,t.removed)):new Zo(this)},nodes:function(e){var t=this.$((function(e){return e.isNode()}));return e?t.filter(e):t},edges:function(e){var t=this.$((function(e){return e.isEdge()}));return e?t.filter(e):t},$:function(e){var t=this._private.elements;return e?t.filter(e):t.spawnSelf()},mutableElements:function(){return this._private.elements}};ms.elements=ms.filter=ms.$;var bs={};bs.apply=function(e){for(var t=this._private.cy.collection(),n=0;n0;if(d||c&&h){var p=void 0;d&&h||d?p=l.properties:h&&(p=l.mappedProperties);for(var f=0;f1&&(g=1),s.color){var w=i.valueMin[0],E=i.valueMax[0],k=i.valueMin[1],C=i.valueMax[1],S=i.valueMin[2],P=i.valueMax[2],D=null==i.valueMin[3]?1:i.valueMin[3],T=null==i.valueMax[3]?1:i.valueMax[3],_=[Math.round(w+(E-w)*g),Math.round(k+(C-k)*g),Math.round(S+(P-S)*g),Math.round(D+(T-D)*g)];n={bypass:i.bypass,name:i.name,value:_,strValue:"rgb("+_[0]+", "+_[1]+", "+_[2]+")"}}else{if(!s.number)return!1;var M=i.valueMin+(i.valueMax-i.valueMin)*g;n=this.parse(i.name,M,i.bypass,"mapping")}if(!n)return f(),!1;n.mapping=i,i=n;break;case o.data:for(var B=i.field.split("."),N=d.data,z=0;z0&&a>0){for(var s={},l=!1,u=0;u0?e.delayAnimation(o).play().promise().then(t):t()})).then((function(){return e.animation({style:s,duration:a,easing:e.pstyle("transition-timing-function").value,queue:!1}).play().promise()})).then((function(){n.removeBypasses(e,i),e.emitAndNotify("style"),r.transitioning=!1}))}else r.transitioning&&(this.removeBypasses(e,i),e.emitAndNotify("style"),r.transitioning=!1)},bs.checkTrigger=function(e,t,n,r,i,a){var o=this.properties[t],s=i(o);null!=s&&s(n,r)&&a(o)},bs.checkZOrderTrigger=function(e,t,n,r){var i=this;this.checkTrigger(e,t,n,r,(function(e){return e.triggersZOrder}),(function(){i._private.cy.notify("zorder",e)}))},bs.checkBoundsTrigger=function(e,t,n,r){this.checkTrigger(e,t,n,r,(function(e){return e.triggersBounds}),(function(i){e.dirtyCompoundBoundsCache(),e.dirtyBoundingBoxCache(),!i.triggersBoundsOfParallelBeziers||("curve-style"!==t||"bezier"!==n&&"bezier"!==r)&&("display"!==t||"none"!==n&&"none"!==r)||e.parallelEdges().forEach((function(e){e.isBundledBezier()&&e.dirtyBoundingBoxCache()}))}))},bs.checkTriggers=function(e,t,n,r){e.dirtyStyleCache(),this.checkZOrderTrigger(e,t,n,r),this.checkBoundsTrigger(e,t,n,r)};var xs={applyBypass:function(e,t,n,r){var i=[];if("*"===t||"**"===t){if(void 0!==n)for(var a=0;at.length?i.substr(t.length):""}function o(){n=n.length>r.length?n.substr(r.length):""}for(i=i.replace(/[/][*](\s|.)+?[*][/]/g,"");;){if(i.match(/^\s*$/))break;var s=i.match(/^\s*((?:.|\s)+?)\s*\{((?:.|\s)+?)\}/);if(!s){Ve("Halting stylesheet parsing: String stylesheet contains more to parse but no selector and block found in: "+i);break}t=s[0];var l=s[1];if("core"!==l)if(new ba(l).invalid){Ve("Skipping parsing of block: Invalid selector found in string stylesheet: "+l),a();continue}var u=s[2],c=!1;n=u;for(var d=[];;){if(n.match(/^\s*$/))break;var h=n.match(/^\s*(.+?)\s*:\s*(.+?)(?:\s*;|\s*$)/);if(!h){Ve("Skipping parsing of block: Invalid formatting of style property and value definitions found in:"+u),c=!0;break}r=h[0];var p=h[1],f=h[2];if(this.properties[p])this.parse(p,f)?(d.push({name:p,val:f}),o()):(Ve("Skipping property: Invalid property definition in: "+r),o());else Ve("Skipping property: Invalid property name in: "+r),o()}if(c){a();break}this.selector(l);for(var g=0;g=7&&"d"===t[0]&&(l=new RegExp(o.data.regex).exec(t))){if(n)return!1;var d=o.data;return{name:e,value:l,strValue:""+t,mapped:d,field:l[1],bypass:n}}if(t.length>=10&&"m"===t[0]&&(u=new RegExp(o.mapData.regex).exec(t))){if(n)return!1;if(c.multiple)return!1;var h=o.mapData;if(!c.color&&!c.number)return!1;var p=this.parse(e,u[4]);if(!p||p.mapped)return!1;var y=this.parse(e,u[5]);if(!y||y.mapped)return!1;if(p.pfValue===y.pfValue||p.strValue===y.strValue)return Ve("`"+e+": "+t+"` is not a valid mapper because the output range is zero; converting to `"+e+": "+p.strValue+"`"),this.parse(e,p.strValue);if(c.color){var b=p.value,x=y.value;if(!(b[0]!==x[0]||b[1]!==x[1]||b[2]!==x[2]||b[3]!==x[3]&&(null!=b[3]&&1!==b[3]||null!=x[3]&&1!==x[3])))return!1}return{name:e,value:u,strValue:""+t,mapped:h,field:u[1],fieldMin:parseFloat(u[2]),fieldMax:parseFloat(u[3]),valueMin:p.value,valueMax:y.value,bypass:n}}}if(c.multiple&&"multiple"!==r){var w;if(w=s?t.split(/\s+/):v(t)?t:[t],c.evenMultiple&&w.length%2!=0)return null;for(var E=[],k=[],C=[],S="",P=!1,D=0;D0?" ":"")+_.strValue}return c.validate&&!c.validate(E,k)?null:c.singleEnum&&P?1===E.length&&f(E[0])?{name:e,value:E[0],strValue:E[0],bypass:n}:null:{name:e,value:E,pfValue:C,strValue:S,bypass:n,units:k}}var M,B,z=function(){for(var r=0;rc.max||c.strictMax&&t===c.max))return null;var V={name:e,value:t,strValue:""+t+(I||""),units:I,bypass:n};return c.unitless||"px"!==I&&"em"!==I?V.pfValue=t:V.pfValue="px"!==I&&I?this.getEmSizeInPixels()*t:t,"ms"!==I&&"s"!==I||(V.pfValue="ms"===I?t:1e3*t),"deg"!==I&&"rad"!==I||(V.pfValue="rad"===I?t:(M=t,Math.PI*M/180)),"%"===I&&(V.pfValue=t/100),V}if(c.propList){var F=[],j=""+t;if("none"===j);else{for(var q=j.split(/\s*,\s*|\s+/),Y=0;Y0&&l>0&&!isNaN(n.w)&&!isNaN(n.h)&&n.w>0&&n.h>0)return{zoom:o=(o=(o=Math.min((s-2*t)/n.w,(l-2*t)/n.h))>this._private.maxZoom?this._private.maxZoom:o)=n.minZoom&&(n.maxZoom=t),this},minZoom:function(e){return void 0===e?this._private.minZoom:this.zoomRange({min:e})},maxZoom:function(e){return void 0===e?this._private.maxZoom:this.zoomRange({max:e})},getZoomedViewport:function(e){var t,n,r=this._private,i=r.pan,a=r.zoom,o=!1;if(r.zoomingEnabled||(o=!0),m(e)?n=e:y(e)&&(n=e.level,null!=e.position?t=gt(e.position,a,i):null!=e.renderedPosition&&(t=e.renderedPosition),null==t||r.panningEnabled||(o=!0)),n=(n=n>r.maxZoom?r.maxZoom:n)t.maxZoom||!t.zoomingEnabled?a=!0:(t.zoom=s,i.push("zoom"))}if(r&&(!a||!e.cancelOnFailedZoom)&&t.panningEnabled){var l=e.pan;m(l.x)&&(t.pan.x=l.x,o=!1),m(l.y)&&(t.pan.y=l.y,o=!1),o||i.push("pan")}return i.length>0&&(i.push("viewport"),this.emit(i.join(" ")),this.notify("viewport")),this},center:function(e){var t=this.getCenterPan(e);return t&&(this._private.pan=t,this.emit("pan viewport"),this.notify("viewport")),this},getCenterPan:function(e,t){if(this._private.panningEnabled){if(f(e)){var n=e;e=this.mutableElements().filter(n)}else x(e)||(e=this.mutableElements());if(0!==e.length){var r=e.boundingBox(),i=this.width(),a=this.height();return{x:(i-(t=void 0===t?this._private.zoom:t)*(r.x1+r.x2))/2,y:(a-t*(r.y1+r.y2))/2}}}},reset:function(){return this._private.panningEnabled&&this._private.zoomingEnabled?(this.viewport({pan:{x:0,y:0},zoom:1}),this):this},invalidateSize:function(){this._private.sizeCache=null},size:function(){var e,t,n=this._private,r=n.container;return n.sizeCache=n.sizeCache||(r?(e=s.getComputedStyle(r),t=function(t){return parseFloat(e.getPropertyValue(t))},{width:r.clientWidth-t("padding-left")-t("padding-right"),height:r.clientHeight-t("padding-top")-t("padding-bottom")}):{width:1,height:1})},width:function(){return this.size().width},height:function(){return this.size().height},extent:function(){var e=this._private.pan,t=this._private.zoom,n=this.renderedExtent(),r={x1:(n.x1-e.x)/t,x2:(n.x2-e.x)/t,y1:(n.y1-e.y)/t,y2:(n.y2-e.y)/t};return r.w=r.x2-r.x1,r.h=r.y2-r.y1,r},renderedExtent:function(){var e=this.width(),t=this.height();return{x1:0,y1:0,x2:e,y2:t,w:e,h:t}},multiClickDebounceTime:function(e){return e?(this._private.multiClickDebounceTime=e,this):this._private.multiClickDebounceTime}};Ms.centre=Ms.center,Ms.autolockNodes=Ms.autolock,Ms.autoungrabifyNodes=Ms.autoungrabify;var Bs={data:Li.data({field:"data",bindingEvent:"data",allowBinding:!0,allowSetting:!0,settingEvent:"data",settingTriggersEvent:!0,triggerFnName:"trigger",allowGetting:!0,updateStyle:!0}),removeData:Li.removeData({field:"data",event:"data",triggerFnName:"trigger",triggerEvent:!0,updateStyle:!0}),scratch:Li.data({field:"scratch",bindingEvent:"scratch",allowBinding:!0,allowSetting:!0,settingEvent:"scratch",settingTriggersEvent:!0,triggerFnName:"trigger",allowGetting:!0,updateStyle:!0}),removeScratch:Li.removeData({field:"scratch",event:"scratch",triggerFnName:"trigger",triggerEvent:!0,updateStyle:!0})};Bs.attr=Bs.data,Bs.removeAttr=Bs.removeData;var Ns=function(e){var t=this,n=(e=I({},e)).container;n&&!b(n)&&b(n[0])&&(n=n[0]);var r=n?n._cyreg:null;(r=r||{})&&r.cy&&(r.cy.destroy(),r={});var i=r.readies=r.readies||[];n&&(n._cyreg=r),r.cy=t;var a=void 0!==s&&void 0!==n&&!e.headless,o=e;o.layout=I({name:a?"grid":"null"},o.layout),o.renderer=I({name:a?"canvas":"null"},o.renderer);var l=function(e,t,n){return void 0!==t?t:void 0!==n?n:e},u=this._private={container:n,ready:!1,options:o,elements:new Zo(this),listeners:[],aniEles:new Zo(this),data:o.data||{},scratch:{},layout:null,renderer:null,destroyed:!1,notificationsEnabled:!0,minZoom:1e-50,maxZoom:1e50,zoomingEnabled:l(!0,o.zoomingEnabled),userZoomingEnabled:l(!0,o.userZoomingEnabled),panningEnabled:l(!0,o.panningEnabled),userPanningEnabled:l(!0,o.userPanningEnabled),boxSelectionEnabled:l(!0,o.boxSelectionEnabled),autolock:l(!1,o.autolock,o.autolockNodes),autoungrabify:l(!1,o.autoungrabify,o.autoungrabifyNodes),autounselectify:l(!1,o.autounselectify),styleEnabled:void 0===o.styleEnabled?a:o.styleEnabled,zoom:m(o.zoom)?o.zoom:1,pan:{x:y(o.pan)&&m(o.pan.x)?o.pan.x:0,y:y(o.pan)&&m(o.pan.y)?o.pan.y:0},animation:{current:[],queue:[]},hasCompoundNodes:!1,multiClickDebounceTime:l(250,o.multiClickDebounceTime)};this.createEmitter(),this.selectionType(o.selectionType),this.zoomRange({min:o.minZoom,max:o.maxZoom});u.styleEnabled&&t.setStyle([]);var c=I({},o,o.renderer);t.initRenderer(c);!function(e,t){if(e.some(P))return hr.all(e).then(t);t(e)}([o.style,o.elements],(function(e){var n=e[0],a=e[1];u.styleEnabled&&t.style().append(n),function(e,n,r){t.notifications(!1);var i=t.mutableElements();i.length>0&&i.remove(),null!=e&&(y(e)||v(e))&&t.add(e),t.one("layoutready",(function(e){t.notifications(!0),t.emit(e),t.one("load",n),t.emitAndNotify("load")})).one("layoutstop",(function(){t.one("done",r),t.emit("done")}));var a=I({},t._private.options.layout);a.eles=t.elements(),t.layout(a).run()}(a,(function(){t.startAnimationLoop(),u.ready=!0,g(o.ready)&&t.on("ready",o.ready);for(var e=0;e0,u=Dt(n.boundingBox?n.boundingBox:{x1:0,y1:0,w:r.width(),h:r.height()});if(x(n.roots))e=n.roots;else if(v(n.roots)){for(var c=[],d=0;d0;){var N=_.shift(),I=T(N,M);if(I)N.outgoers().filter((function(e){return e.isNode()&&i.has(e)})).forEach(B);else if(null===I){Ve("Detected double maximal shift for node `"+N.id()+"`. Bailing maximal adjustment due to cycle. Use `options.maximal: true` only on DAGs.");break}}}D();var A=0;if(n.avoidOverlap)for(var L=0;L0&&b[0].length<=3?l/2:0),d=2*Math.PI/b[r].length*i;return 0===r&&1===b[0].length&&(c=1),{x:G+c*Math.cos(d),y:U+c*Math.sin(d)}}return{x:G+(i+1-(a+1)/2)*o,y:(r+1)*s}})),this};var Vs={fit:!0,padding:30,boundingBox:void 0,avoidOverlap:!0,nodeDimensionsIncludeLabels:!1,spacingFactor:void 0,radius:void 0,startAngle:1.5*Math.PI,sweep:void 0,clockwise:!0,sort:void 0,animate:!1,animationDuration:500,animationEasing:void 0,animateFilter:function(e,t){return!0},ready:void 0,stop:void 0,transform:function(e,t){return t}};function Fs(e){this.options=I({},Vs,e)}Fs.prototype.run=function(){var e=this.options,t=e,n=e.cy,r=t.eles,i=void 0!==t.counterclockwise?!t.counterclockwise:t.clockwise,a=r.nodes().not(":parent");t.sort&&(a=a.sort(t.sort));for(var o,s=Dt(t.boundingBox?t.boundingBox:{x1:0,y1:0,w:n.width(),h:n.height()}),l=s.x1+s.w/2,u=s.y1+s.h/2,c=(void 0===t.sweep?2*Math.PI-2*Math.PI/a.length:t.sweep)/Math.max(1,a.length-1),d=0,h=0;h1&&t.avoidOverlap){d*=1.75;var v=Math.cos(c)-Math.cos(0),y=Math.sin(c)-Math.sin(0),b=Math.sqrt(d*d/(v*v+y*y));o=Math.max(b,o)}return r.nodes().layoutPositions(this,t,(function(e,n){var r=t.startAngle+n*c*(i?1:-1),a=o*Math.cos(r),s=o*Math.sin(r);return{x:l+a,y:u+s}})),this};var js,qs={fit:!0,padding:30,startAngle:1.5*Math.PI,sweep:void 0,clockwise:!0,equidistant:!1,minNodeSpacing:10,boundingBox:void 0,avoidOverlap:!0,nodeDimensionsIncludeLabels:!1,height:void 0,width:void 0,spacingFactor:void 0,concentric:function(e){return e.degree()},levelWidth:function(e){return e.maxDegree()/4},animate:!1,animationDuration:500,animationEasing:void 0,animateFilter:function(e,t){return!0},ready:void 0,stop:void 0,transform:function(e,t){return t}};function Ys(e){this.options=I({},qs,e)}Ys.prototype.run=function(){for(var e=this.options,t=e,n=void 0!==t.counterclockwise?!t.counterclockwise:t.clockwise,r=e.cy,i=t.eles,a=i.nodes().not(":parent"),o=Dt(t.boundingBox?t.boundingBox:{x1:0,y1:0,w:r.width(),h:r.height()}),s=o.x1+o.w/2,l=o.y1+o.h/2,u=[],c=0,d=0;d0)Math.abs(m[0].value-x.value)>=v&&(m=[],y.push(m));m.push(x)}var w=c+t.minNodeSpacing;if(!t.avoidOverlap){var E=y.length>0&&y[0].length>1,k=(Math.min(o.w,o.h)/2-w)/(y.length+E?1:0);w=Math.min(w,k)}for(var C=0,S=0;S1&&t.avoidOverlap){var _=Math.cos(T)-Math.cos(0),M=Math.sin(T)-Math.sin(0),B=Math.sqrt(w*w/(_*_+M*M));C=Math.max(B,C)}P.r=C,C+=w}if(t.equidistant){for(var N=0,z=0,I=0;I=e.numIter)&&(Qs(r,e),r.temperature=r.temperature*e.coolingFactor,!(r.temperature=e.animationThreshold&&a(),me(t)):(cl(r,e),s())}()}else{for(;u;)u=o(l),l++;cl(r,e),s()}return this},Ws.prototype.stop=function(){return this.stopped=!0,this.thread&&this.thread.stop(),this.emit("layoutstop"),this},Ws.prototype.destroy=function(){return this.thread&&this.thread.stop(),this};var Hs=function(e,t,n){for(var r=n.eles.edges(),i=n.eles.nodes(),a=Dt(n.boundingBox?n.boundingBox:{x1:0,y1:0,w:e.width(),h:e.height()}),o={isCompound:e.hasCompoundNodes(),layoutNodes:[],idToIndex:{},nodeSize:i.size(),graphSet:[],indexToGraph:[],layoutEdges:[],edgeSize:r.size(),temperature:n.initialTemp,clientWidth:a.w,clientHeight:a.h,boundingBox:a},s=n.eles.components(),l={},u=0;u0){o.graphSet.push(E);for(u=0;ur.count?0:r.graph},Gs=function e(t,n,r,i){var a=i.graphSet[r];if(-10)var s=(u=r.nodeOverlap*o)*i/(g=Math.sqrt(i*i+a*a)),l=u*a/g;else{var u,c=rl(e,i,a),d=rl(t,-1*i,-1*a),h=d.x-c.x,p=d.y-c.y,f=h*h+p*p,g=Math.sqrt(f);s=(u=(e.nodeRepulsion+t.nodeRepulsion)/f)*h/g,l=u*p/g}e.isLocked||(e.offsetX-=s,e.offsetY-=l),t.isLocked||(t.offsetX+=s,t.offsetY+=l)}},nl=function(e,t,n,r){if(n>0)var i=e.maxX-t.minX;else i=t.maxX-e.minX;if(r>0)var a=e.maxY-t.minY;else a=t.maxY-e.minY;return i>=0&&a>=0?Math.sqrt(i*i+a*a):0},rl=function(e,t,n){var r=e.positionX,i=e.positionY,a=e.height||1,o=e.width||1,s=n/t,l=a/o,u={};return 0===t&&0n?(u.x=r,u.y=i+a/2,u):0t&&-1*l<=s&&s<=l?(u.x=r-o/2,u.y=i-o*n/2/t,u):0=l)?(u.x=r+a*t/2/n,u.y=i+a/2,u):0>n&&(s<=-1*l||s>=l)?(u.x=r-a*t/2/n,u.y=i-a/2,u):u},il=function(e,t){for(var n=0;n1){var f=t.gravity*d/p,g=t.gravity*h/p;c.offsetX+=f,c.offsetY+=g}}}}},ol=function(e,t){var n=[],r=0,i=-1;for(n.push.apply(n,e.graphSet[0]),i+=e.graphSet[0].length;r<=i;){var a=n[r++],o=e.idToIndex[a],s=e.layoutNodes[o],l=s.children;if(0n)var i={x:n*e/r,y:n*t/r};else i={x:e,y:t};return i},ul=function e(t,n){var r=t.parentId;if(null!=r){var i=n.layoutNodes[n.idToIndex[r]],a=!1;return(null==i.maxX||t.maxX+i.padRight>i.maxX)&&(i.maxX=t.maxX+i.padRight,a=!0),(null==i.minX||t.minX-i.padLefti.maxY)&&(i.maxY=t.maxY+i.padBottom,a=!0),(null==i.minY||t.minY-i.padTopf&&(d+=p+t.componentSpacing,c=0,h=0,p=0)}}},dl={fit:!0,padding:30,boundingBox:void 0,avoidOverlap:!0,avoidOverlapPadding:10,nodeDimensionsIncludeLabels:!1,spacingFactor:void 0,condense:!1,rows:void 0,cols:void 0,position:function(e){},sort:void 0,animate:!1,animationDuration:500,animationEasing:void 0,animateFilter:function(e,t){return!0},ready:void 0,stop:void 0,transform:function(e,t){return t}};function hl(e){this.options=I({},dl,e)}hl.prototype.run=function(){var e=this.options,t=e,n=e.cy,r=t.eles,i=r.nodes().not(":parent");t.sort&&(i=i.sort(t.sort));var a=Dt(t.boundingBox?t.boundingBox:{x1:0,y1:0,w:n.width(),h:n.height()});if(0===a.h||0===a.w)r.nodes().layoutPositions(this,t,(function(e){return{x:a.x1,y:a.y1}}));else{var o=i.size(),s=Math.sqrt(o*a.h/a.w),l=Math.round(s),u=Math.round(a.w/a.h*s),c=function(e){if(null==e)return Math.min(l,u);Math.min(l,u)==l?l=e:u=e},d=function(e){if(null==e)return Math.max(l,u);Math.max(l,u)==l?l=e:u=e},h=t.rows,p=null!=t.cols?t.cols:t.columns;if(null!=h&&null!=p)l=h,u=p;else if(null!=h&&null==p)l=h,u=Math.ceil(o/l);else if(null==h&&null!=p)u=p,l=Math.ceil(o/u);else if(u*l>o){var f=c(),g=d();(f-1)*g>=o?c(f-1):(g-1)*f>=o&&d(g-1)}else for(;u*l=o?d(y+1):c(v+1)}var m=a.w/u,b=a.h/l;if(t.condense&&(m=0,b=0),t.avoidOverlap)for(var x=0;x=u&&(B=0,M++)},z={},I=0;I(r=Vt(e,t,x[w],x[w+1],x[w+2],x[w+3])))return v(n,r),!0}else if("bezier"===a.edgeType||"multibezier"===a.edgeType||"self"===a.edgeType||"compound"===a.edgeType)for(x=a.allpts,w=0;w+5(r=Rt(e,t,x[w],x[w+1],x[w+2],x[w+3],x[w+4],x[w+5])))return v(n,r),!0;m=m||i.source,b=b||i.target;var E=o.getArrowWidth(l,c),k=[{name:"source",x:a.arrowStartX,y:a.arrowStartY,angle:a.srcArrowAngle},{name:"target",x:a.arrowEndX,y:a.arrowEndY,angle:a.tgtArrowAngle},{name:"mid-source",x:a.midX,y:a.midY,angle:a.midsrcArrowAngle},{name:"mid-target",x:a.midX,y:a.midY,angle:a.midtgtArrowAngle}];for(w=0;w0&&(y(m),y(b))}function b(e,t,n){return Ke(e,t,n)}function x(n,r){var i,a=n._private,o=f;i=r?r+"-":"",n.boundingBox();var s=a.labelBounds[r||"main"],l=n.pstyle(i+"label").value;if("yes"===n.pstyle("text-events").strValue&&l){var u=b(a.rscratch,"labelX",r),c=b(a.rscratch,"labelY",r),d=b(a.rscratch,"labelAngle",r),h=n.pstyle(i+"text-margin-x").pfValue,p=n.pstyle(i+"text-margin-y").pfValue,g=s.x1-o-h,y=s.x2+o-h,m=s.y1-o-p,x=s.y2+o-p;if(d){var w=Math.cos(d),E=Math.sin(d),k=function(e,t){return{x:(e-=u)*w-(t-=c)*E+u,y:e*E+t*w+c}},C=k(g,m),S=k(g,x),P=k(y,m),D=k(y,x),T=[C.x+h,C.y+p,P.x+h,P.y+p,D.x+h,D.y+p,S.x+h,S.y+p];if(Ft(e,t,T))return v(n),!0}else if(zt(s,e,t))return v(n),!0}}n&&(l=l.interactive);for(var w=l.length-1;w>=0;w--){var E=l[w];E.isNode()?y(E)||x(E):m(E)||x(E)||x(E,"source")||x(E,"target")}return u},getAllInBox:function(e,t,n,r){for(var i,a,o=this.getCachedZSortedEles().interactive,s=[],l=Math.min(e,n),u=Math.max(e,n),c=Math.min(t,r),d=Math.max(t,r),h=Dt({x1:e=l,y1:t=c,x2:n=u,y2:r=d}),p=0;p0?Math.max(e-t,0):Math.min(e+t,0)},w=x(m,v),E=x(b,y),k=!1;"auto"===c?u=Math.abs(w)>Math.abs(E)?"horizontal":"vertical":"upward"===c||"downward"===c?(u="vertical",k=!0):"leftward"!==c&&"rightward"!==c||(u="horizontal",k=!0);var C,S="vertical"===u,P=S?E:w,D=S?b:m,T=xt(D),_=!1;(k&&(h||f)||!("downward"===c&&D<0||"upward"===c&&D>0||"leftward"===c&&D>0||"rightward"===c&&D<0)||(P=(T*=-1)*Math.abs(P),_=!0),h)?C=(p<0?1+p:p)*P:C=(p<0?P:0)+p*T;var M=function(e){return Math.abs(e)=Math.abs(P)},B=M(C),N=M(Math.abs(P)-Math.abs(C));if((B||N)&&!_)if(S){var z=Math.abs(D)<=a/2,I=Math.abs(m)<=o/2;if(z){var A=(r.x1+r.x2)/2,L=r.y1,O=r.y2;n.segpts=[A,L,A,O]}else if(I){var R=(r.y1+r.y2)/2,V=r.x1,F=r.x2;n.segpts=[V,R,F,R]}else n.segpts=[r.x1,r.y2]}else{var j=Math.abs(D)<=i/2,q=Math.abs(b)<=s/2;if(j){var Y=(r.y1+r.y2)/2,X=r.x1,W=r.x2;n.segpts=[X,Y,W,Y]}else if(q){var H=(r.x1+r.x2)/2,K=r.y1,G=r.y2;n.segpts=[H,K,H,G]}else n.segpts=[r.x2,r.y1]}else if(S){var U=r.y1+C+(l?a/2*T:0),Z=r.x1,$=r.x2;n.segpts=[Z,U,$,U]}else{var Q=r.x1+C+(l?i/2*T:0),J=r.y1,ee=r.y2;n.segpts=[Q,J,Q,ee]}},Pl.tryToCorrectInvalidPoints=function(e,t){var n=e._private.rscratch;if("bezier"===n.edgeType){var r=t.srcPos,i=t.tgtPos,a=t.srcW,o=t.srcH,s=t.tgtW,l=t.tgtH,u=t.srcShape,c=t.tgtShape,d=!m(n.startX)||!m(n.startY),h=!m(n.arrowStartX)||!m(n.arrowStartY),p=!m(n.endX)||!m(n.endY),f=!m(n.arrowEndX)||!m(n.arrowEndY),g=3*(this.getArrowWidth(e.pstyle("width").pfValue,e.pstyle("arrow-scale").value)*this.arrowShapeWidth),v=wt({x:n.ctrlpts[0],y:n.ctrlpts[1]},{x:n.startX,y:n.startY}),y=vh.poolIndex()){var p=d;d=h,h=p}var f=s.srcPos=d.position(),g=s.tgtPos=h.position(),v=s.srcW=d.outerWidth(),y=s.srcH=d.outerHeight(),b=s.tgtW=h.outerWidth(),x=s.tgtH=h.outerHeight(),w=s.srcShape=n.nodeShapes[t.getNodeShape(d)],E=s.tgtShape=n.nodeShapes[t.getNodeShape(h)];s.dirCounts={north:0,west:0,south:0,east:0,northwest:0,southwest:0,northeast:0,southeast:0};for(var k=0;k0){var q=u,Y=Et(q,yt(t)),X=Et(q,yt(j)),W=Y;if(X2)Et(q,{x:j[2],y:j[3]})0){var ie=c,ae=Et(ie,yt(t)),oe=Et(ie,yt(re)),se=ae;if(oe2)Et(ie,{x:re[2],y:re[3]})=c||b){d={cp:v,segment:m};break}}if(d)break}var x=d.cp,w=d.segment,E=(c-p)/w.length,k=w.t1-w.t0,C=u?w.t0+k*E:w.t1-k*E;C=Pt(0,C,1),t=St(x.p0,x.p1,x.p2,C),l=function(e,t,n,r){var i=Pt(0,r-.001,1),a=Pt(0,r+.001,1),o=St(e,t,n,i),s=St(e,t,n,a);return zl(o,s)}(x.p0,x.p1,x.p2,C);break;case"straight":case"segments":case"haystack":for(var S,P,D,T,_=0,M=r.allpts.length,B=0;B+3=c));B+=2);var N=(c-P)/S;N=Pt(0,N,1),t=function(e,t,n,r){var i=t.x-e.x,a=t.y-e.y,o=wt(e,t),s=i/o,l=a/o;return n=null==n?0:n,r=null!=r?r:n*o,{x:e.x+s*r,y:e.y+l*r}}(D,T,N),l=zl(D,T)}o("labelX",s,t.x),o("labelY",s,t.y),o("labelAutoAngle",s,l)}};l("source"),l("target"),this.applyLabelDimensions(e)}},Bl.applyLabelDimensions=function(e){this.applyPrefixedLabelDimensions(e),e.isEdge()&&(this.applyPrefixedLabelDimensions(e,"source"),this.applyPrefixedLabelDimensions(e,"target"))},Bl.applyPrefixedLabelDimensions=function(e,t){var n=e._private,r=this.getLabelText(e,t),i=this.calculateLabelDimensions(e,r),a=e.pstyle("line-height").pfValue,o=e.pstyle("text-wrap").strValue,s=Ke(n.rscratch,"labelWrapCachedLines",t)||[],l="wrap"!==o?1:Math.max(s.length,1),u=i.height/l,c=u*a,d=i.width,h=i.height+(l-1)*(a-1)*u;Ge(n.rstyle,"labelWidth",t,d),Ge(n.rscratch,"labelWidth",t,d),Ge(n.rstyle,"labelHeight",t,h),Ge(n.rscratch,"labelHeight",t,h),Ge(n.rscratch,"labelLineHeight",t,c)},Bl.getLabelText=function(e,t){var n=e._private,r=t?t+"-":"",i=e.pstyle(r+"label").strValue,a=e.pstyle("text-transform").value,o=function(e,r){return r?(Ge(n.rscratch,e,t,r),r):Ke(n.rscratch,e,t)};if(!i)return"";"none"==a||("uppercase"==a?i=i.toUpperCase():"lowercase"==a&&(i=i.toLowerCase()));var s=e.pstyle("text-wrap").value;if("wrap"===s){var l=o("labelKey");if(null!=l&&o("labelWrapKey")===l)return o("labelWrapCachedText");for(var u=i.split("\n"),c=e.pstyle("text-max-width").pfValue,d="anywhere"===e.pstyle("text-overflow-wrap").value,h=[],p=/[\s\u200b]+/,f=d?"":" ",g=0;gc){for(var b=v.split(p),x="",w=0;wC)break;S+=i[D],D===i.length-1&&(P=!0)}return P||(S+="…"),S}return i},Bl.getLabelJustification=function(e){var t=e.pstyle("text-justification").strValue,n=e.pstyle("text-halign").strValue;if("auto"!==t)return t;if(!e.isNode())return"center";switch(n){case"left":return"right";case"right":return"left";default:return"center"}},Bl.calculateLabelDimensions=function(e,t){var n=Pe(t,e._private.labelDimsKey),r=this.labelDimCache||(this.labelDimCache=[]),i=r[n];if(null!=i)return i;var a=e.pstyle("font-style").strValue,o=e.pstyle("font-size").pfValue,s=e.pstyle("font-family").strValue,l=e.pstyle("font-weight").strValue,u=this.labelCalcCanvas,c=this.labelCalcCanvasContext;if(!u){u=this.labelCalcCanvas=document.createElement("canvas"),c=this.labelCalcCanvasContext=u.getContext("2d");var d=u.style;d.position="absolute",d.left="-9999px",d.top="-9999px",d.zIndex="-1",d.visibility="hidden",d.pointerEvents="none"}c.font="".concat(a," ").concat(l," ").concat(o,"px ").concat(s);for(var h=0,p=0,f=t.split("\n"),g=0;g1&&void 0!==arguments[1])||arguments[1];if(t.merge(e),n)for(var r=0;r=e.desktopTapThreshold2}var D=r(t);v&&(e.hoverData.tapholdCancelled=!0);a=!0,n(g,["mousemove","vmousemove","tapdrag"],t,{x:d[0],y:d[1]});var T=function(){e.data.bgActivePosistion=void 0,e.hoverData.selecting||o.emit({originalEvent:t,type:"boxstart",position:{x:d[0],y:d[1]}}),f[4]=1,e.hoverData.selecting=!0,e.redrawHint("select",!0),e.redraw()};if(3===e.hoverData.which){if(v){var _={originalEvent:t,type:"cxtdrag",position:{x:d[0],y:d[1]}};b?b.emit(_):o.emit(_),e.hoverData.cxtDragged=!0,e.hoverData.cxtOver&&g===e.hoverData.cxtOver||(e.hoverData.cxtOver&&e.hoverData.cxtOver.emit({originalEvent:t,type:"cxtdragout",position:{x:d[0],y:d[1]}}),e.hoverData.cxtOver=g,g&&g.emit({originalEvent:t,type:"cxtdragover",position:{x:d[0],y:d[1]}}))}}else if(e.hoverData.dragging){if(a=!0,o.panningEnabled()&&o.userPanningEnabled()){var M;if(e.hoverData.justStartedPan){var B=e.hoverData.mdownPos;M={x:(d[0]-B[0])*s,y:(d[1]-B[1])*s},e.hoverData.justStartedPan=!1}else M={x:x[0]*s,y:x[1]*s};o.panBy(M),o.emit("dragpan"),e.hoverData.dragged=!0}d=e.projectIntoViewport(t.clientX,t.clientY)}else if(1!=f[4]||null!=b&&!b.pannable()){if(b&&b.pannable()&&b.active()&&b.unactivate(),b&&b.grabbed()||g==y||(y&&n(y,["mouseout","tapdragout"],t,{x:d[0],y:d[1]}),g&&n(g,["mouseover","tapdragover"],t,{x:d[0],y:d[1]}),e.hoverData.last=g),b)if(v){if(o.boxSelectionEnabled()&&D)b&&b.grabbed()&&(c(E),b.emit("freeon"),E.emit("free"),e.dragData.didDrag&&(b.emit("dragfreeon"),E.emit("dragfree"))),T();else if(b&&b.grabbed()&&e.nodeIsDraggable(b)){var N=!e.dragData.didDrag;N&&e.redrawHint("eles",!0),e.dragData.didDrag=!0,e.hoverData.draggingEles||l(E,{inDragLayer:!0});var z={x:0,y:0};if(m(x[0])&&m(x[1])&&(z.x+=x[0],z.y+=x[1],N)){var I=e.hoverData.dragDelta;I&&m(I[0])&&m(I[1])&&(z.x+=I[0],z.y+=I[1])}e.hoverData.draggingEles=!0,E.silentShift(z).emit("position drag"),e.redrawHint("drag",!0),e.redraw()}}else!function(){var t=e.hoverData.dragDelta=e.hoverData.dragDelta||[];0===t.length?(t.push(x[0]),t.push(x[1])):(t[0]+=x[0],t[1]+=x[1])}();a=!0}else if(v){if(e.hoverData.dragging||!o.boxSelectionEnabled()||!D&&o.panningEnabled()&&o.userPanningEnabled()){if(!e.hoverData.selecting&&o.panningEnabled()&&o.userPanningEnabled()){i(b,e.hoverData.downs)&&(e.hoverData.dragging=!0,e.hoverData.justStartedPan=!0,f[4]=0,e.data.bgActivePosistion=yt(h),e.redrawHint("select",!0),e.redraw())}}else T();b&&b.pannable()&&b.active()&&b.unactivate()}return f[2]=d[0],f[3]=d[1],a?(t.stopPropagation&&t.stopPropagation(),t.preventDefault&&t.preventDefault(),!1):void 0}}),!1),e.registerBinding(window,"mouseup",(function(i){if(e.hoverData.capture){e.hoverData.capture=!1;var a=e.cy,o=e.projectIntoViewport(i.clientX,i.clientY),s=e.selection,l=e.findNearestElement(o[0],o[1],!0,!1),u=e.dragData.possibleDragElements,d=e.hoverData.down,h=r(i);if(e.data.bgActivePosistion&&(e.redrawHint("select",!0),e.redraw()),e.hoverData.tapholdCancelled=!0,e.data.bgActivePosistion=void 0,d&&d.unactivate(),3===e.hoverData.which){var p={originalEvent:i,type:"cxttapend",position:{x:o[0],y:o[1]}};if(d?d.emit(p):a.emit(p),!e.hoverData.cxtDragged){var f={originalEvent:i,type:"cxttap",position:{x:o[0],y:o[1]}};d?d.emit(f):a.emit(f)}e.hoverData.cxtDragged=!1,e.hoverData.which=null}else if(1===e.hoverData.which){if(n(l,["mouseup","tapend","vmouseup"],i,{x:o[0],y:o[1]}),e.dragData.didDrag||e.hoverData.dragged||e.hoverData.selecting||e.hoverData.isOverThresholdDrag||(n(d,["click","tap","vclick"],i,{x:o[0],y:o[1]}),b=!1,i.timeStamp-x<=a.multiClickDebounceTime()?(y&&clearTimeout(y),b=!0,x=null,n(d,["dblclick","dbltap","vdblclick"],i,{x:o[0],y:o[1]})):(y=setTimeout((function(){b||n(d,["oneclick","onetap","voneclick"],i,{x:o[0],y:o[1]})}),a.multiClickDebounceTime()),x=i.timeStamp)),null!=d||e.dragData.didDrag||e.hoverData.selecting||e.hoverData.dragged||r(i)||(a.$(t).unselect(["tapunselect"]),u.length>0&&e.redrawHint("eles",!0),e.dragData.possibleDragElements=u=a.collection()),l!=d||e.dragData.didDrag||e.hoverData.selecting||null!=l&&l._private.selectable&&(e.hoverData.dragging||("additive"===a.selectionType()||h?l.selected()?l.unselect(["tapunselect"]):l.select(["tapselect"]):h||(a.$(t).unmerge(l).unselect(["tapunselect"]),l.select(["tapselect"]))),e.redrawHint("eles",!0)),e.hoverData.selecting){var g=a.collection(e.getAllInBox(s[0],s[1],s[2],s[3]));e.redrawHint("select",!0),g.length>0&&e.redrawHint("eles",!0),a.emit({type:"boxend",originalEvent:i,position:{x:o[0],y:o[1]}});var v=function(e){return e.selectable()&&!e.selected()};"additive"===a.selectionType()||h||a.$(t).unmerge(g).unselect(),g.emit("box").stdFilter(v).select().emit("boxselect"),e.redraw()}if(e.hoverData.dragging&&(e.hoverData.dragging=!1,e.redrawHint("select",!0),e.redrawHint("eles",!0),e.redraw()),!s[4]){e.redrawHint("drag",!0),e.redrawHint("eles",!0);var m=d&&d.grabbed();c(u),m&&(d.emit("freeon"),u.emit("free"),e.dragData.didDrag&&(d.emit("dragfreeon"),u.emit("dragfree")))}}s[4]=0,e.hoverData.down=null,e.hoverData.cxtStarted=!1,e.hoverData.draggingEles=!1,e.hoverData.selecting=!1,e.hoverData.isOverThresholdDrag=!1,e.dragData.didDrag=!1,e.hoverData.dragged=!1,e.hoverData.dragDelta=[],e.hoverData.mdownPos=null,e.hoverData.mdownGPos=null}}),!1);var E,k,C,S,P,D,T,_,M,B,N,z,I,A=function(t){if(!e.scrollingPage){var n=e.cy,r=n.zoom(),i=n.pan(),a=e.projectIntoViewport(t.clientX,t.clientY),o=[a[0]*r+i.x,a[1]*r+i.y];if(e.hoverData.draggingEles||e.hoverData.dragging||e.hoverData.cxtStarted||0!==e.selection[4])t.preventDefault();else if(n.panningEnabled()&&n.userPanningEnabled()&&n.zoomingEnabled()&&n.userZoomingEnabled()){var s;t.preventDefault(),e.data.wheelZooming=!0,clearTimeout(e.data.wheelTimeout),e.data.wheelTimeout=setTimeout((function(){e.data.wheelZooming=!1,e.redrawHint("eles",!0),e.redraw()}),150),s=null!=t.deltaY?t.deltaY/-250:null!=t.wheelDeltaY?t.wheelDeltaY/1e3:t.wheelDelta/1e3,s*=e.wheelSensitivity,1===t.deltaMode&&(s*=33);var l=n.zoom()*Math.pow(10,s);"gesturechange"===t.type&&(l=e.gestureStartZoom*t.scale),n.zoom({level:l,renderedPosition:{x:o[0],y:o[1]}}),n.emit("gesturechange"===t.type?"pinchzoom":"scrollzoom")}}};e.registerBinding(e.container,"wheel",A,!0),e.registerBinding(window,"scroll",(function(t){e.scrollingPage=!0,clearTimeout(e.scrollingPageTimeout),e.scrollingPageTimeout=setTimeout((function(){e.scrollingPage=!1}),250)}),!0),e.registerBinding(e.container,"gesturestart",(function(t){e.gestureStartZoom=e.cy.zoom(),e.hasTouchStarted||t.preventDefault()}),!0),e.registerBinding(e.container,"gesturechange",(function(t){e.hasTouchStarted||A(t)}),!0),e.registerBinding(e.container,"mouseout",(function(t){var n=e.projectIntoViewport(t.clientX,t.clientY);e.cy.emit({originalEvent:t,type:"mouseout",position:{x:n[0],y:n[1]}})}),!1),e.registerBinding(e.container,"mouseover",(function(t){var n=e.projectIntoViewport(t.clientX,t.clientY);e.cy.emit({originalEvent:t,type:"mouseover",position:{x:n[0],y:n[1]}})}),!1);var L,O,R,V,F,j,q,Y=function(e,t,n,r){return Math.sqrt((n-e)*(n-e)+(r-t)*(r-t))},X=function(e,t,n,r){return(n-e)*(n-e)+(r-t)*(r-t)};if(e.registerBinding(e.container,"touchstart",L=function(t){if(e.hasTouchStarted=!0,w(t)){h(),e.touchData.capture=!0,e.data.bgActivePosistion=void 0;var r=e.cy,i=e.touchData.now,a=e.touchData.earlier;if(t.touches[0]){var s=e.projectIntoViewport(t.touches[0].clientX,t.touches[0].clientY);i[0]=s[0],i[1]=s[1]}if(t.touches[1]){s=e.projectIntoViewport(t.touches[1].clientX,t.touches[1].clientY);i[2]=s[0],i[3]=s[1]}if(t.touches[2]){s=e.projectIntoViewport(t.touches[2].clientX,t.touches[2].clientY);i[4]=s[0],i[5]=s[1]}if(t.touches[1]){e.touchData.singleTouchMoved=!0,c(e.dragData.touchDragEles);var d=e.findContainerClientCoords();M=d[0],B=d[1],N=d[2],z=d[3],E=t.touches[0].clientX-M,k=t.touches[0].clientY-B,C=t.touches[1].clientX-M,S=t.touches[1].clientY-B,I=0<=E&&E<=N&&0<=C&&C<=N&&0<=k&&k<=z&&0<=S&&S<=z;var p=r.pan(),f=r.zoom();P=Y(E,k,C,S),D=X(E,k,C,S),_=[((T=[(E+C)/2,(k+S)/2])[0]-p.x)/f,(T[1]-p.y)/f];if(D<4e4&&!t.touches[2]){var g=e.findNearestElement(i[0],i[1],!0,!0),v=e.findNearestElement(i[2],i[3],!0,!0);return g&&g.isNode()?(g.activate().emit({originalEvent:t,type:"cxttapstart",position:{x:i[0],y:i[1]}}),e.touchData.start=g):v&&v.isNode()?(v.activate().emit({originalEvent:t,type:"cxttapstart",position:{x:i[0],y:i[1]}}),e.touchData.start=v):r.emit({originalEvent:t,type:"cxttapstart",position:{x:i[0],y:i[1]}}),e.touchData.start&&(e.touchData.start._private.grabbed=!1),e.touchData.cxt=!0,e.touchData.cxtDragged=!1,e.data.bgActivePosistion=void 0,void e.redraw()}}if(t.touches[2])r.boxSelectionEnabled()&&t.preventDefault();else if(t.touches[1]);else if(t.touches[0]){var y=e.findNearestElements(i[0],i[1],!0,!0),m=y[0];if(null!=m&&(m.activate(),e.touchData.start=m,e.touchData.starts=y,e.nodeIsGrabbable(m))){var b=e.dragData.touchDragEles=r.collection(),x=null;e.redrawHint("eles",!0),e.redrawHint("drag",!0),m.selected()?(x=r.$((function(t){return t.selected()&&e.nodeIsGrabbable(t)})),l(x,{addToList:b})):u(m,{addToList:b}),o(m);var A=function(e){return{originalEvent:t,type:e,position:{x:i[0],y:i[1]}}};m.emit(A("grabon")),x?x.forEach((function(e){e.emit(A("grab"))})):m.emit(A("grab"))}n(m,["touchstart","tapstart","vmousedown"],t,{x:i[0],y:i[1]}),null==m&&(e.data.bgActivePosistion={x:s[0],y:s[1]},e.redrawHint("select",!0),e.redraw()),e.touchData.singleTouchMoved=!1,e.touchData.singleTouchStartTime=+new Date,clearTimeout(e.touchData.tapholdTimeout),e.touchData.tapholdTimeout=setTimeout((function(){!1!==e.touchData.singleTouchMoved||e.pinching||e.touchData.selecting||n(e.touchData.start,["taphold"],t,{x:i[0],y:i[1]})}),e.tapholdDuration)}if(t.touches.length>=1){for(var L=e.touchData.startPosition=[],O=0;O=e.touchTapThreshold2}if(r&&e.touchData.cxt){t.preventDefault();var T=t.touches[0].clientX-M,N=t.touches[0].clientY-B,z=t.touches[1].clientX-M,A=t.touches[1].clientY-B,L=X(T,N,z,A);if(L/D>=2.25||L>=22500){e.touchData.cxt=!1,e.data.bgActivePosistion=void 0,e.redrawHint("select",!0);var O={originalEvent:t,type:"cxttapend",position:{x:s[0],y:s[1]}};e.touchData.start?(e.touchData.start.unactivate().emit(O),e.touchData.start=null):o.emit(O)}}if(r&&e.touchData.cxt){O={originalEvent:t,type:"cxtdrag",position:{x:s[0],y:s[1]}};e.data.bgActivePosistion=void 0,e.redrawHint("select",!0),e.touchData.start?e.touchData.start.emit(O):o.emit(O),e.touchData.start&&(e.touchData.start._private.grabbed=!1),e.touchData.cxtDragged=!0;var R=e.findNearestElement(s[0],s[1],!0,!0);e.touchData.cxtOver&&R===e.touchData.cxtOver||(e.touchData.cxtOver&&e.touchData.cxtOver.emit({originalEvent:t,type:"cxtdragout",position:{x:s[0],y:s[1]}}),e.touchData.cxtOver=R,R&&R.emit({originalEvent:t,type:"cxtdragover",position:{x:s[0],y:s[1]}}))}else if(r&&t.touches[2]&&o.boxSelectionEnabled())t.preventDefault(),e.data.bgActivePosistion=void 0,this.lastThreeTouch=+new Date,e.touchData.selecting||o.emit({originalEvent:t,type:"boxstart",position:{x:s[0],y:s[1]}}),e.touchData.selecting=!0,e.touchData.didSelect=!0,a[4]=1,a&&0!==a.length&&void 0!==a[0]?(a[2]=(s[0]+s[2]+s[4])/3,a[3]=(s[1]+s[3]+s[5])/3):(a[0]=(s[0]+s[2]+s[4])/3,a[1]=(s[1]+s[3]+s[5])/3,a[2]=(s[0]+s[2]+s[4])/3+1,a[3]=(s[1]+s[3]+s[5])/3+1),e.redrawHint("select",!0),e.redraw();else if(r&&t.touches[1]&&!e.touchData.didSelect&&o.zoomingEnabled()&&o.panningEnabled()&&o.userZoomingEnabled()&&o.userPanningEnabled()){if(t.preventDefault(),e.data.bgActivePosistion=void 0,e.redrawHint("select",!0),ee=e.dragData.touchDragEles){e.redrawHint("drag",!0);for(var V=0;V0&&!e.hoverData.draggingEles&&!e.swipePanning&&null!=e.data.bgActivePosistion&&(e.data.bgActivePosistion=void 0,e.redrawHint("select",!0),e.redraw())}},!1),e.registerBinding(window,"touchcancel",R=function(t){var n=e.touchData.start;e.touchData.capture=!1,n&&n.unactivate()}),e.registerBinding(window,"touchend",V=function(r){var i=e.touchData.start;if(e.touchData.capture){0===r.touches.length&&(e.touchData.capture=!1),r.preventDefault();var a=e.selection;e.swipePanning=!1,e.hoverData.draggingEles=!1;var o,s=e.cy,l=s.zoom(),u=e.touchData.now,d=e.touchData.earlier;if(r.touches[0]){var h=e.projectIntoViewport(r.touches[0].clientX,r.touches[0].clientY);u[0]=h[0],u[1]=h[1]}if(r.touches[1]){h=e.projectIntoViewport(r.touches[1].clientX,r.touches[1].clientY);u[2]=h[0],u[3]=h[1]}if(r.touches[2]){h=e.projectIntoViewport(r.touches[2].clientX,r.touches[2].clientY);u[4]=h[0],u[5]=h[1]}if(i&&i.unactivate(),e.touchData.cxt){if(o={originalEvent:r,type:"cxttapend",position:{x:u[0],y:u[1]}},i?i.emit(o):s.emit(o),!e.touchData.cxtDragged){var p={originalEvent:r,type:"cxttap",position:{x:u[0],y:u[1]}};i?i.emit(p):s.emit(p)}return e.touchData.start&&(e.touchData.start._private.grabbed=!1),e.touchData.cxt=!1,e.touchData.start=null,void e.redraw()}if(!r.touches[2]&&s.boxSelectionEnabled()&&e.touchData.selecting){e.touchData.selecting=!1;var f=s.collection(e.getAllInBox(a[0],a[1],a[2],a[3]));a[0]=void 0,a[1]=void 0,a[2]=void 0,a[3]=void 0,a[4]=0,e.redrawHint("select",!0),s.emit({type:"boxend",originalEvent:r,position:{x:u[0],y:u[1]}});f.emit("box").stdFilter((function(e){return e.selectable()&&!e.selected()})).select().emit("boxselect"),f.nonempty()&&e.redrawHint("eles",!0),e.redraw()}if(null!=i&&i.unactivate(),r.touches[2])e.data.bgActivePosistion=void 0,e.redrawHint("select",!0);else if(r.touches[1]);else if(r.touches[0]);else if(!r.touches[0]){e.data.bgActivePosistion=void 0,e.redrawHint("select",!0);var g=e.dragData.touchDragEles;if(null!=i){var v=i._private.grabbed;c(g),e.redrawHint("drag",!0),e.redrawHint("eles",!0),v&&(i.emit("freeon"),g.emit("free"),e.dragData.didDrag&&(i.emit("dragfreeon"),g.emit("dragfree"))),n(i,["touchend","tapend","vmouseup","tapdragout"],r,{x:u[0],y:u[1]}),i.unactivate(),e.touchData.start=null}else{var y=e.findNearestElement(u[0],u[1],!0,!0);n(y,["touchend","tapend","vmouseup","tapdragout"],r,{x:u[0],y:u[1]})}var m=e.touchData.startPosition[0]-u[0],b=m*m,x=e.touchData.startPosition[1]-u[1],w=(b+x*x)*l*l;e.touchData.singleTouchMoved||(i||s.$(":selected").unselect(["tapunselect"]),n(i,["tap","vclick"],r,{x:u[0],y:u[1]}),F=!1,r.timeStamp-q<=s.multiClickDebounceTime()?(j&&clearTimeout(j),F=!0,q=null,n(i,["dbltap","vdblclick"],r,{x:u[0],y:u[1]})):(j=setTimeout((function(){F||n(i,["onetap","voneclick"],r,{x:u[0],y:u[1]})}),s.multiClickDebounceTime()),q=r.timeStamp)),null!=i&&!e.dragData.didDrag&&i._private.selectable&&w2){for(var T=[u[0],u[1]],_=Math.pow(T[0]-e,2)+Math.pow(T[1]-t,2),M=1;M0)return g[0]}return null},h=Object.keys(c),p=0;p0?l:At(i,a,e,t,n,r,o)},checkPoint:function(e,t,n,r,i,a,o){var s=Jt(r,i),l=2*s;if(jt(e,t,this.points,a,o,r,i-l,[0,-1],n))return!0;if(jt(e,t,this.points,a,o,r-l,i,[0,-1],n))return!0;var u=r/2+2*n,c=i/2+2*n;return!!Ft(e,t,[a-u,o-c,a-u,o,a+u,o,a+u,o-c])||(!!Xt(e,t,l,l,a+r/2-s,o+i/2-s,n)||!!Xt(e,t,l,l,a-r/2+s,o+i/2-s,n))}}},jl.registerNodeShapes=function(){var e=this.nodeShapes={},t=this;this.generateEllipse(),this.generatePolygon("triangle",Zt(3,0)),this.generateRoundPolygon("round-triangle",Zt(3,0)),this.generatePolygon("rectangle",Zt(4,0)),e.square=e.rectangle,this.generateRoundRectangle(),this.generateCutRectangle(),this.generateBarrel(),this.generateBottomRoundrectangle();var n=[0,1,1,0,0,-1,-1,0];this.generatePolygon("diamond",n),this.generateRoundPolygon("round-diamond",n),this.generatePolygon("pentagon",Zt(5,0)),this.generateRoundPolygon("round-pentagon",Zt(5,0)),this.generatePolygon("hexagon",Zt(6,0)),this.generateRoundPolygon("round-hexagon",Zt(6,0)),this.generatePolygon("heptagon",Zt(7,0)),this.generateRoundPolygon("round-heptagon",Zt(7,0)),this.generatePolygon("octagon",Zt(8,0)),this.generateRoundPolygon("round-octagon",Zt(8,0));var r=new Array(20),i=Qt(5,0),a=Qt(5,Math.PI/5),o=.5*(3-Math.sqrt(5));o*=1.57;for(var s=0;s=e.deqFastCost*g)break}else if(i){if(p>=e.deqCost*l||p>=e.deqAvgCost*s)break}else if(f>=e.deqNoDrawCost*(1e3/60))break;var v=e.deq(t,d,c);if(!(v.length>0))break;for(var y=0;y0&&(e.onDeqd(t,u),!i&&e.shouldRedraw(t,u,d,c)&&r())}),i(t))}}},Kl=function(){function e(n){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:Ie;t(this,e),this.idsByKey=new Ue,this.keyForId=new Ue,this.cachesByLvl=new Ue,this.lvls=[],this.getKey=n,this.doesEleInvalidateKey=r}return r(e,[{key:"getIdsFor",value:function(e){null==e&&Oe("Can not get id list for null key");var t=this.idsByKey,n=this.idsByKey.get(e);return n||(n=new $e,t.set(e,n)),n}},{key:"addIdForKey",value:function(e,t){null!=e&&this.getIdsFor(e).add(t)}},{key:"deleteIdForKey",value:function(e,t){null!=e&&this.getIdsFor(e).delete(t)}},{key:"getNumberOfIdsForKey",value:function(e){return null==e?0:this.getIdsFor(e).size}},{key:"updateKeyMappingFor",value:function(e){var t=e.id(),n=this.keyForId.get(t),r=this.getKey(e);this.deleteIdForKey(n,t),this.addIdForKey(r,t),this.keyForId.set(t,r)}},{key:"deleteKeyMappingFor",value:function(e){var t=e.id(),n=this.keyForId.get(t);this.deleteIdForKey(n,t),this.keyForId.delete(t)}},{key:"keyHasChangedFor",value:function(e){var t=e.id();return this.keyForId.get(t)!==this.getKey(e)}},{key:"isInvalid",value:function(e){return this.keyHasChangedFor(e)||this.doesEleInvalidateKey(e)}},{key:"getCachesAt",value:function(e){var t=this.cachesByLvl,n=this.lvls,r=t.get(e);return r||(r=new Ue,t.set(e,r),n.push(e)),r}},{key:"getCache",value:function(e,t){return this.getCachesAt(t).get(e)}},{key:"get",value:function(e,t){var n=this.getKey(e),r=this.getCache(n,t);return null!=r&&this.updateKeyMappingFor(e),r}},{key:"getForCachedKey",value:function(e,t){var n=this.keyForId.get(e.id());return this.getCache(n,t)}},{key:"hasCache",value:function(e,t){return this.getCachesAt(t).has(e)}},{key:"has",value:function(e,t){var n=this.getKey(e);return this.hasCache(n,t)}},{key:"setCache",value:function(e,t,n){n.key=e,this.getCachesAt(t).set(e,n)}},{key:"set",value:function(e,t,n){var r=this.getKey(e);this.setCache(r,t,n),this.updateKeyMappingFor(e)}},{key:"deleteCache",value:function(e,t){this.getCachesAt(t).delete(e)}},{key:"delete",value:function(e,t){var n=this.getKey(e);this.deleteCache(n,t)}},{key:"invalidateKey",value:function(e){var t=this;this.lvls.forEach((function(n){return t.deleteCache(e,n)}))}},{key:"invalidate",value:function(e){var t=e.id(),n=this.keyForId.get(t);this.deleteKeyMappingFor(e);var r=this.doesEleInvalidateKey(e);return r&&this.invalidateKey(n),r||0===this.getNumberOfIdsForKey(n)}}]),e}(),Gl={dequeue:"dequeue",downscale:"downscale",highQuality:"highQuality"},Ul=Xe({getKey:null,doesEleInvalidateKey:Ie,drawElement:null,getBoundingBox:null,getRotationPoint:null,getRotationOffset:null,isVisible:ze,allowEdgeTxrCaching:!0,allowParentTxrCaching:!0}),Zl=function(e,t){this.renderer=e,this.onDequeues=[];var n=Ul(t);I(this,n),this.lookup=new Kl(n.getKey,n.doesEleInvalidateKey),this.setupDequeueing()},$l=Zl.prototype;$l.reasons=Gl,$l.getTextureQueue=function(e){return this.eleImgCaches=this.eleImgCaches||{},this.eleImgCaches[e]=this.eleImgCaches[e]||[]},$l.getRetiredTextureQueue=function(e){var t=this.eleImgCaches.retired=this.eleImgCaches.retired||{};return t[e]=t[e]||[]},$l.getElementQueue=function(){return this.eleCacheQueue=this.eleCacheQueue||new tt((function(e,t){return t.reqs-e.reqs}))},$l.getElementKeyToQueue=function(){return this.eleKeyToCacheQueue=this.eleKeyToCacheQueue||{}},$l.getElement=function(e,t,n,r,i){var a=this,o=this.renderer,s=o.cy.zoom(),l=this.lookup;if(!t||0===t.w||0===t.h||isNaN(t.w)||isNaN(t.h)||!e.visible()||e.removed())return null;if(!a.allowEdgeTxrCaching&&e.isEdge()||!a.allowParentTxrCaching&&e.isParent())return null;if(null==r&&(r=Math.ceil(bt(s*n))),r<-4)r=-4;else if(s>=7.99||r>3)return null;var u=Math.pow(2,r),c=t.h*u,d=t.w*u,h=o.eleTextBiggerThanMin(e,u);if(!this.isVisible(e,h))return null;var p,f=l.get(e,r);if(f&&f.invalidated&&(f.invalidated=!1,f.texture.invalidatedWidth-=f.width),f)return f;if(p=c<=25?25:c<=50?50:50*Math.ceil(c/50),c>1024||d>1024)return null;var g=a.getTextureQueue(p),v=g[g.length-2],y=function(){return a.recycleTexture(p,d)||a.addTexture(p,d)};v||(v=g[g.length-1]),v||(v=y()),v.width-v.usedWidthr;D--)S=a.getElement(e,t,n,D,Gl.downscale);P()}else{var T;if(!x&&!w&&!E)for(var _=r-1;_>=-4;_--){var M=l.get(e,_);if(M){T=M;break}}if(b(T))return a.queueElement(e,r),T;v.context.translate(v.usedWidth,0),v.context.scale(u,u),this.drawElement(v.context,e,t,h,!1),v.context.scale(1/u,1/u),v.context.translate(-v.usedWidth,0)}return f={x:v.usedWidth,texture:v,level:r,scale:u,width:d,height:c,scaledLabelShown:h},v.usedWidth+=Math.ceil(d+8),v.eleCaches.push(f),l.set(e,r,f),a.checkTextureFullness(v),f},$l.invalidateElements=function(e){for(var t=0;t=.2*e.width&&this.retireTexture(e)},$l.checkTextureFullness=function(e){var t=this.getTextureQueue(e.height);e.usedWidth/e.width>.8&&e.fullnessChecks>=10?We(t,e):e.fullnessChecks++},$l.retireTexture=function(e){var t=e.height,n=this.getTextureQueue(t),r=this.lookup;We(n,e),e.retired=!0;for(var i=e.eleCaches,a=0;a=t)return a.retired=!1,a.usedWidth=0,a.invalidatedWidth=0,a.fullnessChecks=0,He(a.eleCaches),a.context.setTransform(1,0,0,1,0,0),a.context.clearRect(0,0,a.width,a.height),We(r,a),n.push(a),a}},$l.queueElement=function(e,t){var n=this.getElementQueue(),r=this.getElementKeyToQueue(),i=this.getKey(e),a=r[i];if(a)a.level=Math.max(a.level,t),a.eles.merge(e),a.reqs++,n.updateItem(a);else{var o={eles:e.spawn().merge(e),level:t,reqs:1,key:i};n.push(o),r[i]=o}},$l.dequeue=function(e){for(var t=this.getElementQueue(),n=this.getElementKeyToQueue(),r=[],i=this.lookup,a=0;a<1&&t.size()>0;a++){var o=t.pop(),s=o.key,l=o.eles[0],u=i.hasCache(l,o.level);if(n[s]=null,!u){r.push(o);var c=this.getBoundingBox(l);this.getElement(l,c,e,o.level,Gl.dequeue)}}return r},$l.removeFromQueue=function(e){var t=this.getElementQueue(),n=this.getElementKeyToQueue(),r=this.getKey(e),i=n[r];null!=i&&(1===i.eles.length?(i.reqs=Ne,t.updateItem(i),t.pop(),n[r]=null):i.eles.unmerge(e))},$l.onDequeue=function(e){this.onDequeues.push(e)},$l.offDequeue=function(e){We(this.onDequeues,e)},$l.setupDequeueing=Hl({deqRedrawThreshold:100,deqCost:.15,deqAvgCost:.1,deqNoDrawCost:.9,deqFastCost:.9,deq:function(e,t,n){return e.dequeue(t,n)},onDeqd:function(e,t){for(var n=0;n=3.99||n>2)return null;r.validateLayersElesOrdering(n,e);var o,s,l=r.layersByLevel,u=Math.pow(2,n),c=l[n]=l[n]||[];if(r.levelIsComplete(n,e))return c;!function(){var t=function(t){if(r.validateLayersElesOrdering(t,e),r.levelIsComplete(t,e))return s=l[t],!0},i=function(e){if(!s)for(var r=n+e;-4<=r&&r<=2&&!t(r);r+=e);};i(1),i(-1);for(var a=c.length-1;a>=0;a--){var o=c[a];o.invalid&&We(c,o)}}();var d=function(t){var i=(t=t||{}).after;if(function(){if(!o){o=Dt();for(var t=0;t16e6)return null;var a=r.makeLayer(o,n);if(null!=i){var s=c.indexOf(i)+1;c.splice(s,0,a)}else(void 0===t.insert||t.insert)&&c.unshift(a);return a};if(r.skipping&&!a)return null;for(var h=null,p=e.length/1,f=!a,g=0;g=p||!It(h.bb,v.boundingBox()))&&!(h=d({insert:!0,after:h})))return null;s||f?r.queueLayer(h,v):r.drawEleInLayer(h,v,n,t),h.eles.push(v),m[n]=h}}return s||(f?null:c)},Jl.getEleLevelForLayerLevel=function(e,t){return e},Jl.drawEleInLayer=function(e,t,n,r){var i=this.renderer,a=e.context,o=t.boundingBox();0!==o.w&&0!==o.h&&t.visible()&&(n=this.getEleLevelForLayerLevel(n,r),i.setImgSmoothing(a,!1),i.drawCachedElement(a,t,null,null,n,!0),i.setImgSmoothing(a,!0))},Jl.levelIsComplete=function(e,t){var n=this.layersByLevel[e];if(!n||0===n.length)return!1;for(var r=0,i=0;i0)return!1;if(a.invalid)return!1;r+=a.eles.length}return r===t.length},Jl.validateLayersElesOrdering=function(e,t){var n=this.layersByLevel[e];if(n)for(var r=0;r0){e=!0;break}}return e},Jl.invalidateElements=function(e){var t=this;0!==e.length&&(t.lastInvalidationTime=be(),0!==e.length&&t.haveLayers()&&t.updateElementsInLayers(e,(function(e,n,r){t.invalidateLayer(e)})))},Jl.invalidateLayer=function(e){if(this.lastInvalidationTime=be(),!e.invalid){var t=e.level,n=e.eles,r=this.layersByLevel[t];We(r,e),e.elesQueue=[],e.invalid=!0,e.replacement&&(e.replacement.invalid=!0);for(var i=0;i3&&void 0!==arguments[3])||arguments[3],i=!(arguments.length>4&&void 0!==arguments[4])||arguments[4],a=!(arguments.length>5&&void 0!==arguments[5])||arguments[5],o=this,s=t._private.rscratch;if((!a||t.visible())&&!s.badLine&&null!=s.allpts&&!isNaN(s.allpts[0])){var l;n&&(l=n,e.translate(-l.x1,-l.y1));var u=a?t.pstyle("opacity").value:1,c=a?t.pstyle("line-opacity").value:1,d=t.pstyle("curve-style").value,h=t.pstyle("line-style").value,p=t.pstyle("width").pfValue,f=t.pstyle("line-cap").value,g=u*c,v=u*c,y=function(){var n=arguments.length>0&&void 0!==arguments[0]?arguments[0]:g;"straight-triangle"===d?(o.eleStrokeStyle(e,t,n),o.drawEdgeTrianglePath(t,e,s.allpts)):(e.lineWidth=p,e.lineCap=f,o.eleStrokeStyle(e,t,n),o.drawEdgePath(t,e,s.allpts,h),e.lineCap="butt")},m=function(){i&&o.drawEdgeOverlay(e,t)},b=function(){i&&o.drawEdgeUnderlay(e,t)},x=function(){var n=arguments.length>0&&void 0!==arguments[0]?arguments[0]:v;o.drawArrowheads(e,t,n)},w=function(){o.drawElementText(e,t,null,r)};e.lineJoin="round";var E="yes"===t.pstyle("ghost").value;if(E){var k=t.pstyle("ghost-offset-x").pfValue,C=t.pstyle("ghost-offset-y").pfValue,S=t.pstyle("ghost-opacity").value,P=g*S;e.translate(k,C),y(P),x(P),e.translate(-k,-C)}b(),y(),x(),m(),w(),n&&e.translate(l.x1,l.y1)}}},yu=function(e){if(!["overlay","underlay"].includes(e))throw new Error("Invalid state");return function(t,n){if(n.visible()){var r=n.pstyle("".concat(e,"-opacity")).value;if(0!==r){var i=this,a=i.usePaths(),o=n._private.rscratch,s=2*n.pstyle("".concat(e,"-padding")).pfValue,l=n.pstyle("".concat(e,"-color")).value;t.lineWidth=s,"self"!==o.edgeType||a?t.lineCap="round":t.lineCap="butt",i.colorStrokeStyle(t,l[0],l[1],l[2],r),i.drawEdgePath(n,t,o.allpts,"solid")}}}};vu.drawEdgeOverlay=yu("overlay"),vu.drawEdgeUnderlay=yu("underlay"),vu.drawEdgePath=function(e,t,n,r){var i,a=e._private.rscratch,o=t,s=!1,l=this.usePaths(),u=e.pstyle("line-dash-pattern").pfValue,c=e.pstyle("line-dash-offset").pfValue;if(l){var d=n.join("$");a.pathCacheKey&&a.pathCacheKey===d?(i=t=a.pathCache,s=!0):(i=t=new Path2D,a.pathCacheKey=d,a.pathCache=i)}if(o.setLineDash)switch(r){case"dotted":o.setLineDash([1,1]);break;case"dashed":o.setLineDash(u),o.lineDashOffset=c;break;case"solid":o.setLineDash([])}if(!s&&!a.badLine)switch(t.beginPath&&t.beginPath(),t.moveTo(n[0],n[1]),a.edgeType){case"bezier":case"self":case"compound":case"multibezier":for(var h=2;h+35&&void 0!==arguments[5]?arguments[5]:5;e.beginPath(),e.moveTo(t+a,n),e.lineTo(t+r-a,n),e.quadraticCurveTo(t+r,n,t+r,n+a),e.lineTo(t+r,n+i-a),e.quadraticCurveTo(t+r,n+i,t+r-a,n+i),e.lineTo(t+a,n+i),e.quadraticCurveTo(t,n+i,t,n+i-a),e.lineTo(t,n+a),e.quadraticCurveTo(t,n,t+a,n),e.closePath(),e.fill()}bu.eleTextBiggerThanMin=function(e,t){if(!t){var n=e.cy().zoom(),r=this.getPixelRatio(),i=Math.ceil(bt(n*r));t=Math.pow(2,i)}return!(e.pstyle("font-size").pfValue*t5&&void 0!==arguments[5])||arguments[5],o=this;if(null==r){if(a&&!o.eleTextBiggerThanMin(t))return}else if(!1===r)return;if(t.isNode()){var s=t.pstyle("label");if(!s||!s.value)return;var l=o.getLabelJustification(t);e.textAlign=l,e.textBaseline="bottom"}else{var u=t.element()._private.rscratch.badLine,c=t.pstyle("label"),d=t.pstyle("source-label"),h=t.pstyle("target-label");if(u||(!c||!c.value)&&(!d||!d.value)&&(!h||!h.value))return;e.textAlign="center",e.textBaseline="bottom"}var p,f=!n;n&&(p=n,e.translate(-p.x1,-p.y1)),null==i?(o.drawText(e,t,null,f,a),t.isEdge()&&(o.drawText(e,t,"source",f,a),o.drawText(e,t,"target",f,a))):o.drawText(e,t,i,f,a),n&&e.translate(p.x1,p.y1)},bu.getFontCache=function(e){var t;this.fontCaches=this.fontCaches||[];for(var n=0;n2&&void 0!==arguments[2])||arguments[2],r=t.pstyle("font-style").strValue,i=t.pstyle("font-size").pfValue+"px",a=t.pstyle("font-family").strValue,o=t.pstyle("font-weight").strValue,s=n?t.effectiveOpacity()*t.pstyle("text-opacity").value:1,l=t.pstyle("text-outline-opacity").value*s,u=t.pstyle("color").value,c=t.pstyle("text-outline-color").value;e.font=r+" "+o+" "+i+" "+a,e.lineJoin="round",this.colorFillStyle(e,u[0],u[1],u[2],s),this.colorStrokeStyle(e,c[0],c[1],c[2],l)},bu.getTextAngle=function(e,t){var n=e._private.rscratch,r=t?t+"-":"",i=e.pstyle(r+"text-rotation"),a=Ke(n,"labelAngle",t);return"autorotate"===i.strValue?e.isEdge()?a:0:"none"===i.strValue?0:i.pfValue},bu.drawText=function(e,t,n){var r=!(arguments.length>3&&void 0!==arguments[3])||arguments[3],i=!(arguments.length>4&&void 0!==arguments[4])||arguments[4],a=t._private,o=a.rscratch,s=i?t.effectiveOpacity():1;if(!i||0!==s&&0!==t.pstyle("text-opacity").value){"main"===n&&(n=null);var l,u,c=Ke(o,"labelX",n),d=Ke(o,"labelY",n),h=this.getLabelText(t,n);if(null!=h&&""!==h&&!isNaN(c)&&!isNaN(d)){this.setupTextStyle(e,t,i);var p,f=n?n+"-":"",g=Ke(o,"labelWidth",n),v=Ke(o,"labelHeight",n),y=t.pstyle(f+"text-margin-x").pfValue,m=t.pstyle(f+"text-margin-y").pfValue,b=t.isEdge(),x=t.pstyle("text-halign").value,w=t.pstyle("text-valign").value;switch(b&&(x="center",w="center"),c+=y,d+=m,0!==(p=r?this.getTextAngle(t,n):0)&&(l=c,u=d,e.translate(l,u),e.rotate(p),c=0,d=0),w){case"top":break;case"center":d+=v/2;break;case"bottom":d+=v}var E=t.pstyle("text-background-opacity").value,k=t.pstyle("text-border-opacity").value,C=t.pstyle("text-border-width").pfValue,S=t.pstyle("text-background-padding").pfValue;if(E>0||C>0&&k>0){var P=c-S;switch(x){case"left":P-=g;break;case"center":P-=g/2}var D=d-v-S,T=g+2*S,_=v+2*S;if(E>0){var M=e.fillStyle,B=t.pstyle("text-background-color").value;e.fillStyle="rgba("+B[0]+","+B[1]+","+B[2]+","+E*s+")";var N=t.pstyle("text-background-shape").strValue;0===N.indexOf("round")?xu(e,P,D,T,_,2):e.fillRect(P,D,T,_),e.fillStyle=M}if(C>0&&k>0){var z=e.strokeStyle,I=e.lineWidth,A=t.pstyle("text-border-color").value,L=t.pstyle("text-border-style").value;if(e.strokeStyle="rgba("+A[0]+","+A[1]+","+A[2]+","+k*s+")",e.lineWidth=C,e.setLineDash)switch(L){case"dotted":e.setLineDash([1,1]);break;case"dashed":e.setLineDash([4,2]);break;case"double":e.lineWidth=C/4,e.setLineDash([]);break;case"solid":e.setLineDash([])}if(e.strokeRect(P,D,T,_),"double"===L){var O=C/2;e.strokeRect(P+O,D+O,T-2*O,_-2*O)}e.setLineDash&&e.setLineDash([]),e.lineWidth=I,e.strokeStyle=z}}var R=2*t.pstyle("text-outline-width").pfValue;if(R>0&&(e.lineWidth=R),"wrap"===t.pstyle("text-wrap").value){var V=Ke(o,"labelWrapCachedLines",n),F=Ke(o,"labelLineHeight",n),j=g/2,q=this.getLabelJustification(t);switch("auto"===q||("left"===x?"left"===q?c+=-g:"center"===q&&(c+=-j):"center"===x?"left"===q?c+=-j:"right"===q&&(c+=j):"right"===x&&("center"===q?c+=j:"right"===q&&(c+=g))),w){case"top":d-=(V.length-1)*F;break;case"center":case"bottom":d-=(V.length-1)*F}for(var Y=0;Y0&&e.strokeText(V[Y],c,d),e.fillText(V[Y],c,d),d+=F}else R>0&&e.strokeText(h,c,d),e.fillText(h,c,d);0!==p&&(e.rotate(-p),e.translate(-l,-u))}}};var wu={drawNode:function(e,t,n){var r,i,a=!(arguments.length>3&&void 0!==arguments[3])||arguments[3],o=!(arguments.length>4&&void 0!==arguments[4])||arguments[4],s=!(arguments.length>5&&void 0!==arguments[5])||arguments[5],l=this,u=t._private,c=u.rscratch,d=t.position();if(m(d.x)&&m(d.y)&&(!s||t.visible())){var h,p,f=s?t.effectiveOpacity():1,g=l.usePaths(),v=!1,y=t.padding();r=t.width()+2*y,i=t.height()+2*y,n&&(p=n,e.translate(-p.x1,-p.y1));for(var b=t.pstyle("background-image"),x=b.value,w=new Array(x.length),E=new Array(x.length),k=0,C=0;C0&&void 0!==arguments[0]?arguments[0]:M;l.eleFillStyle(e,t,n)},A=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:z;l.colorStrokeStyle(e,B[0],B[1],B[2],t)},L=t.pstyle("shape").strValue,O=t.pstyle("shape-polygon-points").pfValue;if(g){e.translate(d.x,d.y);var R=l.nodePathCache=l.nodePathCache||[],V=De("polygon"===L?L+","+O.join(","):L,""+i,""+r),F=R[V];null!=F?(h=F,v=!0,c.pathCache=h):(h=new Path2D,R[V]=c.pathCache=h)}var j=function(){if(!v){var n=d;g&&(n={x:0,y:0}),l.nodeShapes[l.getNodeShape(t)].draw(h||e,n.x,n.y,r,i)}g?e.fill(h):e.fill()},q=function(){for(var n=arguments.length>0&&void 0!==arguments[0]?arguments[0]:f,r=!(arguments.length>1&&void 0!==arguments[1])||arguments[1],i=u.backgrounding,a=0,o=0;o0&&void 0!==arguments[0]&&arguments[0],a=arguments.length>1&&void 0!==arguments[1]?arguments[1]:f;l.hasPie(t)&&(l.drawPie(e,t,a),n&&(g||l.nodeShapes[l.getNodeShape(t)].draw(e,d.x,d.y,r,i)))},X=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:f,n=(T>0?T:-T)*t,r=T>0?0:255;0!==T&&(l.colorFillStyle(e,r,r,r,n),g?e.fill(h):e.fill())},W=function(){if(_>0){if(e.lineWidth=_,e.lineCap="butt",e.setLineDash)switch(N){case"dotted":e.setLineDash([1,1]);break;case"dashed":e.setLineDash([4,2]);break;case"solid":case"double":e.setLineDash([])}if(g?e.stroke(h):e.stroke(),"double"===N){e.lineWidth=_/3;var t=e.globalCompositeOperation;e.globalCompositeOperation="destination-out",g?e.stroke(h):e.stroke(),e.globalCompositeOperation=t}e.setLineDash&&e.setLineDash([])}},H=function(){o&&l.drawNodeOverlay(e,t,d,r,i)},K=function(){o&&l.drawNodeUnderlay(e,t,d,r,i)},G=function(){l.drawElementText(e,t,null,a)},U="yes"===t.pstyle("ghost").value;if(U){var Z=t.pstyle("ghost-offset-x").pfValue,$=t.pstyle("ghost-offset-y").pfValue,Q=t.pstyle("ghost-opacity").value,J=Q*f;e.translate(Z,$),I(Q*M),j(),q(J,!0),A(Q*z),W(),Y(0!==T||0!==_),q(J,!1),X(J),e.translate(-Z,-$)}g&&e.translate(-d.x,-d.y),K(),g&&e.translate(d.x,d.y),I(),j(),q(f,!0),A(),W(),Y(0!==T||0!==_),q(f,!1),X(),g&&e.translate(-d.x,-d.y),G(),H(),n&&e.translate(p.x1,p.y1)}}},Eu=function(e){if(!["overlay","underlay"].includes(e))throw new Error("Invalid state");return function(t,n,r,i,a){if(n.visible()){var o=n.pstyle("".concat(e,"-padding")).pfValue,s=n.pstyle("".concat(e,"-opacity")).value,l=n.pstyle("".concat(e,"-color")).value,u=n.pstyle("".concat(e,"-shape")).value;if(s>0){if(r=r||n.position(),null==i||null==a){var c=n.padding();i=n.width()+2*c,a=n.height()+2*c}this.colorFillStyle(t,l[0],l[1],l[2],s),this.nodeShapes[u].draw(t,r.x,r.y,i+2*o,a+2*o),t.fill()}}}};wu.drawNodeOverlay=Eu("overlay"),wu.drawNodeUnderlay=Eu("underlay"),wu.hasPie=function(e){return(e=e[0])._private.hasPie},wu.drawPie=function(e,t,n,r){t=t[0],r=r||t.position();var i=t.cy().style(),a=t.pstyle("pie-size"),o=r.x,s=r.y,l=t.width(),u=t.height(),c=Math.min(l,u)/2,d=0;this.usePaths()&&(o=0,s=0),"%"===a.units?c*=a.pfValue:void 0!==a.pfValue&&(c=a.pfValue/2);for(var h=1;h<=i.pieBackgroundN;h++){var p=t.pstyle("pie-"+h+"-background-size").value,f=t.pstyle("pie-"+h+"-background-color").value,g=t.pstyle("pie-"+h+"-background-opacity").value*n,v=p/100;v+d>1&&(v=1-d);var y=1.5*Math.PI+2*Math.PI*d,m=y+2*Math.PI*v;0===p||d>=1||d+v>1||(e.beginPath(),e.moveTo(o,s),e.arc(o,s,c,y,m),e.closePath(),this.colorFillStyle(e,f[0],f[1],f[2],g),e.fill(),d+=v)}};var ku={};ku.getPixelRatio=function(){var e=this.data.contexts[0];if(null!=this.forcedPixelRatio)return this.forcedPixelRatio;var t=e.backingStorePixelRatio||e.webkitBackingStorePixelRatio||e.mozBackingStorePixelRatio||e.msBackingStorePixelRatio||e.oBackingStorePixelRatio||e.backingStorePixelRatio||1;return(window.devicePixelRatio||1)/t},ku.paintCache=function(e){for(var t,n=this.paintCaches=this.paintCaches||[],r=!0,i=0;io.minMbLowQualFrames&&(o.motionBlurPxRatio=o.mbPxRBlurry)),o.clearingMotionBlur&&(o.motionBlurPxRatio=1),o.textureDrawLastFrame&&!d&&(c[o.NODE]=!0,c[o.SELECT_BOX]=!0);var m=l.style(),b=l.zoom(),x=void 0!==i?i:b,w=l.pan(),E={x:w.x,y:w.y},k={zoom:b,pan:{x:w.x,y:w.y}},C=o.prevViewport;void 0===C||k.zoom!==C.zoom||k.pan.x!==C.pan.x||k.pan.y!==C.pan.y||g&&!f||(o.motionBlurPxRatio=1),a&&(E=a),x*=s,E.x*=s,E.y*=s;var S=o.getCachedZSortedEles();function P(e,t,n,r,i){var a=e.globalCompositeOperation;e.globalCompositeOperation="destination-out",o.colorFillStyle(e,255,255,255,o.motionBlurTransparency),e.fillRect(t,n,r,i),e.globalCompositeOperation=a}function D(e,r){var s,l,c,d;o.clearingMotionBlur||e!==u.bufferContexts[o.MOTIONBLUR_BUFFER_NODE]&&e!==u.bufferContexts[o.MOTIONBLUR_BUFFER_DRAG]?(s=E,l=x,c=o.canvasWidth,d=o.canvasHeight):(s={x:w.x*p,y:w.y*p},l=b*p,c=o.canvasWidth*p,d=o.canvasHeight*p),e.setTransform(1,0,0,1,0,0),"motionBlur"===r?P(e,0,0,c,d):t||void 0!==r&&!r||e.clearRect(0,0,c,d),n||(e.translate(s.x,s.y),e.scale(l,l)),a&&e.translate(a.x,a.y),i&&e.scale(i,i)}if(d||(o.textureDrawLastFrame=!1),d){if(o.textureDrawLastFrame=!0,!o.textureCache){o.textureCache={},o.textureCache.bb=l.mutableElements().boundingBox(),o.textureCache.texture=o.data.bufferCanvases[o.TEXTURE_BUFFER];var T=o.data.bufferContexts[o.TEXTURE_BUFFER];T.setTransform(1,0,0,1,0,0),T.clearRect(0,0,o.canvasWidth*o.textureMult,o.canvasHeight*o.textureMult),o.render({forcedContext:T,drawOnlyNodeLayer:!0,forcedPxRatio:s*o.textureMult}),(k=o.textureCache.viewport={zoom:l.zoom(),pan:l.pan(),width:o.canvasWidth,height:o.canvasHeight}).mpan={x:(0-k.pan.x)/k.zoom,y:(0-k.pan.y)/k.zoom}}c[o.DRAG]=!1,c[o.NODE]=!1;var _=u.contexts[o.NODE],M=o.textureCache.texture;k=o.textureCache.viewport;_.setTransform(1,0,0,1,0,0),h?P(_,0,0,k.width,k.height):_.clearRect(0,0,k.width,k.height);var B=m.core("outside-texture-bg-color").value,N=m.core("outside-texture-bg-opacity").value;o.colorFillStyle(_,B[0],B[1],B[2],N),_.fillRect(0,0,k.width,k.height);b=l.zoom();D(_,!1),_.clearRect(k.mpan.x,k.mpan.y,k.width/k.zoom/s,k.height/k.zoom/s),_.drawImage(M,k.mpan.x,k.mpan.y,k.width/k.zoom/s,k.height/k.zoom/s)}else o.textureOnViewport&&!t&&(o.textureCache=null);var z=l.extent(),I=o.pinching||o.hoverData.dragging||o.swipePanning||o.data.wheelZooming||o.hoverData.draggingEles||o.cy.animated(),A=o.hideEdgesOnViewport&&I,L=[];if(L[o.NODE]=!c[o.NODE]&&h&&!o.clearedForMotionBlur[o.NODE]||o.clearingMotionBlur,L[o.NODE]&&(o.clearedForMotionBlur[o.NODE]=!0),L[o.DRAG]=!c[o.DRAG]&&h&&!o.clearedForMotionBlur[o.DRAG]||o.clearingMotionBlur,L[o.DRAG]&&(o.clearedForMotionBlur[o.DRAG]=!0),c[o.NODE]||n||r||L[o.NODE]){var O=h&&!L[o.NODE]&&1!==p;D(_=t||(O?o.data.bufferContexts[o.MOTIONBLUR_BUFFER_NODE]:u.contexts[o.NODE]),h&&!O?"motionBlur":void 0),A?o.drawCachedNodes(_,S.nondrag,s,z):o.drawLayeredElements(_,S.nondrag,s,z),o.debug&&o.drawDebugPoints(_,S.nondrag),n||h||(c[o.NODE]=!1)}if(!r&&(c[o.DRAG]||n||L[o.DRAG])){O=h&&!L[o.DRAG]&&1!==p;D(_=t||(O?o.data.bufferContexts[o.MOTIONBLUR_BUFFER_DRAG]:u.contexts[o.DRAG]),h&&!O?"motionBlur":void 0),A?o.drawCachedNodes(_,S.drag,s,z):o.drawCachedElements(_,S.drag,s,z),o.debug&&o.drawDebugPoints(_,S.drag),n||h||(c[o.DRAG]=!1)}if(o.showFps||!r&&c[o.SELECT_BOX]&&!n){if(D(_=t||u.contexts[o.SELECT_BOX]),1==o.selection[4]&&(o.hoverData.selecting||o.touchData.selecting)){b=o.cy.zoom();var R=m.core("selection-box-border-width").value/b;_.lineWidth=R,_.fillStyle="rgba("+m.core("selection-box-color").value[0]+","+m.core("selection-box-color").value[1]+","+m.core("selection-box-color").value[2]+","+m.core("selection-box-opacity").value+")",_.fillRect(o.selection[0],o.selection[1],o.selection[2]-o.selection[0],o.selection[3]-o.selection[1]),R>0&&(_.strokeStyle="rgba("+m.core("selection-box-border-color").value[0]+","+m.core("selection-box-border-color").value[1]+","+m.core("selection-box-border-color").value[2]+","+m.core("selection-box-opacity").value+")",_.strokeRect(o.selection[0],o.selection[1],o.selection[2]-o.selection[0],o.selection[3]-o.selection[1]))}if(u.bgActivePosistion&&!o.hoverData.selecting){b=o.cy.zoom();var V=u.bgActivePosistion;_.fillStyle="rgba("+m.core("active-bg-color").value[0]+","+m.core("active-bg-color").value[1]+","+m.core("active-bg-color").value[2]+","+m.core("active-bg-opacity").value+")",_.beginPath(),_.arc(V.x,V.y,m.core("active-bg-size").pfValue/b,0,2*Math.PI),_.fill()}var F=o.lastRedrawTime;if(o.showFps&&F){F=Math.round(F);var j=Math.round(1e3/F);_.setTransform(1,0,0,1,0,0),_.fillStyle="rgba(255, 0, 0, 0.75)",_.strokeStyle="rgba(255, 0, 0, 0.75)",_.lineWidth=1,_.fillText("1 frame = "+F+" ms = "+j+" fps",0,20);_.strokeRect(0,30,250,20),_.fillRect(0,30,250*Math.min(j/60,1),20)}n||(c[o.SELECT_BOX]=!1)}if(h&&1!==p){var q=u.contexts[o.NODE],Y=o.data.bufferCanvases[o.MOTIONBLUR_BUFFER_NODE],X=u.contexts[o.DRAG],W=o.data.bufferCanvases[o.MOTIONBLUR_BUFFER_DRAG],H=function(e,t,n){e.setTransform(1,0,0,1,0,0),n||!y?e.clearRect(0,0,o.canvasWidth,o.canvasHeight):P(e,0,0,o.canvasWidth,o.canvasHeight);var r=p;e.drawImage(t,0,0,o.canvasWidth*r,o.canvasHeight*r,0,0,o.canvasWidth,o.canvasHeight)};(c[o.NODE]||L[o.NODE])&&(H(q,Y,L[o.NODE]),c[o.NODE]=!1),(c[o.DRAG]||L[o.DRAG])&&(H(X,W,L[o.DRAG]),c[o.DRAG]=!1)}o.prevViewport=k,o.clearingMotionBlur&&(o.clearingMotionBlur=!1,o.motionBlurCleared=!0,o.motionBlur=!0),h&&(o.motionBlurTimeout=setTimeout((function(){o.motionBlurTimeout=null,o.clearedForMotionBlur[o.NODE]=!1,o.clearedForMotionBlur[o.DRAG]=!1,o.motionBlur=!1,o.clearingMotionBlur=!d,o.mbFrames=0,c[o.NODE]=!0,c[o.DRAG]=!0,o.redraw()}),100)),t||l.emit("render")};for(var Cu={drawPolygonPath:function(e,t,n,r,i,a){var o=r/2,s=i/2;e.beginPath&&e.beginPath(),e.moveTo(t+o*a[0],n+s*a[1]);for(var l=1;l0&&a>0){h.clearRect(0,0,i,a),h.globalCompositeOperation="source-over";var p=this.getCachedZSortedEles();if(e.full)h.translate(-n.x1*l,-n.y1*l),h.scale(l,l),this.drawElements(h,p),h.scale(1/l,1/l),h.translate(n.x1*l,n.y1*l);else{var f=t.pan(),g={x:f.x*l,y:f.y*l};l*=t.zoom(),h.translate(g.x,g.y),h.scale(l,l),this.drawElements(h,p),h.scale(1/l,1/l),h.translate(-g.x,-g.y)}e.bg&&(h.globalCompositeOperation="destination-over",h.fillStyle=e.bg,h.rect(0,0,i,a),h.fill())}return d},Bu.png=function(e){return zu(e,this.bufferCanvasImage(e),"image/png")},Bu.jpg=function(e){return zu(e,this.bufferCanvasImage(e),"image/jpeg")};var Iu={nodeShapeImpl:function(e,t,n,r,i,a,o){switch(e){case"ellipse":return this.drawEllipsePath(t,n,r,i,a);case"polygon":return this.drawPolygonPath(t,n,r,i,a,o);case"round-polygon":return this.drawRoundPolygonPath(t,n,r,i,a,o);case"roundrectangle":case"round-rectangle":return this.drawRoundRectanglePath(t,n,r,i,a);case"cutrectangle":case"cut-rectangle":return this.drawCutRectanglePath(t,n,r,i,a);case"bottomroundrectangle":case"bottom-round-rectangle":return this.drawBottomRoundRectanglePath(t,n,r,i,a);case"barrel":return this.drawBarrelPath(t,n,r,i,a)}}},Au=Ou,Lu=Ou.prototype;function Ou(e){var t=this;t.data={canvases:new Array(Lu.CANVAS_LAYERS),contexts:new Array(Lu.CANVAS_LAYERS),canvasNeedsRedraw:new Array(Lu.CANVAS_LAYERS),bufferCanvases:new Array(Lu.BUFFER_COUNT),bufferContexts:new Array(Lu.CANVAS_LAYERS)};t.data.canvasContainer=document.createElement("div");var n=t.data.canvasContainer.style;t.data.canvasContainer.style["-webkit-tap-highlight-color"]="rgba(0,0,0,0)",n.position="relative",n.zIndex="0",n.overflow="hidden";var r=e.cy.container();r.appendChild(t.data.canvasContainer),r.style["-webkit-tap-highlight-color"]="rgba(0,0,0,0)";var i={"-webkit-user-select":"none","-moz-user-select":"-moz-none","user-select":"none","-webkit-tap-highlight-color":"rgba(0,0,0,0)","outline-style":"none"};l&&l.userAgent.match(/msie|trident|edge/i)&&(i["-ms-touch-action"]="none",i["touch-action"]="none");for(var a=0;a -1)) { + // throw "assert failed"; + // } + // } + + return this.owner; +}; + +LNode.prototype.getWidth = function () { + return this.rect.width; +}; + +LNode.prototype.setWidth = function (width) { + this.rect.width = width; +}; + +LNode.prototype.getHeight = function () { + return this.rect.height; +}; + +LNode.prototype.setHeight = function (height) { + this.rect.height = height; +}; + +LNode.prototype.getCenterX = function () { + return this.rect.x + this.rect.width / 2; +}; + +LNode.prototype.getCenterY = function () { + return this.rect.y + this.rect.height / 2; +}; + +LNode.prototype.getCenter = function () { + return new PointD(this.rect.x + this.rect.width / 2, this.rect.y + this.rect.height / 2); +}; + +LNode.prototype.getLocation = function () { + return new PointD(this.rect.x, this.rect.y); +}; + +LNode.prototype.getRect = function () { + return this.rect; +}; + +LNode.prototype.getDiagonal = function () { + return Math.sqrt(this.rect.width * this.rect.width + this.rect.height * this.rect.height); +}; + +/** + * This method returns half the diagonal length of this node. + */ +LNode.prototype.getHalfTheDiagonal = function () { + return Math.sqrt(this.rect.height * this.rect.height + this.rect.width * this.rect.width) / 2; +}; + +LNode.prototype.setRect = function (upperLeft, dimension) { + this.rect.x = upperLeft.x; + this.rect.y = upperLeft.y; + this.rect.width = dimension.width; + this.rect.height = dimension.height; +}; + +LNode.prototype.setCenter = function (cx, cy) { + this.rect.x = cx - this.rect.width / 2; + this.rect.y = cy - this.rect.height / 2; +}; + +LNode.prototype.setLocation = function (x, y) { + this.rect.x = x; + this.rect.y = y; +}; + +LNode.prototype.moveBy = function (dx, dy) { + this.rect.x += dx; + this.rect.y += dy; +}; + +LNode.prototype.getEdgeListToNode = function (to) { + var edgeList = []; + var edge; + var self = this; + + self.edges.forEach(function (edge) { + + if (edge.target == to) { + if (edge.source != self) throw "Incorrect edge source!"; + + edgeList.push(edge); + } + }); + + return edgeList; +}; + +LNode.prototype.getEdgesBetween = function (other) { + var edgeList = []; + var edge; + + var self = this; + self.edges.forEach(function (edge) { + + if (!(edge.source == self || edge.target == self)) throw "Incorrect edge source and/or target"; + + if (edge.target == other || edge.source == other) { + edgeList.push(edge); + } + }); + + return edgeList; +}; + +LNode.prototype.getNeighborsList = function () { + var neighbors = new Set(); + + var self = this; + self.edges.forEach(function (edge) { + + if (edge.source == self) { + neighbors.add(edge.target); + } else { + if (edge.target != self) { + throw "Incorrect incidency!"; + } + + neighbors.add(edge.source); + } + }); + + return neighbors; +}; + +LNode.prototype.withChildren = function () { + var withNeighborsList = new Set(); + var childNode; + var children; + + withNeighborsList.add(this); + + if (this.child != null) { + var nodes = this.child.getNodes(); + for (var i = 0; i < nodes.length; i++) { + childNode = nodes[i]; + children = childNode.withChildren(); + children.forEach(function (node) { + withNeighborsList.add(node); + }); + } + } + + return withNeighborsList; +}; + +LNode.prototype.getNoOfChildren = function () { + var noOfChildren = 0; + var childNode; + + if (this.child == null) { + noOfChildren = 1; + } else { + var nodes = this.child.getNodes(); + for (var i = 0; i < nodes.length; i++) { + childNode = nodes[i]; + + noOfChildren += childNode.getNoOfChildren(); + } + } + + if (noOfChildren == 0) { + noOfChildren = 1; + } + return noOfChildren; +}; + +LNode.prototype.getEstimatedSize = function () { + if (this.estimatedSize == Integer.MIN_VALUE) { + throw "assert failed"; + } + return this.estimatedSize; +}; + +LNode.prototype.calcEstimatedSize = function () { + if (this.child == null) { + return this.estimatedSize = (this.rect.width + this.rect.height) / 2; + } else { + this.estimatedSize = this.child.calcEstimatedSize(); + this.rect.width = this.estimatedSize; + this.rect.height = this.estimatedSize; + + return this.estimatedSize; + } +}; + +LNode.prototype.scatter = function () { + var randomCenterX; + var randomCenterY; + + var minX = -LayoutConstants.INITIAL_WORLD_BOUNDARY; + var maxX = LayoutConstants.INITIAL_WORLD_BOUNDARY; + randomCenterX = LayoutConstants.WORLD_CENTER_X + RandomSeed.nextDouble() * (maxX - minX) + minX; + + var minY = -LayoutConstants.INITIAL_WORLD_BOUNDARY; + var maxY = LayoutConstants.INITIAL_WORLD_BOUNDARY; + randomCenterY = LayoutConstants.WORLD_CENTER_Y + RandomSeed.nextDouble() * (maxY - minY) + minY; + + this.rect.x = randomCenterX; + this.rect.y = randomCenterY; +}; + +LNode.prototype.updateBounds = function () { + if (this.getChild() == null) { + throw "assert failed"; + } + if (this.getChild().getNodes().length != 0) { + // wrap the children nodes by re-arranging the boundaries + var childGraph = this.getChild(); + childGraph.updateBounds(true); + + this.rect.x = childGraph.getLeft(); + this.rect.y = childGraph.getTop(); + + this.setWidth(childGraph.getRight() - childGraph.getLeft()); + this.setHeight(childGraph.getBottom() - childGraph.getTop()); + + // Update compound bounds considering its label properties + if (LayoutConstants.NODE_DIMENSIONS_INCLUDE_LABELS) { + + var width = childGraph.getRight() - childGraph.getLeft(); + var height = childGraph.getBottom() - childGraph.getTop(); + + if (this.labelWidth) { + if (this.labelPosHorizontal == "left") { + this.rect.x -= this.labelWidth; + this.setWidth(width + this.labelWidth); + } else if (this.labelPosHorizontal == "center" && this.labelWidth > width) { + this.rect.x -= (this.labelWidth - width) / 2; + this.setWidth(this.labelWidth); + } else if (this.labelPosHorizontal == "right") { + this.setWidth(width + this.labelWidth); + } + } + + if (this.labelHeight) { + if (this.labelPosVertical == "top") { + this.rect.y -= this.labelHeight; + this.setHeight(height + this.labelHeight); + } else if (this.labelPosVertical == "center" && this.labelHeight > height) { + this.rect.y -= (this.labelHeight - height) / 2; + this.setHeight(this.labelHeight); + } else if (this.labelPosVertical == "bottom") { + this.setHeight(height + this.labelHeight); + } + } + } + } +}; + +LNode.prototype.getInclusionTreeDepth = function () { + if (this.inclusionTreeDepth == Integer.MAX_VALUE) { + throw "assert failed"; + } + return this.inclusionTreeDepth; +}; + +LNode.prototype.transform = function (trans) { + var left = this.rect.x; + + if (left > LayoutConstants.WORLD_BOUNDARY) { + left = LayoutConstants.WORLD_BOUNDARY; + } else if (left < -LayoutConstants.WORLD_BOUNDARY) { + left = -LayoutConstants.WORLD_BOUNDARY; + } + + var top = this.rect.y; + + if (top > LayoutConstants.WORLD_BOUNDARY) { + top = LayoutConstants.WORLD_BOUNDARY; + } else if (top < -LayoutConstants.WORLD_BOUNDARY) { + top = -LayoutConstants.WORLD_BOUNDARY; + } + + var leftTop = new PointD(left, top); + var vLeftTop = trans.inverseTransformPoint(leftTop); + + this.setLocation(vLeftTop.x, vLeftTop.y); +}; + +LNode.prototype.getLeft = function () { + return this.rect.x; +}; + +LNode.prototype.getRight = function () { + return this.rect.x + this.rect.width; +}; + +LNode.prototype.getTop = function () { + return this.rect.y; +}; + +LNode.prototype.getBottom = function () { + return this.rect.y + this.rect.height; +}; + +LNode.prototype.getParent = function () { + if (this.owner == null) { + return null; + } + + return this.owner.getParent(); +}; + +module.exports = LNode; + +/***/ }), +/* 4 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var LayoutConstants = __webpack_require__(0); + +function FDLayoutConstants() {} + +//FDLayoutConstants inherits static props in LayoutConstants +for (var prop in LayoutConstants) { + FDLayoutConstants[prop] = LayoutConstants[prop]; +} + +FDLayoutConstants.MAX_ITERATIONS = 2500; + +FDLayoutConstants.DEFAULT_EDGE_LENGTH = 50; +FDLayoutConstants.DEFAULT_SPRING_STRENGTH = 0.45; +FDLayoutConstants.DEFAULT_REPULSION_STRENGTH = 4500.0; +FDLayoutConstants.DEFAULT_GRAVITY_STRENGTH = 0.4; +FDLayoutConstants.DEFAULT_COMPOUND_GRAVITY_STRENGTH = 1.0; +FDLayoutConstants.DEFAULT_GRAVITY_RANGE_FACTOR = 3.8; +FDLayoutConstants.DEFAULT_COMPOUND_GRAVITY_RANGE_FACTOR = 1.5; +FDLayoutConstants.DEFAULT_USE_SMART_IDEAL_EDGE_LENGTH_CALCULATION = true; +FDLayoutConstants.DEFAULT_USE_SMART_REPULSION_RANGE_CALCULATION = true; +FDLayoutConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL = 0.3; +FDLayoutConstants.COOLING_ADAPTATION_FACTOR = 0.33; +FDLayoutConstants.ADAPTATION_LOWER_NODE_LIMIT = 1000; +FDLayoutConstants.ADAPTATION_UPPER_NODE_LIMIT = 5000; +FDLayoutConstants.MAX_NODE_DISPLACEMENT_INCREMENTAL = 100.0; +FDLayoutConstants.MAX_NODE_DISPLACEMENT = FDLayoutConstants.MAX_NODE_DISPLACEMENT_INCREMENTAL * 3; +FDLayoutConstants.MIN_REPULSION_DIST = FDLayoutConstants.DEFAULT_EDGE_LENGTH / 10.0; +FDLayoutConstants.CONVERGENCE_CHECK_PERIOD = 100; +FDLayoutConstants.PER_LEVEL_IDEAL_EDGE_LENGTH_FACTOR = 0.1; +FDLayoutConstants.MIN_EDGE_LENGTH = 1; +FDLayoutConstants.GRID_CALCULATION_CHECK_PERIOD = 10; + +module.exports = FDLayoutConstants; + +/***/ }), +/* 5 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function PointD(x, y) { + if (x == null && y == null) { + this.x = 0; + this.y = 0; + } else { + this.x = x; + this.y = y; + } +} + +PointD.prototype.getX = function () { + return this.x; +}; + +PointD.prototype.getY = function () { + return this.y; +}; + +PointD.prototype.setX = function (x) { + this.x = x; +}; + +PointD.prototype.setY = function (y) { + this.y = y; +}; + +PointD.prototype.getDifference = function (pt) { + return new DimensionD(this.x - pt.x, this.y - pt.y); +}; + +PointD.prototype.getCopy = function () { + return new PointD(this.x, this.y); +}; + +PointD.prototype.translate = function (dim) { + this.x += dim.width; + this.y += dim.height; + return this; +}; + +module.exports = PointD; + +/***/ }), +/* 6 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var LGraphObject = __webpack_require__(2); +var Integer = __webpack_require__(10); +var LayoutConstants = __webpack_require__(0); +var LGraphManager = __webpack_require__(7); +var LNode = __webpack_require__(3); +var LEdge = __webpack_require__(1); +var RectangleD = __webpack_require__(13); +var Point = __webpack_require__(12); +var LinkedList = __webpack_require__(11); + +function LGraph(parent, obj2, vGraph) { + LGraphObject.call(this, vGraph); + this.estimatedSize = Integer.MIN_VALUE; + this.margin = LayoutConstants.DEFAULT_GRAPH_MARGIN; + this.edges = []; + this.nodes = []; + this.isConnected = false; + this.parent = parent; + + if (obj2 != null && obj2 instanceof LGraphManager) { + this.graphManager = obj2; + } else if (obj2 != null && obj2 instanceof Layout) { + this.graphManager = obj2.graphManager; + } +} + +LGraph.prototype = Object.create(LGraphObject.prototype); +for (var prop in LGraphObject) { + LGraph[prop] = LGraphObject[prop]; +} + +LGraph.prototype.getNodes = function () { + return this.nodes; +}; + +LGraph.prototype.getEdges = function () { + return this.edges; +}; + +LGraph.prototype.getGraphManager = function () { + return this.graphManager; +}; + +LGraph.prototype.getParent = function () { + return this.parent; +}; + +LGraph.prototype.getLeft = function () { + return this.left; +}; + +LGraph.prototype.getRight = function () { + return this.right; +}; + +LGraph.prototype.getTop = function () { + return this.top; +}; + +LGraph.prototype.getBottom = function () { + return this.bottom; +}; + +LGraph.prototype.isConnected = function () { + return this.isConnected; +}; + +LGraph.prototype.add = function (obj1, sourceNode, targetNode) { + if (sourceNode == null && targetNode == null) { + var newNode = obj1; + if (this.graphManager == null) { + throw "Graph has no graph mgr!"; + } + if (this.getNodes().indexOf(newNode) > -1) { + throw "Node already in graph!"; + } + newNode.owner = this; + this.getNodes().push(newNode); + + return newNode; + } else { + var newEdge = obj1; + if (!(this.getNodes().indexOf(sourceNode) > -1 && this.getNodes().indexOf(targetNode) > -1)) { + throw "Source or target not in graph!"; + } + + if (!(sourceNode.owner == targetNode.owner && sourceNode.owner == this)) { + throw "Both owners must be this graph!"; + } + + if (sourceNode.owner != targetNode.owner) { + return null; + } + + // set source and target + newEdge.source = sourceNode; + newEdge.target = targetNode; + + // set as intra-graph edge + newEdge.isInterGraph = false; + + // add to graph edge list + this.getEdges().push(newEdge); + + // add to incidency lists + sourceNode.edges.push(newEdge); + + if (targetNode != sourceNode) { + targetNode.edges.push(newEdge); + } + + return newEdge; + } +}; + +LGraph.prototype.remove = function (obj) { + var node = obj; + if (obj instanceof LNode) { + if (node == null) { + throw "Node is null!"; + } + if (!(node.owner != null && node.owner == this)) { + throw "Owner graph is invalid!"; + } + if (this.graphManager == null) { + throw "Owner graph manager is invalid!"; + } + // remove incident edges first (make a copy to do it safely) + var edgesToBeRemoved = node.edges.slice(); + var edge; + var s = edgesToBeRemoved.length; + for (var i = 0; i < s; i++) { + edge = edgesToBeRemoved[i]; + + if (edge.isInterGraph) { + this.graphManager.remove(edge); + } else { + edge.source.owner.remove(edge); + } + } + + // now the node itself + var index = this.nodes.indexOf(node); + if (index == -1) { + throw "Node not in owner node list!"; + } + + this.nodes.splice(index, 1); + } else if (obj instanceof LEdge) { + var edge = obj; + if (edge == null) { + throw "Edge is null!"; + } + if (!(edge.source != null && edge.target != null)) { + throw "Source and/or target is null!"; + } + if (!(edge.source.owner != null && edge.target.owner != null && edge.source.owner == this && edge.target.owner == this)) { + throw "Source and/or target owner is invalid!"; + } + + var sourceIndex = edge.source.edges.indexOf(edge); + var targetIndex = edge.target.edges.indexOf(edge); + if (!(sourceIndex > -1 && targetIndex > -1)) { + throw "Source and/or target doesn't know this edge!"; + } + + edge.source.edges.splice(sourceIndex, 1); + + if (edge.target != edge.source) { + edge.target.edges.splice(targetIndex, 1); + } + + var index = edge.source.owner.getEdges().indexOf(edge); + if (index == -1) { + throw "Not in owner's edge list!"; + } + + edge.source.owner.getEdges().splice(index, 1); + } +}; + +LGraph.prototype.updateLeftTop = function () { + var top = Integer.MAX_VALUE; + var left = Integer.MAX_VALUE; + var nodeTop; + var nodeLeft; + var margin; + + var nodes = this.getNodes(); + var s = nodes.length; + + for (var i = 0; i < s; i++) { + var lNode = nodes[i]; + nodeTop = lNode.getTop(); + nodeLeft = lNode.getLeft(); + + if (top > nodeTop) { + top = nodeTop; + } + + if (left > nodeLeft) { + left = nodeLeft; + } + } + + // Do we have any nodes in this graph? + if (top == Integer.MAX_VALUE) { + return null; + } + + if (nodes[0].getParent().paddingLeft != undefined) { + margin = nodes[0].getParent().paddingLeft; + } else { + margin = this.margin; + } + + this.left = left - margin; + this.top = top - margin; + + // Apply the margins and return the result + return new Point(this.left, this.top); +}; + +LGraph.prototype.updateBounds = function (recursive) { + // calculate bounds + var left = Integer.MAX_VALUE; + var right = -Integer.MAX_VALUE; + var top = Integer.MAX_VALUE; + var bottom = -Integer.MAX_VALUE; + var nodeLeft; + var nodeRight; + var nodeTop; + var nodeBottom; + var margin; + + var nodes = this.nodes; + var s = nodes.length; + for (var i = 0; i < s; i++) { + var lNode = nodes[i]; + + if (recursive && lNode.child != null) { + lNode.updateBounds(); + } + nodeLeft = lNode.getLeft(); + nodeRight = lNode.getRight(); + nodeTop = lNode.getTop(); + nodeBottom = lNode.getBottom(); + + if (left > nodeLeft) { + left = nodeLeft; + } + + if (right < nodeRight) { + right = nodeRight; + } + + if (top > nodeTop) { + top = nodeTop; + } + + if (bottom < nodeBottom) { + bottom = nodeBottom; + } + } + + var boundingRect = new RectangleD(left, top, right - left, bottom - top); + if (left == Integer.MAX_VALUE) { + this.left = this.parent.getLeft(); + this.right = this.parent.getRight(); + this.top = this.parent.getTop(); + this.bottom = this.parent.getBottom(); + } + + if (nodes[0].getParent().paddingLeft != undefined) { + margin = nodes[0].getParent().paddingLeft; + } else { + margin = this.margin; + } + + this.left = boundingRect.x - margin; + this.right = boundingRect.x + boundingRect.width + margin; + this.top = boundingRect.y - margin; + this.bottom = boundingRect.y + boundingRect.height + margin; +}; + +LGraph.calculateBounds = function (nodes) { + var left = Integer.MAX_VALUE; + var right = -Integer.MAX_VALUE; + var top = Integer.MAX_VALUE; + var bottom = -Integer.MAX_VALUE; + var nodeLeft; + var nodeRight; + var nodeTop; + var nodeBottom; + + var s = nodes.length; + + for (var i = 0; i < s; i++) { + var lNode = nodes[i]; + nodeLeft = lNode.getLeft(); + nodeRight = lNode.getRight(); + nodeTop = lNode.getTop(); + nodeBottom = lNode.getBottom(); + + if (left > nodeLeft) { + left = nodeLeft; + } + + if (right < nodeRight) { + right = nodeRight; + } + + if (top > nodeTop) { + top = nodeTop; + } + + if (bottom < nodeBottom) { + bottom = nodeBottom; + } + } + + var boundingRect = new RectangleD(left, top, right - left, bottom - top); + + return boundingRect; +}; + +LGraph.prototype.getInclusionTreeDepth = function () { + if (this == this.graphManager.getRoot()) { + return 1; + } else { + return this.parent.getInclusionTreeDepth(); + } +}; + +LGraph.prototype.getEstimatedSize = function () { + if (this.estimatedSize == Integer.MIN_VALUE) { + throw "assert failed"; + } + return this.estimatedSize; +}; + +LGraph.prototype.calcEstimatedSize = function () { + var size = 0; + var nodes = this.nodes; + var s = nodes.length; + + for (var i = 0; i < s; i++) { + var lNode = nodes[i]; + size += lNode.calcEstimatedSize(); + } + + if (size == 0) { + this.estimatedSize = LayoutConstants.EMPTY_COMPOUND_NODE_SIZE; + } else { + this.estimatedSize = size / Math.sqrt(this.nodes.length); + } + + return this.estimatedSize; +}; + +LGraph.prototype.updateConnected = function () { + var self = this; + if (this.nodes.length == 0) { + this.isConnected = true; + return; + } + + var queue = new LinkedList(); + var visited = new Set(); + var currentNode = this.nodes[0]; + var neighborEdges; + var currentNeighbor; + var childrenOfNode = currentNode.withChildren(); + childrenOfNode.forEach(function (node) { + queue.push(node); + visited.add(node); + }); + + while (queue.length !== 0) { + currentNode = queue.shift(); + + // Traverse all neighbors of this node + neighborEdges = currentNode.getEdges(); + var size = neighborEdges.length; + for (var i = 0; i < size; i++) { + var neighborEdge = neighborEdges[i]; + currentNeighbor = neighborEdge.getOtherEndInGraph(currentNode, this); + + // Add unvisited neighbors to the list to visit + if (currentNeighbor != null && !visited.has(currentNeighbor)) { + var childrenOfNeighbor = currentNeighbor.withChildren(); + + childrenOfNeighbor.forEach(function (node) { + queue.push(node); + visited.add(node); + }); + } + } + } + + this.isConnected = false; + + if (visited.size >= this.nodes.length) { + var noOfVisitedInThisGraph = 0; + + visited.forEach(function (visitedNode) { + if (visitedNode.owner == self) { + noOfVisitedInThisGraph++; + } + }); + + if (noOfVisitedInThisGraph == this.nodes.length) { + this.isConnected = true; + } + } +}; + +module.exports = LGraph; + +/***/ }), +/* 7 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var LGraph; +var LEdge = __webpack_require__(1); + +function LGraphManager(layout) { + LGraph = __webpack_require__(6); // It may be better to initilize this out of this function but it gives an error (Right-hand side of 'instanceof' is not callable) now. + this.layout = layout; + + this.graphs = []; + this.edges = []; +} + +LGraphManager.prototype.addRoot = function () { + var ngraph = this.layout.newGraph(); + var nnode = this.layout.newNode(null); + var root = this.add(ngraph, nnode); + this.setRootGraph(root); + return this.rootGraph; +}; + +LGraphManager.prototype.add = function (newGraph, parentNode, newEdge, sourceNode, targetNode) { + //there are just 2 parameters are passed then it adds an LGraph else it adds an LEdge + if (newEdge == null && sourceNode == null && targetNode == null) { + if (newGraph == null) { + throw "Graph is null!"; + } + if (parentNode == null) { + throw "Parent node is null!"; + } + if (this.graphs.indexOf(newGraph) > -1) { + throw "Graph already in this graph mgr!"; + } + + this.graphs.push(newGraph); + + if (newGraph.parent != null) { + throw "Already has a parent!"; + } + if (parentNode.child != null) { + throw "Already has a child!"; + } + + newGraph.parent = parentNode; + parentNode.child = newGraph; + + return newGraph; + } else { + //change the order of the parameters + targetNode = newEdge; + sourceNode = parentNode; + newEdge = newGraph; + var sourceGraph = sourceNode.getOwner(); + var targetGraph = targetNode.getOwner(); + + if (!(sourceGraph != null && sourceGraph.getGraphManager() == this)) { + throw "Source not in this graph mgr!"; + } + if (!(targetGraph != null && targetGraph.getGraphManager() == this)) { + throw "Target not in this graph mgr!"; + } + + if (sourceGraph == targetGraph) { + newEdge.isInterGraph = false; + return sourceGraph.add(newEdge, sourceNode, targetNode); + } else { + newEdge.isInterGraph = true; + + // set source and target + newEdge.source = sourceNode; + newEdge.target = targetNode; + + // add edge to inter-graph edge list + if (this.edges.indexOf(newEdge) > -1) { + throw "Edge already in inter-graph edge list!"; + } + + this.edges.push(newEdge); + + // add edge to source and target incidency lists + if (!(newEdge.source != null && newEdge.target != null)) { + throw "Edge source and/or target is null!"; + } + + if (!(newEdge.source.edges.indexOf(newEdge) == -1 && newEdge.target.edges.indexOf(newEdge) == -1)) { + throw "Edge already in source and/or target incidency list!"; + } + + newEdge.source.edges.push(newEdge); + newEdge.target.edges.push(newEdge); + + return newEdge; + } + } +}; + +LGraphManager.prototype.remove = function (lObj) { + if (lObj instanceof LGraph) { + var graph = lObj; + if (graph.getGraphManager() != this) { + throw "Graph not in this graph mgr"; + } + if (!(graph == this.rootGraph || graph.parent != null && graph.parent.graphManager == this)) { + throw "Invalid parent node!"; + } + + // first the edges (make a copy to do it safely) + var edgesToBeRemoved = []; + + edgesToBeRemoved = edgesToBeRemoved.concat(graph.getEdges()); + + var edge; + var s = edgesToBeRemoved.length; + for (var i = 0; i < s; i++) { + edge = edgesToBeRemoved[i]; + graph.remove(edge); + } + + // then the nodes (make a copy to do it safely) + var nodesToBeRemoved = []; + + nodesToBeRemoved = nodesToBeRemoved.concat(graph.getNodes()); + + var node; + s = nodesToBeRemoved.length; + for (var i = 0; i < s; i++) { + node = nodesToBeRemoved[i]; + graph.remove(node); + } + + // check if graph is the root + if (graph == this.rootGraph) { + this.setRootGraph(null); + } + + // now remove the graph itself + var index = this.graphs.indexOf(graph); + this.graphs.splice(index, 1); + + // also reset the parent of the graph + graph.parent = null; + } else if (lObj instanceof LEdge) { + edge = lObj; + if (edge == null) { + throw "Edge is null!"; + } + if (!edge.isInterGraph) { + throw "Not an inter-graph edge!"; + } + if (!(edge.source != null && edge.target != null)) { + throw "Source and/or target is null!"; + } + + // remove edge from source and target nodes' incidency lists + + if (!(edge.source.edges.indexOf(edge) != -1 && edge.target.edges.indexOf(edge) != -1)) { + throw "Source and/or target doesn't know this edge!"; + } + + var index = edge.source.edges.indexOf(edge); + edge.source.edges.splice(index, 1); + index = edge.target.edges.indexOf(edge); + edge.target.edges.splice(index, 1); + + // remove edge from owner graph manager's inter-graph edge list + + if (!(edge.source.owner != null && edge.source.owner.getGraphManager() != null)) { + throw "Edge owner graph or owner graph manager is null!"; + } + if (edge.source.owner.getGraphManager().edges.indexOf(edge) == -1) { + throw "Not in owner graph manager's edge list!"; + } + + var index = edge.source.owner.getGraphManager().edges.indexOf(edge); + edge.source.owner.getGraphManager().edges.splice(index, 1); + } +}; + +LGraphManager.prototype.updateBounds = function () { + this.rootGraph.updateBounds(true); +}; + +LGraphManager.prototype.getGraphs = function () { + return this.graphs; +}; + +LGraphManager.prototype.getAllNodes = function () { + if (this.allNodes == null) { + var nodeList = []; + var graphs = this.getGraphs(); + var s = graphs.length; + for (var i = 0; i < s; i++) { + nodeList = nodeList.concat(graphs[i].getNodes()); + } + this.allNodes = nodeList; + } + return this.allNodes; +}; + +LGraphManager.prototype.resetAllNodes = function () { + this.allNodes = null; +}; + +LGraphManager.prototype.resetAllEdges = function () { + this.allEdges = null; +}; + +LGraphManager.prototype.resetAllNodesToApplyGravitation = function () { + this.allNodesToApplyGravitation = null; +}; + +LGraphManager.prototype.getAllEdges = function () { + if (this.allEdges == null) { + var edgeList = []; + var graphs = this.getGraphs(); + var s = graphs.length; + for (var i = 0; i < graphs.length; i++) { + edgeList = edgeList.concat(graphs[i].getEdges()); + } + + edgeList = edgeList.concat(this.edges); + + this.allEdges = edgeList; + } + return this.allEdges; +}; + +LGraphManager.prototype.getAllNodesToApplyGravitation = function () { + return this.allNodesToApplyGravitation; +}; + +LGraphManager.prototype.setAllNodesToApplyGravitation = function (nodeList) { + if (this.allNodesToApplyGravitation != null) { + throw "assert failed"; + } + + this.allNodesToApplyGravitation = nodeList; +}; + +LGraphManager.prototype.getRoot = function () { + return this.rootGraph; +}; + +LGraphManager.prototype.setRootGraph = function (graph) { + if (graph.getGraphManager() != this) { + throw "Root not in this graph mgr!"; + } + + this.rootGraph = graph; + // root graph must have a root node associated with it for convenience + if (graph.parent == null) { + graph.parent = this.layout.newNode("Root node"); + } +}; + +LGraphManager.prototype.getLayout = function () { + return this.layout; +}; + +LGraphManager.prototype.isOneAncestorOfOther = function (firstNode, secondNode) { + if (!(firstNode != null && secondNode != null)) { + throw "assert failed"; + } + + if (firstNode == secondNode) { + return true; + } + // Is second node an ancestor of the first one? + var ownerGraph = firstNode.getOwner(); + var parentNode; + + do { + parentNode = ownerGraph.getParent(); + + if (parentNode == null) { + break; + } + + if (parentNode == secondNode) { + return true; + } + + ownerGraph = parentNode.getOwner(); + if (ownerGraph == null) { + break; + } + } while (true); + // Is first node an ancestor of the second one? + ownerGraph = secondNode.getOwner(); + + do { + parentNode = ownerGraph.getParent(); + + if (parentNode == null) { + break; + } + + if (parentNode == firstNode) { + return true; + } + + ownerGraph = parentNode.getOwner(); + if (ownerGraph == null) { + break; + } + } while (true); + + return false; +}; + +LGraphManager.prototype.calcLowestCommonAncestors = function () { + var edge; + var sourceNode; + var targetNode; + var sourceAncestorGraph; + var targetAncestorGraph; + + var edges = this.getAllEdges(); + var s = edges.length; + for (var i = 0; i < s; i++) { + edge = edges[i]; + + sourceNode = edge.source; + targetNode = edge.target; + edge.lca = null; + edge.sourceInLca = sourceNode; + edge.targetInLca = targetNode; + + if (sourceNode == targetNode) { + edge.lca = sourceNode.getOwner(); + continue; + } + + sourceAncestorGraph = sourceNode.getOwner(); + + while (edge.lca == null) { + edge.targetInLca = targetNode; + targetAncestorGraph = targetNode.getOwner(); + + while (edge.lca == null) { + if (targetAncestorGraph == sourceAncestorGraph) { + edge.lca = targetAncestorGraph; + break; + } + + if (targetAncestorGraph == this.rootGraph) { + break; + } + + if (edge.lca != null) { + throw "assert failed"; + } + edge.targetInLca = targetAncestorGraph.getParent(); + targetAncestorGraph = edge.targetInLca.getOwner(); + } + + if (sourceAncestorGraph == this.rootGraph) { + break; + } + + if (edge.lca == null) { + edge.sourceInLca = sourceAncestorGraph.getParent(); + sourceAncestorGraph = edge.sourceInLca.getOwner(); + } + } + + if (edge.lca == null) { + throw "assert failed"; + } + } +}; + +LGraphManager.prototype.calcLowestCommonAncestor = function (firstNode, secondNode) { + if (firstNode == secondNode) { + return firstNode.getOwner(); + } + var firstOwnerGraph = firstNode.getOwner(); + + do { + if (firstOwnerGraph == null) { + break; + } + var secondOwnerGraph = secondNode.getOwner(); + + do { + if (secondOwnerGraph == null) { + break; + } + + if (secondOwnerGraph == firstOwnerGraph) { + return secondOwnerGraph; + } + secondOwnerGraph = secondOwnerGraph.getParent().getOwner(); + } while (true); + + firstOwnerGraph = firstOwnerGraph.getParent().getOwner(); + } while (true); + + return firstOwnerGraph; +}; + +LGraphManager.prototype.calcInclusionTreeDepths = function (graph, depth) { + if (graph == null && depth == null) { + graph = this.rootGraph; + depth = 1; + } + var node; + + var nodes = graph.getNodes(); + var s = nodes.length; + for (var i = 0; i < s; i++) { + node = nodes[i]; + node.inclusionTreeDepth = depth; + + if (node.child != null) { + this.calcInclusionTreeDepths(node.child, depth + 1); + } + } +}; + +LGraphManager.prototype.includesInvalidEdge = function () { + var edge; + var edgesToRemove = []; + + var s = this.edges.length; + for (var i = 0; i < s; i++) { + edge = this.edges[i]; + + if (this.isOneAncestorOfOther(edge.source, edge.target)) { + edgesToRemove.push(edge); + } + } + + // Remove invalid edges from graph manager + for (var i = 0; i < edgesToRemove.length; i++) { + this.remove(edgesToRemove[i]); + } + + // Invalid edges are cleared, so return false + return false; +}; + +module.exports = LGraphManager; + +/***/ }), +/* 8 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +/** + * This class maintains a list of static geometry related utility methods. + * + * + * Copyright: i-Vis Research Group, Bilkent University, 2007 - present + */ + +var Point = __webpack_require__(12); + +function IGeometry() {} + +/** + * This method calculates *half* the amount in x and y directions of the two + * input rectangles needed to separate them keeping their respective + * positioning, and returns the result in the input array. An input + * separation buffer added to the amount in both directions. We assume that + * the two rectangles do intersect. + */ +IGeometry.calcSeparationAmount = function (rectA, rectB, overlapAmount, separationBuffer) { + if (!rectA.intersects(rectB)) { + throw "assert failed"; + } + + var directions = new Array(2); + + this.decideDirectionsForOverlappingNodes(rectA, rectB, directions); + + overlapAmount[0] = Math.min(rectA.getRight(), rectB.getRight()) - Math.max(rectA.x, rectB.x); + overlapAmount[1] = Math.min(rectA.getBottom(), rectB.getBottom()) - Math.max(rectA.y, rectB.y); + + // update the overlapping amounts for the following cases: + if (rectA.getX() <= rectB.getX() && rectA.getRight() >= rectB.getRight()) { + /* Case x.1: + * + * rectA + * | | + * | _________ | + * | | | | + * |________|_______|______| + * | | + * | | + * rectB + */ + overlapAmount[0] += Math.min(rectB.getX() - rectA.getX(), rectA.getRight() - rectB.getRight()); + } else if (rectB.getX() <= rectA.getX() && rectB.getRight() >= rectA.getRight()) { + /* Case x.2: + * + * rectB + * | | + * | _________ | + * | | | | + * |________|_______|______| + * | | + * | | + * rectA + */ + overlapAmount[0] += Math.min(rectA.getX() - rectB.getX(), rectB.getRight() - rectA.getRight()); + } + if (rectA.getY() <= rectB.getY() && rectA.getBottom() >= rectB.getBottom()) { + /* Case y.1: + * ________ rectA + * | + * | + * ______|____ rectB + * | | + * | | + * ______|____| + * | + * | + * |________ + * + */ + overlapAmount[1] += Math.min(rectB.getY() - rectA.getY(), rectA.getBottom() - rectB.getBottom()); + } else if (rectB.getY() <= rectA.getY() && rectB.getBottom() >= rectA.getBottom()) { + /* Case y.2: + * ________ rectB + * | + * | + * ______|____ rectA + * | | + * | | + * ______|____| + * | + * | + * |________ + * + */ + overlapAmount[1] += Math.min(rectA.getY() - rectB.getY(), rectB.getBottom() - rectA.getBottom()); + } + + // find slope of the line passes two centers + var slope = Math.abs((rectB.getCenterY() - rectA.getCenterY()) / (rectB.getCenterX() - rectA.getCenterX())); + // if centers are overlapped + if (rectB.getCenterY() === rectA.getCenterY() && rectB.getCenterX() === rectA.getCenterX()) { + // assume the slope is 1 (45 degree) + slope = 1.0; + } + + var moveByY = slope * overlapAmount[0]; + var moveByX = overlapAmount[1] / slope; + if (overlapAmount[0] < moveByX) { + moveByX = overlapAmount[0]; + } else { + moveByY = overlapAmount[1]; + } + // return half the amount so that if each rectangle is moved by these + // amounts in opposite directions, overlap will be resolved + overlapAmount[0] = -1 * directions[0] * (moveByX / 2 + separationBuffer); + overlapAmount[1] = -1 * directions[1] * (moveByY / 2 + separationBuffer); +}; + +/** + * This method decides the separation direction of overlapping nodes + * + * if directions[0] = -1, then rectA goes left + * if directions[0] = 1, then rectA goes right + * if directions[1] = -1, then rectA goes up + * if directions[1] = 1, then rectA goes down + */ +IGeometry.decideDirectionsForOverlappingNodes = function (rectA, rectB, directions) { + if (rectA.getCenterX() < rectB.getCenterX()) { + directions[0] = -1; + } else { + directions[0] = 1; + } + + if (rectA.getCenterY() < rectB.getCenterY()) { + directions[1] = -1; + } else { + directions[1] = 1; + } +}; + +/** + * This method calculates the intersection (clipping) points of the two + * input rectangles with line segment defined by the centers of these two + * rectangles. The clipping points are saved in the input double array and + * whether or not the two rectangles overlap is returned. + */ +IGeometry.getIntersection2 = function (rectA, rectB, result) { + //result[0-1] will contain clipPoint of rectA, result[2-3] will contain clipPoint of rectB + var p1x = rectA.getCenterX(); + var p1y = rectA.getCenterY(); + var p2x = rectB.getCenterX(); + var p2y = rectB.getCenterY(); + + //if two rectangles intersect, then clipping points are centers + if (rectA.intersects(rectB)) { + result[0] = p1x; + result[1] = p1y; + result[2] = p2x; + result[3] = p2y; + return true; + } + //variables for rectA + var topLeftAx = rectA.getX(); + var topLeftAy = rectA.getY(); + var topRightAx = rectA.getRight(); + var bottomLeftAx = rectA.getX(); + var bottomLeftAy = rectA.getBottom(); + var bottomRightAx = rectA.getRight(); + var halfWidthA = rectA.getWidthHalf(); + var halfHeightA = rectA.getHeightHalf(); + //variables for rectB + var topLeftBx = rectB.getX(); + var topLeftBy = rectB.getY(); + var topRightBx = rectB.getRight(); + var bottomLeftBx = rectB.getX(); + var bottomLeftBy = rectB.getBottom(); + var bottomRightBx = rectB.getRight(); + var halfWidthB = rectB.getWidthHalf(); + var halfHeightB = rectB.getHeightHalf(); + + //flag whether clipping points are found + var clipPointAFound = false; + var clipPointBFound = false; + + // line is vertical + if (p1x === p2x) { + if (p1y > p2y) { + result[0] = p1x; + result[1] = topLeftAy; + result[2] = p2x; + result[3] = bottomLeftBy; + return false; + } else if (p1y < p2y) { + result[0] = p1x; + result[1] = bottomLeftAy; + result[2] = p2x; + result[3] = topLeftBy; + return false; + } else { + //not line, return null; + } + } + // line is horizontal + else if (p1y === p2y) { + if (p1x > p2x) { + result[0] = topLeftAx; + result[1] = p1y; + result[2] = topRightBx; + result[3] = p2y; + return false; + } else if (p1x < p2x) { + result[0] = topRightAx; + result[1] = p1y; + result[2] = topLeftBx; + result[3] = p2y; + return false; + } else { + //not valid line, return null; + } + } else { + //slopes of rectA's and rectB's diagonals + var slopeA = rectA.height / rectA.width; + var slopeB = rectB.height / rectB.width; + + //slope of line between center of rectA and center of rectB + var slopePrime = (p2y - p1y) / (p2x - p1x); + var cardinalDirectionA = void 0; + var cardinalDirectionB = void 0; + var tempPointAx = void 0; + var tempPointAy = void 0; + var tempPointBx = void 0; + var tempPointBy = void 0; + + //determine whether clipping point is the corner of nodeA + if (-slopeA === slopePrime) { + if (p1x > p2x) { + result[0] = bottomLeftAx; + result[1] = bottomLeftAy; + clipPointAFound = true; + } else { + result[0] = topRightAx; + result[1] = topLeftAy; + clipPointAFound = true; + } + } else if (slopeA === slopePrime) { + if (p1x > p2x) { + result[0] = topLeftAx; + result[1] = topLeftAy; + clipPointAFound = true; + } else { + result[0] = bottomRightAx; + result[1] = bottomLeftAy; + clipPointAFound = true; + } + } + + //determine whether clipping point is the corner of nodeB + if (-slopeB === slopePrime) { + if (p2x > p1x) { + result[2] = bottomLeftBx; + result[3] = bottomLeftBy; + clipPointBFound = true; + } else { + result[2] = topRightBx; + result[3] = topLeftBy; + clipPointBFound = true; + } + } else if (slopeB === slopePrime) { + if (p2x > p1x) { + result[2] = topLeftBx; + result[3] = topLeftBy; + clipPointBFound = true; + } else { + result[2] = bottomRightBx; + result[3] = bottomLeftBy; + clipPointBFound = true; + } + } + + //if both clipping points are corners + if (clipPointAFound && clipPointBFound) { + return false; + } + + //determine Cardinal Direction of rectangles + if (p1x > p2x) { + if (p1y > p2y) { + cardinalDirectionA = this.getCardinalDirection(slopeA, slopePrime, 4); + cardinalDirectionB = this.getCardinalDirection(slopeB, slopePrime, 2); + } else { + cardinalDirectionA = this.getCardinalDirection(-slopeA, slopePrime, 3); + cardinalDirectionB = this.getCardinalDirection(-slopeB, slopePrime, 1); + } + } else { + if (p1y > p2y) { + cardinalDirectionA = this.getCardinalDirection(-slopeA, slopePrime, 1); + cardinalDirectionB = this.getCardinalDirection(-slopeB, slopePrime, 3); + } else { + cardinalDirectionA = this.getCardinalDirection(slopeA, slopePrime, 2); + cardinalDirectionB = this.getCardinalDirection(slopeB, slopePrime, 4); + } + } + //calculate clipping Point if it is not found before + if (!clipPointAFound) { + switch (cardinalDirectionA) { + case 1: + tempPointAy = topLeftAy; + tempPointAx = p1x + -halfHeightA / slopePrime; + result[0] = tempPointAx; + result[1] = tempPointAy; + break; + case 2: + tempPointAx = bottomRightAx; + tempPointAy = p1y + halfWidthA * slopePrime; + result[0] = tempPointAx; + result[1] = tempPointAy; + break; + case 3: + tempPointAy = bottomLeftAy; + tempPointAx = p1x + halfHeightA / slopePrime; + result[0] = tempPointAx; + result[1] = tempPointAy; + break; + case 4: + tempPointAx = bottomLeftAx; + tempPointAy = p1y + -halfWidthA * slopePrime; + result[0] = tempPointAx; + result[1] = tempPointAy; + break; + } + } + if (!clipPointBFound) { + switch (cardinalDirectionB) { + case 1: + tempPointBy = topLeftBy; + tempPointBx = p2x + -halfHeightB / slopePrime; + result[2] = tempPointBx; + result[3] = tempPointBy; + break; + case 2: + tempPointBx = bottomRightBx; + tempPointBy = p2y + halfWidthB * slopePrime; + result[2] = tempPointBx; + result[3] = tempPointBy; + break; + case 3: + tempPointBy = bottomLeftBy; + tempPointBx = p2x + halfHeightB / slopePrime; + result[2] = tempPointBx; + result[3] = tempPointBy; + break; + case 4: + tempPointBx = bottomLeftBx; + tempPointBy = p2y + -halfWidthB * slopePrime; + result[2] = tempPointBx; + result[3] = tempPointBy; + break; + } + } + } + return false; +}; + +/** + * This method returns in which cardinal direction does input point stays + * 1: North + * 2: East + * 3: South + * 4: West + */ +IGeometry.getCardinalDirection = function (slope, slopePrime, line) { + if (slope > slopePrime) { + return line; + } else { + return 1 + line % 4; + } +}; + +/** + * This method calculates the intersection of the two lines defined by + * point pairs (s1,s2) and (f1,f2). + */ +IGeometry.getIntersection = function (s1, s2, f1, f2) { + if (f2 == null) { + return this.getIntersection2(s1, s2, f1); + } + + var x1 = s1.x; + var y1 = s1.y; + var x2 = s2.x; + var y2 = s2.y; + var x3 = f1.x; + var y3 = f1.y; + var x4 = f2.x; + var y4 = f2.y; + var x = void 0, + y = void 0; // intersection point + var a1 = void 0, + a2 = void 0, + b1 = void 0, + b2 = void 0, + c1 = void 0, + c2 = void 0; // coefficients of line eqns. + var denom = void 0; + + a1 = y2 - y1; + b1 = x1 - x2; + c1 = x2 * y1 - x1 * y2; // { a1*x + b1*y + c1 = 0 is line 1 } + + a2 = y4 - y3; + b2 = x3 - x4; + c2 = x4 * y3 - x3 * y4; // { a2*x + b2*y + c2 = 0 is line 2 } + + denom = a1 * b2 - a2 * b1; + + if (denom === 0) { + return null; + } + + x = (b1 * c2 - b2 * c1) / denom; + y = (a2 * c1 - a1 * c2) / denom; + + return new Point(x, y); +}; + +/** + * This method finds and returns the angle of the vector from the + x-axis + * in clockwise direction (compatible w/ Java coordinate system!). + */ +IGeometry.angleOfVector = function (Cx, Cy, Nx, Ny) { + var C_angle = void 0; + + if (Cx !== Nx) { + C_angle = Math.atan((Ny - Cy) / (Nx - Cx)); + + if (Nx < Cx) { + C_angle += Math.PI; + } else if (Ny < Cy) { + C_angle += this.TWO_PI; + } + } else if (Ny < Cy) { + C_angle = this.ONE_AND_HALF_PI; // 270 degrees + } else { + C_angle = this.HALF_PI; // 90 degrees + } + + return C_angle; +}; + +/** + * This method checks whether the given two line segments (one with point + * p1 and p2, the other with point p3 and p4) intersect at a point other + * than these points. + */ +IGeometry.doIntersect = function (p1, p2, p3, p4) { + var a = p1.x; + var b = p1.y; + var c = p2.x; + var d = p2.y; + var p = p3.x; + var q = p3.y; + var r = p4.x; + var s = p4.y; + var det = (c - a) * (s - q) - (r - p) * (d - b); + + if (det === 0) { + return false; + } else { + var lambda = ((s - q) * (r - a) + (p - r) * (s - b)) / det; + var gamma = ((b - d) * (r - a) + (c - a) * (s - b)) / det; + return 0 < lambda && lambda < 1 && 0 < gamma && gamma < 1; + } +}; + +/** + * This method checks and calculates the intersection of + * a line segment and a circle. + */ +IGeometry.findCircleLineIntersections = function (Ex, Ey, Lx, Ly, Cx, Cy, r) { + + // E is the starting point of the ray, + // L is the end point of the ray, + // C is the center of sphere you're testing against + // r is the radius of that sphere + + // Compute: + // d = L - E ( Direction vector of ray, from start to end ) + // f = E - C ( Vector from center sphere to ray start ) + + // Then the intersection is found by.. + // P = E + t * d + // This is a parametric equation: + // Px = Ex + tdx + // Py = Ey + tdy + + // get a, b, c values + var a = (Lx - Ex) * (Lx - Ex) + (Ly - Ey) * (Ly - Ey); + var b = 2 * ((Ex - Cx) * (Lx - Ex) + (Ey - Cy) * (Ly - Ey)); + var c = (Ex - Cx) * (Ex - Cx) + (Ey - Cy) * (Ey - Cy) - r * r; + + // get discriminant + var disc = b * b - 4 * a * c; + if (disc >= 0) { + // insert into quadratic formula + var t1 = (-b + Math.sqrt(b * b - 4 * a * c)) / (2 * a); + var t2 = (-b - Math.sqrt(b * b - 4 * a * c)) / (2 * a); + var intersections = null; + if (t1 >= 0 && t1 <= 1) { + // t1 is the intersection, and it's closer than t2 + // (since t1 uses -b - discriminant) + // Impale, Poke + return [t1]; + } + + // here t1 didn't intersect so we are either started + // inside the sphere or completely past it + if (t2 >= 0 && t2 <= 1) { + // ExitWound + return [t2]; + } + + return intersections; + } else return null; +}; + +// ----------------------------------------------------------------------------- +// Section: Class Constants +// ----------------------------------------------------------------------------- +/** + * Some useful pre-calculated constants + */ +IGeometry.HALF_PI = 0.5 * Math.PI; +IGeometry.ONE_AND_HALF_PI = 1.5 * Math.PI; +IGeometry.TWO_PI = 2.0 * Math.PI; +IGeometry.THREE_PI = 3.0 * Math.PI; + +module.exports = IGeometry; + +/***/ }), +/* 9 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function IMath() {} + +/** + * This method returns the sign of the input value. + */ +IMath.sign = function (value) { + if (value > 0) { + return 1; + } else if (value < 0) { + return -1; + } else { + return 0; + } +}; + +IMath.floor = function (value) { + return value < 0 ? Math.ceil(value) : Math.floor(value); +}; + +IMath.ceil = function (value) { + return value < 0 ? Math.floor(value) : Math.ceil(value); +}; + +module.exports = IMath; + +/***/ }), +/* 10 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function Integer() {} + +Integer.MAX_VALUE = 2147483647; +Integer.MIN_VALUE = -2147483648; + +module.exports = Integer; + +/***/ }), +/* 11 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +var nodeFrom = function nodeFrom(value) { + return { value: value, next: null, prev: null }; +}; + +var add = function add(prev, node, next, list) { + if (prev !== null) { + prev.next = node; + } else { + list.head = node; + } + + if (next !== null) { + next.prev = node; + } else { + list.tail = node; + } + + node.prev = prev; + node.next = next; + + list.length++; + + return node; +}; + +var _remove = function _remove(node, list) { + var prev = node.prev, + next = node.next; + + + if (prev !== null) { + prev.next = next; + } else { + list.head = next; + } + + if (next !== null) { + next.prev = prev; + } else { + list.tail = prev; + } + + node.prev = node.next = null; + + list.length--; + + return node; +}; + +var LinkedList = function () { + function LinkedList(vals) { + var _this = this; + + _classCallCheck(this, LinkedList); + + this.length = 0; + this.head = null; + this.tail = null; + + if (vals != null) { + vals.forEach(function (v) { + return _this.push(v); + }); + } + } + + _createClass(LinkedList, [{ + key: "size", + value: function size() { + return this.length; + } + }, { + key: "insertBefore", + value: function insertBefore(val, otherNode) { + return add(otherNode.prev, nodeFrom(val), otherNode, this); + } + }, { + key: "insertAfter", + value: function insertAfter(val, otherNode) { + return add(otherNode, nodeFrom(val), otherNode.next, this); + } + }, { + key: "insertNodeBefore", + value: function insertNodeBefore(newNode, otherNode) { + return add(otherNode.prev, newNode, otherNode, this); + } + }, { + key: "insertNodeAfter", + value: function insertNodeAfter(newNode, otherNode) { + return add(otherNode, newNode, otherNode.next, this); + } + }, { + key: "push", + value: function push(val) { + return add(this.tail, nodeFrom(val), null, this); + } + }, { + key: "unshift", + value: function unshift(val) { + return add(null, nodeFrom(val), this.head, this); + } + }, { + key: "remove", + value: function remove(node) { + return _remove(node, this); + } + }, { + key: "pop", + value: function pop() { + return _remove(this.tail, this).value; + } + }, { + key: "popNode", + value: function popNode() { + return _remove(this.tail, this); + } + }, { + key: "shift", + value: function shift() { + return _remove(this.head, this).value; + } + }, { + key: "shiftNode", + value: function shiftNode() { + return _remove(this.head, this); + } + }, { + key: "get_object_at", + value: function get_object_at(index) { + if (index <= this.length()) { + var i = 1; + var current = this.head; + while (i < index) { + current = current.next; + i++; + } + return current.value; + } + } + }, { + key: "set_object_at", + value: function set_object_at(index, value) { + if (index <= this.length()) { + var i = 1; + var current = this.head; + while (i < index) { + current = current.next; + i++; + } + current.value = value; + } + } + }]); + + return LinkedList; +}(); + +module.exports = LinkedList; + +/***/ }), +/* 12 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +/* + *This class is the javascript implementation of the Point.java class in jdk + */ +function Point(x, y, p) { + this.x = null; + this.y = null; + if (x == null && y == null && p == null) { + this.x = 0; + this.y = 0; + } else if (typeof x == 'number' && typeof y == 'number' && p == null) { + this.x = x; + this.y = y; + } else if (x.constructor.name == 'Point' && y == null && p == null) { + p = x; + this.x = p.x; + this.y = p.y; + } +} + +Point.prototype.getX = function () { + return this.x; +}; + +Point.prototype.getY = function () { + return this.y; +}; + +Point.prototype.getLocation = function () { + return new Point(this.x, this.y); +}; + +Point.prototype.setLocation = function (x, y, p) { + if (x.constructor.name == 'Point' && y == null && p == null) { + p = x; + this.setLocation(p.x, p.y); + } else if (typeof x == 'number' && typeof y == 'number' && p == null) { + //if both parameters are integer just move (x,y) location + if (parseInt(x) == x && parseInt(y) == y) { + this.move(x, y); + } else { + this.x = Math.floor(x + 0.5); + this.y = Math.floor(y + 0.5); + } + } +}; + +Point.prototype.move = function (x, y) { + this.x = x; + this.y = y; +}; + +Point.prototype.translate = function (dx, dy) { + this.x += dx; + this.y += dy; +}; + +Point.prototype.equals = function (obj) { + if (obj.constructor.name == "Point") { + var pt = obj; + return this.x == pt.x && this.y == pt.y; + } + return this == obj; +}; + +Point.prototype.toString = function () { + return new Point().constructor.name + "[x=" + this.x + ",y=" + this.y + "]"; +}; + +module.exports = Point; + +/***/ }), +/* 13 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function RectangleD(x, y, width, height) { + this.x = 0; + this.y = 0; + this.width = 0; + this.height = 0; + + if (x != null && y != null && width != null && height != null) { + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } +} + +RectangleD.prototype.getX = function () { + return this.x; +}; + +RectangleD.prototype.setX = function (x) { + this.x = x; +}; + +RectangleD.prototype.getY = function () { + return this.y; +}; + +RectangleD.prototype.setY = function (y) { + this.y = y; +}; + +RectangleD.prototype.getWidth = function () { + return this.width; +}; + +RectangleD.prototype.setWidth = function (width) { + this.width = width; +}; + +RectangleD.prototype.getHeight = function () { + return this.height; +}; + +RectangleD.prototype.setHeight = function (height) { + this.height = height; +}; + +RectangleD.prototype.getRight = function () { + return this.x + this.width; +}; + +RectangleD.prototype.getBottom = function () { + return this.y + this.height; +}; + +RectangleD.prototype.intersects = function (a) { + if (this.getRight() < a.x) { + return false; + } + + if (this.getBottom() < a.y) { + return false; + } + + if (a.getRight() < this.x) { + return false; + } + + if (a.getBottom() < this.y) { + return false; + } + + return true; +}; + +RectangleD.prototype.getCenterX = function () { + return this.x + this.width / 2; +}; + +RectangleD.prototype.getMinX = function () { + return this.getX(); +}; + +RectangleD.prototype.getMaxX = function () { + return this.getX() + this.width; +}; + +RectangleD.prototype.getCenterY = function () { + return this.y + this.height / 2; +}; + +RectangleD.prototype.getMinY = function () { + return this.getY(); +}; + +RectangleD.prototype.getMaxY = function () { + return this.getY() + this.height; +}; + +RectangleD.prototype.getWidthHalf = function () { + return this.width / 2; +}; + +RectangleD.prototype.getHeightHalf = function () { + return this.height / 2; +}; + +module.exports = RectangleD; + +/***/ }), +/* 14 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; + +function UniqueIDGeneretor() {} + +UniqueIDGeneretor.lastID = 0; + +UniqueIDGeneretor.createID = function (obj) { + if (UniqueIDGeneretor.isPrimitive(obj)) { + return obj; + } + if (obj.uniqueID != null) { + return obj.uniqueID; + } + obj.uniqueID = UniqueIDGeneretor.getString(); + UniqueIDGeneretor.lastID++; + return obj.uniqueID; +}; + +UniqueIDGeneretor.getString = function (id) { + if (id == null) id = UniqueIDGeneretor.lastID; + return "Object#" + id + ""; +}; + +UniqueIDGeneretor.isPrimitive = function (arg) { + var type = typeof arg === "undefined" ? "undefined" : _typeof(arg); + return arg == null || type != "object" && type != "function"; +}; + +module.exports = UniqueIDGeneretor; + +/***/ }), +/* 15 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } + +var LayoutConstants = __webpack_require__(0); +var LGraphManager = __webpack_require__(7); +var LNode = __webpack_require__(3); +var LEdge = __webpack_require__(1); +var LGraph = __webpack_require__(6); +var PointD = __webpack_require__(5); +var Transform = __webpack_require__(17); +var Emitter = __webpack_require__(29); + +function Layout(isRemoteUse) { + Emitter.call(this); + + //Layout Quality: 0:draft, 1:default, 2:proof + this.layoutQuality = LayoutConstants.QUALITY; + //Whether layout should create bendpoints as needed or not + this.createBendsAsNeeded = LayoutConstants.DEFAULT_CREATE_BENDS_AS_NEEDED; + //Whether layout should be incremental or not + this.incremental = LayoutConstants.DEFAULT_INCREMENTAL; + //Whether we animate from before to after layout node positions + this.animationOnLayout = LayoutConstants.DEFAULT_ANIMATION_ON_LAYOUT; + //Whether we animate the layout process or not + this.animationDuringLayout = LayoutConstants.DEFAULT_ANIMATION_DURING_LAYOUT; + //Number iterations that should be done between two successive animations + this.animationPeriod = LayoutConstants.DEFAULT_ANIMATION_PERIOD; + /** + * Whether or not leaf nodes (non-compound nodes) are of uniform sizes. When + * they are, both spring and repulsion forces between two leaf nodes can be + * calculated without the expensive clipping point calculations, resulting + * in major speed-up. + */ + this.uniformLeafNodeSizes = LayoutConstants.DEFAULT_UNIFORM_LEAF_NODE_SIZES; + /** + * This is used for creation of bendpoints by using dummy nodes and edges. + * Maps an LEdge to its dummy bendpoint path. + */ + this.edgeToDummyNodes = new Map(); + this.graphManager = new LGraphManager(this); + this.isLayoutFinished = false; + this.isSubLayout = false; + this.isRemoteUse = false; + + if (isRemoteUse != null) { + this.isRemoteUse = isRemoteUse; + } +} + +Layout.RANDOM_SEED = 1; + +Layout.prototype = Object.create(Emitter.prototype); + +Layout.prototype.getGraphManager = function () { + return this.graphManager; +}; + +Layout.prototype.getAllNodes = function () { + return this.graphManager.getAllNodes(); +}; + +Layout.prototype.getAllEdges = function () { + return this.graphManager.getAllEdges(); +}; + +Layout.prototype.getAllNodesToApplyGravitation = function () { + return this.graphManager.getAllNodesToApplyGravitation(); +}; + +Layout.prototype.newGraphManager = function () { + var gm = new LGraphManager(this); + this.graphManager = gm; + return gm; +}; + +Layout.prototype.newGraph = function (vGraph) { + return new LGraph(null, this.graphManager, vGraph); +}; + +Layout.prototype.newNode = function (vNode) { + return new LNode(this.graphManager, vNode); +}; + +Layout.prototype.newEdge = function (vEdge) { + return new LEdge(null, null, vEdge); +}; + +Layout.prototype.checkLayoutSuccess = function () { + return this.graphManager.getRoot() == null || this.graphManager.getRoot().getNodes().length == 0 || this.graphManager.includesInvalidEdge(); +}; + +Layout.prototype.runLayout = function () { + this.isLayoutFinished = false; + + if (this.tilingPreLayout) { + this.tilingPreLayout(); + } + + this.initParameters(); + var isLayoutSuccessfull; + + if (this.checkLayoutSuccess()) { + isLayoutSuccessfull = false; + } else { + isLayoutSuccessfull = this.layout(); + } + + if (LayoutConstants.ANIMATE === 'during') { + // If this is a 'during' layout animation. Layout is not finished yet. + // We need to perform these in index.js when layout is really finished. + return false; + } + + if (isLayoutSuccessfull) { + if (!this.isSubLayout) { + this.doPostLayout(); + } + } + + if (this.tilingPostLayout) { + this.tilingPostLayout(); + } + + this.isLayoutFinished = true; + + return isLayoutSuccessfull; +}; + +/** + * This method performs the operations required after layout. + */ +Layout.prototype.doPostLayout = function () { + //assert !isSubLayout : "Should not be called on sub-layout!"; + // Propagate geometric changes to v-level objects + if (!this.incremental) { + this.transform(); + } + this.update(); +}; + +/** + * This method updates the geometry of the target graph according to + * calculated layout. + */ +Layout.prototype.update2 = function () { + // update bend points + if (this.createBendsAsNeeded) { + this.createBendpointsFromDummyNodes(); + + // reset all edges, since the topology has changed + this.graphManager.resetAllEdges(); + } + + // perform edge, node and root updates if layout is not called + // remotely + if (!this.isRemoteUse) { + // update all edges + var edge; + var allEdges = this.graphManager.getAllEdges(); + for (var i = 0; i < allEdges.length; i++) { + edge = allEdges[i]; + // this.update(edge); + } + + // recursively update nodes + var node; + var nodes = this.graphManager.getRoot().getNodes(); + for (var i = 0; i < nodes.length; i++) { + node = nodes[i]; + // this.update(node); + } + + // update root graph + this.update(this.graphManager.getRoot()); + } +}; + +Layout.prototype.update = function (obj) { + if (obj == null) { + this.update2(); + } else if (obj instanceof LNode) { + var node = obj; + if (node.getChild() != null) { + // since node is compound, recursively update child nodes + var nodes = node.getChild().getNodes(); + for (var i = 0; i < nodes.length; i++) { + update(nodes[i]); + } + } + + // if the l-level node is associated with a v-level graph object, + // then it is assumed that the v-level node implements the + // interface Updatable. + if (node.vGraphObject != null) { + // cast to Updatable without any type check + var vNode = node.vGraphObject; + + // call the update method of the interface + vNode.update(node); + } + } else if (obj instanceof LEdge) { + var edge = obj; + // if the l-level edge is associated with a v-level graph object, + // then it is assumed that the v-level edge implements the + // interface Updatable. + + if (edge.vGraphObject != null) { + // cast to Updatable without any type check + var vEdge = edge.vGraphObject; + + // call the update method of the interface + vEdge.update(edge); + } + } else if (obj instanceof LGraph) { + var graph = obj; + // if the l-level graph is associated with a v-level graph object, + // then it is assumed that the v-level object implements the + // interface Updatable. + + if (graph.vGraphObject != null) { + // cast to Updatable without any type check + var vGraph = graph.vGraphObject; + + // call the update method of the interface + vGraph.update(graph); + } + } +}; + +/** + * This method is used to set all layout parameters to default values + * determined at compile time. + */ +Layout.prototype.initParameters = function () { + if (!this.isSubLayout) { + this.layoutQuality = LayoutConstants.QUALITY; + this.animationDuringLayout = LayoutConstants.DEFAULT_ANIMATION_DURING_LAYOUT; + this.animationPeriod = LayoutConstants.DEFAULT_ANIMATION_PERIOD; + this.animationOnLayout = LayoutConstants.DEFAULT_ANIMATION_ON_LAYOUT; + this.incremental = LayoutConstants.DEFAULT_INCREMENTAL; + this.createBendsAsNeeded = LayoutConstants.DEFAULT_CREATE_BENDS_AS_NEEDED; + this.uniformLeafNodeSizes = LayoutConstants.DEFAULT_UNIFORM_LEAF_NODE_SIZES; + } + + if (this.animationDuringLayout) { + this.animationOnLayout = false; + } +}; + +Layout.prototype.transform = function (newLeftTop) { + if (newLeftTop == undefined) { + this.transform(new PointD(0, 0)); + } else { + // create a transformation object (from Eclipse to layout). When an + // inverse transform is applied, we get upper-left coordinate of the + // drawing or the root graph at given input coordinate (some margins + // already included in calculation of left-top). + + var trans = new Transform(); + var leftTop = this.graphManager.getRoot().updateLeftTop(); + + if (leftTop != null) { + trans.setWorldOrgX(newLeftTop.x); + trans.setWorldOrgY(newLeftTop.y); + + trans.setDeviceOrgX(leftTop.x); + trans.setDeviceOrgY(leftTop.y); + + var nodes = this.getAllNodes(); + var node; + + for (var i = 0; i < nodes.length; i++) { + node = nodes[i]; + node.transform(trans); + } + } + } +}; + +Layout.prototype.positionNodesRandomly = function (graph) { + + if (graph == undefined) { + //assert !this.incremental; + this.positionNodesRandomly(this.getGraphManager().getRoot()); + this.getGraphManager().getRoot().updateBounds(true); + } else { + var lNode; + var childGraph; + + var nodes = graph.getNodes(); + for (var i = 0; i < nodes.length; i++) { + lNode = nodes[i]; + childGraph = lNode.getChild(); + + if (childGraph == null) { + lNode.scatter(); + } else if (childGraph.getNodes().length == 0) { + lNode.scatter(); + } else { + this.positionNodesRandomly(childGraph); + lNode.updateBounds(); + } + } + } +}; + +/** + * This method returns a list of trees where each tree is represented as a + * list of l-nodes. The method returns a list of size 0 when: + * - The graph is not flat or + * - One of the component(s) of the graph is not a tree. + */ +Layout.prototype.getFlatForest = function () { + var flatForest = []; + var isForest = true; + + // Quick reference for all nodes in the graph manager associated with + // this layout. The list should not be changed. + var allNodes = this.graphManager.getRoot().getNodes(); + + // First be sure that the graph is flat + var isFlat = true; + + for (var i = 0; i < allNodes.length; i++) { + if (allNodes[i].getChild() != null) { + isFlat = false; + } + } + + // Return empty forest if the graph is not flat. + if (!isFlat) { + return flatForest; + } + + // Run BFS for each component of the graph. + + var visited = new Set(); + var toBeVisited = []; + var parents = new Map(); + var unProcessedNodes = []; + + unProcessedNodes = unProcessedNodes.concat(allNodes); + + // Each iteration of this loop finds a component of the graph and + // decides whether it is a tree or not. If it is a tree, adds it to the + // forest and continued with the next component. + + while (unProcessedNodes.length > 0 && isForest) { + toBeVisited.push(unProcessedNodes[0]); + + // Start the BFS. Each iteration of this loop visits a node in a + // BFS manner. + while (toBeVisited.length > 0 && isForest) { + //pool operation + var currentNode = toBeVisited[0]; + toBeVisited.splice(0, 1); + visited.add(currentNode); + + // Traverse all neighbors of this node + var neighborEdges = currentNode.getEdges(); + + for (var i = 0; i < neighborEdges.length; i++) { + var currentNeighbor = neighborEdges[i].getOtherEnd(currentNode); + + // If BFS is not growing from this neighbor. + if (parents.get(currentNode) != currentNeighbor) { + // We haven't previously visited this neighbor. + if (!visited.has(currentNeighbor)) { + toBeVisited.push(currentNeighbor); + parents.set(currentNeighbor, currentNode); + } + // Since we have previously visited this neighbor and + // this neighbor is not parent of currentNode, given + // graph contains a component that is not tree, hence + // it is not a forest. + else { + isForest = false; + break; + } + } + } + } + + // The graph contains a component that is not a tree. Empty + // previously found trees. The method will end. + if (!isForest) { + flatForest = []; + } + // Save currently visited nodes as a tree in our forest. Reset + // visited and parents lists. Continue with the next component of + // the graph, if any. + else { + var temp = [].concat(_toConsumableArray(visited)); + flatForest.push(temp); + //flatForest = flatForest.concat(temp); + //unProcessedNodes.removeAll(visited); + for (var i = 0; i < temp.length; i++) { + var value = temp[i]; + var index = unProcessedNodes.indexOf(value); + if (index > -1) { + unProcessedNodes.splice(index, 1); + } + } + visited = new Set(); + parents = new Map(); + } + } + + return flatForest; +}; + +/** + * This method creates dummy nodes (an l-level node with minimal dimensions) + * for the given edge (one per bendpoint). The existing l-level structure + * is updated accordingly. + */ +Layout.prototype.createDummyNodesForBendpoints = function (edge) { + var dummyNodes = []; + var prev = edge.source; + + var graph = this.graphManager.calcLowestCommonAncestor(edge.source, edge.target); + + for (var i = 0; i < edge.bendpoints.length; i++) { + // create new dummy node + var dummyNode = this.newNode(null); + dummyNode.setRect(new Point(0, 0), new Dimension(1, 1)); + + graph.add(dummyNode); + + // create new dummy edge between prev and dummy node + var dummyEdge = this.newEdge(null); + this.graphManager.add(dummyEdge, prev, dummyNode); + + dummyNodes.add(dummyNode); + prev = dummyNode; + } + + var dummyEdge = this.newEdge(null); + this.graphManager.add(dummyEdge, prev, edge.target); + + this.edgeToDummyNodes.set(edge, dummyNodes); + + // remove real edge from graph manager if it is inter-graph + if (edge.isInterGraph()) { + this.graphManager.remove(edge); + } + // else, remove the edge from the current graph + else { + graph.remove(edge); + } + + return dummyNodes; +}; + +/** + * This method creates bendpoints for edges from the dummy nodes + * at l-level. + */ +Layout.prototype.createBendpointsFromDummyNodes = function () { + var edges = []; + edges = edges.concat(this.graphManager.getAllEdges()); + edges = [].concat(_toConsumableArray(this.edgeToDummyNodes.keys())).concat(edges); + + for (var k = 0; k < edges.length; k++) { + var lEdge = edges[k]; + + if (lEdge.bendpoints.length > 0) { + var path = this.edgeToDummyNodes.get(lEdge); + + for (var i = 0; i < path.length; i++) { + var dummyNode = path[i]; + var p = new PointD(dummyNode.getCenterX(), dummyNode.getCenterY()); + + // update bendpoint's location according to dummy node + var ebp = lEdge.bendpoints.get(i); + ebp.x = p.x; + ebp.y = p.y; + + // remove the dummy node, dummy edges incident with this + // dummy node is also removed (within the remove method) + dummyNode.getOwner().remove(dummyNode); + } + + // add the real edge to graph + this.graphManager.add(lEdge, lEdge.source, lEdge.target); + } + } +}; + +Layout.transform = function (sliderValue, defaultValue, minDiv, maxMul) { + if (minDiv != undefined && maxMul != undefined) { + var value = defaultValue; + + if (sliderValue <= 50) { + var minValue = defaultValue / minDiv; + value -= (defaultValue - minValue) / 50 * (50 - sliderValue); + } else { + var maxValue = defaultValue * maxMul; + value += (maxValue - defaultValue) / 50 * (sliderValue - 50); + } + + return value; + } else { + var a, b; + + if (sliderValue <= 50) { + a = 9.0 * defaultValue / 500.0; + b = defaultValue / 10.0; + } else { + a = 9.0 * defaultValue / 50.0; + b = -8 * defaultValue; + } + + return a * sliderValue + b; + } +}; + +/** + * This method finds and returns the center of the given nodes, assuming + * that the given nodes form a tree in themselves. + */ +Layout.findCenterOfTree = function (nodes) { + var list = []; + list = list.concat(nodes); + + var removedNodes = []; + var remainingDegrees = new Map(); + var foundCenter = false; + var centerNode = null; + + if (list.length == 1 || list.length == 2) { + foundCenter = true; + centerNode = list[0]; + } + + for (var i = 0; i < list.length; i++) { + var node = list[i]; + var degree = node.getNeighborsList().size; + remainingDegrees.set(node, node.getNeighborsList().size); + + if (degree == 1) { + removedNodes.push(node); + } + } + + var tempList = []; + tempList = tempList.concat(removedNodes); + + while (!foundCenter) { + var tempList2 = []; + tempList2 = tempList2.concat(tempList); + tempList = []; + + for (var i = 0; i < list.length; i++) { + var node = list[i]; + + var index = list.indexOf(node); + if (index >= 0) { + list.splice(index, 1); + } + + var neighbours = node.getNeighborsList(); + + neighbours.forEach(function (neighbour) { + if (removedNodes.indexOf(neighbour) < 0) { + var otherDegree = remainingDegrees.get(neighbour); + var newDegree = otherDegree - 1; + + if (newDegree == 1) { + tempList.push(neighbour); + } + + remainingDegrees.set(neighbour, newDegree); + } + }); + } + + removedNodes = removedNodes.concat(tempList); + + if (list.length == 1 || list.length == 2) { + foundCenter = true; + centerNode = list[0]; + } + } + + return centerNode; +}; + +/** + * During the coarsening process, this layout may be referenced by two graph managers + * this setter function grants access to change the currently being used graph manager + */ +Layout.prototype.setGraphManager = function (gm) { + this.graphManager = gm; +}; + +module.exports = Layout; + +/***/ }), +/* 16 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function RandomSeed() {} +// adapted from: https://stackoverflow.com/a/19303725 +RandomSeed.seed = 1; +RandomSeed.x = 0; + +RandomSeed.nextDouble = function () { + RandomSeed.x = Math.sin(RandomSeed.seed++) * 10000; + return RandomSeed.x - Math.floor(RandomSeed.x); +}; + +module.exports = RandomSeed; + +/***/ }), +/* 17 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var PointD = __webpack_require__(5); + +function Transform(x, y) { + this.lworldOrgX = 0.0; + this.lworldOrgY = 0.0; + this.ldeviceOrgX = 0.0; + this.ldeviceOrgY = 0.0; + this.lworldExtX = 1.0; + this.lworldExtY = 1.0; + this.ldeviceExtX = 1.0; + this.ldeviceExtY = 1.0; +} + +Transform.prototype.getWorldOrgX = function () { + return this.lworldOrgX; +}; + +Transform.prototype.setWorldOrgX = function (wox) { + this.lworldOrgX = wox; +}; + +Transform.prototype.getWorldOrgY = function () { + return this.lworldOrgY; +}; + +Transform.prototype.setWorldOrgY = function (woy) { + this.lworldOrgY = woy; +}; + +Transform.prototype.getWorldExtX = function () { + return this.lworldExtX; +}; + +Transform.prototype.setWorldExtX = function (wex) { + this.lworldExtX = wex; +}; + +Transform.prototype.getWorldExtY = function () { + return this.lworldExtY; +}; + +Transform.prototype.setWorldExtY = function (wey) { + this.lworldExtY = wey; +}; + +/* Device related */ + +Transform.prototype.getDeviceOrgX = function () { + return this.ldeviceOrgX; +}; + +Transform.prototype.setDeviceOrgX = function (dox) { + this.ldeviceOrgX = dox; +}; + +Transform.prototype.getDeviceOrgY = function () { + return this.ldeviceOrgY; +}; + +Transform.prototype.setDeviceOrgY = function (doy) { + this.ldeviceOrgY = doy; +}; + +Transform.prototype.getDeviceExtX = function () { + return this.ldeviceExtX; +}; + +Transform.prototype.setDeviceExtX = function (dex) { + this.ldeviceExtX = dex; +}; + +Transform.prototype.getDeviceExtY = function () { + return this.ldeviceExtY; +}; + +Transform.prototype.setDeviceExtY = function (dey) { + this.ldeviceExtY = dey; +}; + +Transform.prototype.transformX = function (x) { + var xDevice = 0.0; + var worldExtX = this.lworldExtX; + if (worldExtX != 0.0) { + xDevice = this.ldeviceOrgX + (x - this.lworldOrgX) * this.ldeviceExtX / worldExtX; + } + + return xDevice; +}; + +Transform.prototype.transformY = function (y) { + var yDevice = 0.0; + var worldExtY = this.lworldExtY; + if (worldExtY != 0.0) { + yDevice = this.ldeviceOrgY + (y - this.lworldOrgY) * this.ldeviceExtY / worldExtY; + } + + return yDevice; +}; + +Transform.prototype.inverseTransformX = function (x) { + var xWorld = 0.0; + var deviceExtX = this.ldeviceExtX; + if (deviceExtX != 0.0) { + xWorld = this.lworldOrgX + (x - this.ldeviceOrgX) * this.lworldExtX / deviceExtX; + } + + return xWorld; +}; + +Transform.prototype.inverseTransformY = function (y) { + var yWorld = 0.0; + var deviceExtY = this.ldeviceExtY; + if (deviceExtY != 0.0) { + yWorld = this.lworldOrgY + (y - this.ldeviceOrgY) * this.lworldExtY / deviceExtY; + } + return yWorld; +}; + +Transform.prototype.inverseTransformPoint = function (inPoint) { + var outPoint = new PointD(this.inverseTransformX(inPoint.x), this.inverseTransformY(inPoint.y)); + return outPoint; +}; + +module.exports = Transform; + +/***/ }), +/* 18 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } + +var Layout = __webpack_require__(15); +var FDLayoutConstants = __webpack_require__(4); +var LayoutConstants = __webpack_require__(0); +var IGeometry = __webpack_require__(8); +var IMath = __webpack_require__(9); + +function FDLayout() { + Layout.call(this); + + this.useSmartIdealEdgeLengthCalculation = FDLayoutConstants.DEFAULT_USE_SMART_IDEAL_EDGE_LENGTH_CALCULATION; + this.gravityConstant = FDLayoutConstants.DEFAULT_GRAVITY_STRENGTH; + this.compoundGravityConstant = FDLayoutConstants.DEFAULT_COMPOUND_GRAVITY_STRENGTH; + this.gravityRangeFactor = FDLayoutConstants.DEFAULT_GRAVITY_RANGE_FACTOR; + this.compoundGravityRangeFactor = FDLayoutConstants.DEFAULT_COMPOUND_GRAVITY_RANGE_FACTOR; + this.displacementThresholdPerNode = 3.0 * FDLayoutConstants.DEFAULT_EDGE_LENGTH / 100; + this.coolingFactor = FDLayoutConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL; + this.initialCoolingFactor = FDLayoutConstants.DEFAULT_COOLING_FACTOR_INCREMENTAL; + this.totalDisplacement = 0.0; + this.oldTotalDisplacement = 0.0; + this.maxIterations = FDLayoutConstants.MAX_ITERATIONS; +} + +FDLayout.prototype = Object.create(Layout.prototype); + +for (var prop in Layout) { + FDLayout[prop] = Layout[prop]; +} + +FDLayout.prototype.initParameters = function () { + Layout.prototype.initParameters.call(this, arguments); + + this.totalIterations = 0; + this.notAnimatedIterations = 0; + + this.useFRGridVariant = FDLayoutConstants.DEFAULT_USE_SMART_REPULSION_RANGE_CALCULATION; + + this.grid = []; +}; + +FDLayout.prototype.calcIdealEdgeLengths = function () { + var edge; + var originalIdealLength; + var lcaDepth; + var source; + var target; + var sizeOfSourceInLca; + var sizeOfTargetInLca; + + var allEdges = this.getGraphManager().getAllEdges(); + for (var i = 0; i < allEdges.length; i++) { + edge = allEdges[i]; + + originalIdealLength = edge.idealLength; + + if (edge.isInterGraph) { + source = edge.getSource(); + target = edge.getTarget(); + + sizeOfSourceInLca = edge.getSourceInLca().getEstimatedSize(); + sizeOfTargetInLca = edge.getTargetInLca().getEstimatedSize(); + + if (this.useSmartIdealEdgeLengthCalculation) { + edge.idealLength += sizeOfSourceInLca + sizeOfTargetInLca - 2 * LayoutConstants.SIMPLE_NODE_SIZE; + } + + lcaDepth = edge.getLca().getInclusionTreeDepth(); + + edge.idealLength += originalIdealLength * FDLayoutConstants.PER_LEVEL_IDEAL_EDGE_LENGTH_FACTOR * (source.getInclusionTreeDepth() + target.getInclusionTreeDepth() - 2 * lcaDepth); + } + } +}; + +FDLayout.prototype.initSpringEmbedder = function () { + + var s = this.getAllNodes().length; + if (this.incremental) { + if (s > FDLayoutConstants.ADAPTATION_LOWER_NODE_LIMIT) { + this.coolingFactor = Math.max(this.coolingFactor * FDLayoutConstants.COOLING_ADAPTATION_FACTOR, this.coolingFactor - (s - FDLayoutConstants.ADAPTATION_LOWER_NODE_LIMIT) / (FDLayoutConstants.ADAPTATION_UPPER_NODE_LIMIT - FDLayoutConstants.ADAPTATION_LOWER_NODE_LIMIT) * this.coolingFactor * (1 - FDLayoutConstants.COOLING_ADAPTATION_FACTOR)); + } + this.maxNodeDisplacement = FDLayoutConstants.MAX_NODE_DISPLACEMENT_INCREMENTAL; + } else { + if (s > FDLayoutConstants.ADAPTATION_LOWER_NODE_LIMIT) { + this.coolingFactor = Math.max(FDLayoutConstants.COOLING_ADAPTATION_FACTOR, 1.0 - (s - FDLayoutConstants.ADAPTATION_LOWER_NODE_LIMIT) / (FDLayoutConstants.ADAPTATION_UPPER_NODE_LIMIT - FDLayoutConstants.ADAPTATION_LOWER_NODE_LIMIT) * (1 - FDLayoutConstants.COOLING_ADAPTATION_FACTOR)); + } else { + this.coolingFactor = 1.0; + } + this.initialCoolingFactor = this.coolingFactor; + this.maxNodeDisplacement = FDLayoutConstants.MAX_NODE_DISPLACEMENT; + } + + this.maxIterations = Math.max(this.getAllNodes().length * 5, this.maxIterations); + + // Reassign this attribute by using new constant value + this.displacementThresholdPerNode = 3.0 * FDLayoutConstants.DEFAULT_EDGE_LENGTH / 100; + this.totalDisplacementThreshold = this.displacementThresholdPerNode * this.getAllNodes().length; + + this.repulsionRange = this.calcRepulsionRange(); +}; + +FDLayout.prototype.calcSpringForces = function () { + var lEdges = this.getAllEdges(); + var edge; + + for (var i = 0; i < lEdges.length; i++) { + edge = lEdges[i]; + + this.calcSpringForce(edge, edge.idealLength); + } +}; + +FDLayout.prototype.calcRepulsionForces = function () { + var gridUpdateAllowed = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : true; + var forceToNodeSurroundingUpdate = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false; + + var i, j; + var nodeA, nodeB; + var lNodes = this.getAllNodes(); + var processedNodeSet; + + if (this.useFRGridVariant) { + if (this.totalIterations % FDLayoutConstants.GRID_CALCULATION_CHECK_PERIOD == 1 && gridUpdateAllowed) { + this.updateGrid(); + } + + processedNodeSet = new Set(); + + // calculate repulsion forces between each nodes and its surrounding + for (i = 0; i < lNodes.length; i++) { + nodeA = lNodes[i]; + this.calculateRepulsionForceOfANode(nodeA, processedNodeSet, gridUpdateAllowed, forceToNodeSurroundingUpdate); + processedNodeSet.add(nodeA); + } + } else { + for (i = 0; i < lNodes.length; i++) { + nodeA = lNodes[i]; + + for (j = i + 1; j < lNodes.length; j++) { + nodeB = lNodes[j]; + + // If both nodes are not members of the same graph, skip. + if (nodeA.getOwner() != nodeB.getOwner()) { + continue; + } + + this.calcRepulsionForce(nodeA, nodeB); + } + } + } +}; + +FDLayout.prototype.calcGravitationalForces = function () { + var node; + var lNodes = this.getAllNodesToApplyGravitation(); + + for (var i = 0; i < lNodes.length; i++) { + node = lNodes[i]; + this.calcGravitationalForce(node); + } +}; + +FDLayout.prototype.moveNodes = function () { + var lNodes = this.getAllNodes(); + var node; + + for (var i = 0; i < lNodes.length; i++) { + node = lNodes[i]; + node.move(); + } +}; + +FDLayout.prototype.calcSpringForce = function (edge, idealLength) { + var sourceNode = edge.getSource(); + var targetNode = edge.getTarget(); + + var length; + var springForce; + var springForceX; + var springForceY; + + // Update edge length + if (this.uniformLeafNodeSizes && sourceNode.getChild() == null && targetNode.getChild() == null) { + edge.updateLengthSimple(); + } else { + edge.updateLength(); + + if (edge.isOverlapingSourceAndTarget) { + return; + } + } + + length = edge.getLength(); + + if (length == 0) return; + + // Calculate spring forces + springForce = edge.edgeElasticity * (length - idealLength); + + // Project force onto x and y axes + springForceX = springForce * (edge.lengthX / length); + springForceY = springForce * (edge.lengthY / length); + + // Apply forces on the end nodes + sourceNode.springForceX += springForceX; + sourceNode.springForceY += springForceY; + targetNode.springForceX -= springForceX; + targetNode.springForceY -= springForceY; +}; + +FDLayout.prototype.calcRepulsionForce = function (nodeA, nodeB) { + var rectA = nodeA.getRect(); + var rectB = nodeB.getRect(); + var overlapAmount = new Array(2); + var clipPoints = new Array(4); + var distanceX; + var distanceY; + var distanceSquared; + var distance; + var repulsionForce; + var repulsionForceX; + var repulsionForceY; + + if (rectA.intersects(rectB)) // two nodes overlap + { + // calculate separation amount in x and y directions + IGeometry.calcSeparationAmount(rectA, rectB, overlapAmount, FDLayoutConstants.DEFAULT_EDGE_LENGTH / 2.0); + + repulsionForceX = 2 * overlapAmount[0]; + repulsionForceY = 2 * overlapAmount[1]; + + var childrenConstant = nodeA.noOfChildren * nodeB.noOfChildren / (nodeA.noOfChildren + nodeB.noOfChildren); + + // Apply forces on the two nodes + nodeA.repulsionForceX -= childrenConstant * repulsionForceX; + nodeA.repulsionForceY -= childrenConstant * repulsionForceY; + nodeB.repulsionForceX += childrenConstant * repulsionForceX; + nodeB.repulsionForceY += childrenConstant * repulsionForceY; + } else // no overlap + { + // calculate distance + + if (this.uniformLeafNodeSizes && nodeA.getChild() == null && nodeB.getChild() == null) // simply base repulsion on distance of node centers + { + distanceX = rectB.getCenterX() - rectA.getCenterX(); + distanceY = rectB.getCenterY() - rectA.getCenterY(); + } else // use clipping points + { + IGeometry.getIntersection(rectA, rectB, clipPoints); + + distanceX = clipPoints[2] - clipPoints[0]; + distanceY = clipPoints[3] - clipPoints[1]; + } + + // No repulsion range. FR grid variant should take care of this. + if (Math.abs(distanceX) < FDLayoutConstants.MIN_REPULSION_DIST) { + distanceX = IMath.sign(distanceX) * FDLayoutConstants.MIN_REPULSION_DIST; + } + + if (Math.abs(distanceY) < FDLayoutConstants.MIN_REPULSION_DIST) { + distanceY = IMath.sign(distanceY) * FDLayoutConstants.MIN_REPULSION_DIST; + } + + distanceSquared = distanceX * distanceX + distanceY * distanceY; + distance = Math.sqrt(distanceSquared); + + // Here we use half of the nodes' repulsion values for backward compatibility + repulsionForce = (nodeA.nodeRepulsion / 2 + nodeB.nodeRepulsion / 2) * nodeA.noOfChildren * nodeB.noOfChildren / distanceSquared; + + // Project force onto x and y axes + repulsionForceX = repulsionForce * distanceX / distance; + repulsionForceY = repulsionForce * distanceY / distance; + + // Apply forces on the two nodes + nodeA.repulsionForceX -= repulsionForceX; + nodeA.repulsionForceY -= repulsionForceY; + nodeB.repulsionForceX += repulsionForceX; + nodeB.repulsionForceY += repulsionForceY; + } +}; + +FDLayout.prototype.calcGravitationalForce = function (node) { + var ownerGraph; + var ownerCenterX; + var ownerCenterY; + var distanceX; + var distanceY; + var absDistanceX; + var absDistanceY; + var estimatedSize; + ownerGraph = node.getOwner(); + + ownerCenterX = (ownerGraph.getRight() + ownerGraph.getLeft()) / 2; + ownerCenterY = (ownerGraph.getTop() + ownerGraph.getBottom()) / 2; + distanceX = node.getCenterX() - ownerCenterX; + distanceY = node.getCenterY() - ownerCenterY; + absDistanceX = Math.abs(distanceX) + node.getWidth() / 2; + absDistanceY = Math.abs(distanceY) + node.getHeight() / 2; + + if (node.getOwner() == this.graphManager.getRoot()) // in the root graph + { + estimatedSize = ownerGraph.getEstimatedSize() * this.gravityRangeFactor; + + if (absDistanceX > estimatedSize || absDistanceY > estimatedSize) { + node.gravitationForceX = -this.gravityConstant * distanceX; + node.gravitationForceY = -this.gravityConstant * distanceY; + } + } else // inside a compound + { + estimatedSize = ownerGraph.getEstimatedSize() * this.compoundGravityRangeFactor; + + if (absDistanceX > estimatedSize || absDistanceY > estimatedSize) { + node.gravitationForceX = -this.gravityConstant * distanceX * this.compoundGravityConstant; + node.gravitationForceY = -this.gravityConstant * distanceY * this.compoundGravityConstant; + } + } +}; + +FDLayout.prototype.isConverged = function () { + var converged; + var oscilating = false; + + if (this.totalIterations > this.maxIterations / 3) { + oscilating = Math.abs(this.totalDisplacement - this.oldTotalDisplacement) < 2; + } + + converged = this.totalDisplacement < this.totalDisplacementThreshold; + + this.oldTotalDisplacement = this.totalDisplacement; + + return converged || oscilating; +}; + +FDLayout.prototype.animate = function () { + if (this.animationDuringLayout && !this.isSubLayout) { + if (this.notAnimatedIterations == this.animationPeriod) { + this.update(); + this.notAnimatedIterations = 0; + } else { + this.notAnimatedIterations++; + } + } +}; + +//This method calculates the number of children (weight) for all nodes +FDLayout.prototype.calcNoOfChildrenForAllNodes = function () { + var node; + var allNodes = this.graphManager.getAllNodes(); + + for (var i = 0; i < allNodes.length; i++) { + node = allNodes[i]; + node.noOfChildren = node.getNoOfChildren(); + } +}; + +// ----------------------------------------------------------------------------- +// Section: FR-Grid Variant Repulsion Force Calculation +// ----------------------------------------------------------------------------- + +FDLayout.prototype.calcGrid = function (graph) { + + var sizeX = 0; + var sizeY = 0; + + sizeX = parseInt(Math.ceil((graph.getRight() - graph.getLeft()) / this.repulsionRange)); + sizeY = parseInt(Math.ceil((graph.getBottom() - graph.getTop()) / this.repulsionRange)); + + var grid = new Array(sizeX); + + for (var i = 0; i < sizeX; i++) { + grid[i] = new Array(sizeY); + } + + for (var i = 0; i < sizeX; i++) { + for (var j = 0; j < sizeY; j++) { + grid[i][j] = new Array(); + } + } + + return grid; +}; + +FDLayout.prototype.addNodeToGrid = function (v, left, top) { + + var startX = 0; + var finishX = 0; + var startY = 0; + var finishY = 0; + + startX = parseInt(Math.floor((v.getRect().x - left) / this.repulsionRange)); + finishX = parseInt(Math.floor((v.getRect().width + v.getRect().x - left) / this.repulsionRange)); + startY = parseInt(Math.floor((v.getRect().y - top) / this.repulsionRange)); + finishY = parseInt(Math.floor((v.getRect().height + v.getRect().y - top) / this.repulsionRange)); + + for (var i = startX; i <= finishX; i++) { + for (var j = startY; j <= finishY; j++) { + this.grid[i][j].push(v); + v.setGridCoordinates(startX, finishX, startY, finishY); + } + } +}; + +FDLayout.prototype.updateGrid = function () { + var i; + var nodeA; + var lNodes = this.getAllNodes(); + + this.grid = this.calcGrid(this.graphManager.getRoot()); + + // put all nodes to proper grid cells + for (i = 0; i < lNodes.length; i++) { + nodeA = lNodes[i]; + this.addNodeToGrid(nodeA, this.graphManager.getRoot().getLeft(), this.graphManager.getRoot().getTop()); + } +}; + +FDLayout.prototype.calculateRepulsionForceOfANode = function (nodeA, processedNodeSet, gridUpdateAllowed, forceToNodeSurroundingUpdate) { + + if (this.totalIterations % FDLayoutConstants.GRID_CALCULATION_CHECK_PERIOD == 1 && gridUpdateAllowed || forceToNodeSurroundingUpdate) { + var surrounding = new Set(); + nodeA.surrounding = new Array(); + var nodeB; + var grid = this.grid; + + for (var i = nodeA.startX - 1; i < nodeA.finishX + 2; i++) { + for (var j = nodeA.startY - 1; j < nodeA.finishY + 2; j++) { + if (!(i < 0 || j < 0 || i >= grid.length || j >= grid[0].length)) { + for (var k = 0; k < grid[i][j].length; k++) { + nodeB = grid[i][j][k]; + + // If both nodes are not members of the same graph, + // or both nodes are the same, skip. + if (nodeA.getOwner() != nodeB.getOwner() || nodeA == nodeB) { + continue; + } + + // check if the repulsion force between + // nodeA and nodeB has already been calculated + if (!processedNodeSet.has(nodeB) && !surrounding.has(nodeB)) { + var distanceX = Math.abs(nodeA.getCenterX() - nodeB.getCenterX()) - (nodeA.getWidth() / 2 + nodeB.getWidth() / 2); + var distanceY = Math.abs(nodeA.getCenterY() - nodeB.getCenterY()) - (nodeA.getHeight() / 2 + nodeB.getHeight() / 2); + + // if the distance between nodeA and nodeB + // is less then calculation range + if (distanceX <= this.repulsionRange && distanceY <= this.repulsionRange) { + //then add nodeB to surrounding of nodeA + surrounding.add(nodeB); + } + } + } + } + } + } + + nodeA.surrounding = [].concat(_toConsumableArray(surrounding)); + } + for (i = 0; i < nodeA.surrounding.length; i++) { + this.calcRepulsionForce(nodeA, nodeA.surrounding[i]); + } +}; + +FDLayout.prototype.calcRepulsionRange = function () { + return 0.0; +}; + +module.exports = FDLayout; + +/***/ }), +/* 19 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var LEdge = __webpack_require__(1); +var FDLayoutConstants = __webpack_require__(4); + +function FDLayoutEdge(source, target, vEdge) { + LEdge.call(this, source, target, vEdge); + + // Ideal length and elasticity value for this edge + this.idealLength = FDLayoutConstants.DEFAULT_EDGE_LENGTH; + this.edgeElasticity = FDLayoutConstants.DEFAULT_SPRING_STRENGTH; +} + +FDLayoutEdge.prototype = Object.create(LEdge.prototype); + +for (var prop in LEdge) { + FDLayoutEdge[prop] = LEdge[prop]; +} + +module.exports = FDLayoutEdge; + +/***/ }), +/* 20 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var LNode = __webpack_require__(3); +var FDLayoutConstants = __webpack_require__(4); + +function FDLayoutNode(gm, loc, size, vNode) { + // alternative constructor is handled inside LNode + LNode.call(this, gm, loc, size, vNode); + + // Repulsion value of this node + this.nodeRepulsion = FDLayoutConstants.DEFAULT_REPULSION_STRENGTH; + + //Spring, repulsion and gravitational forces acting on this node + this.springForceX = 0; + this.springForceY = 0; + this.repulsionForceX = 0; + this.repulsionForceY = 0; + this.gravitationForceX = 0; + this.gravitationForceY = 0; + //Amount by which this node is to be moved in this iteration + this.displacementX = 0; + this.displacementY = 0; + + //Start and finish grid coordinates that this node is fallen into + this.startX = 0; + this.finishX = 0; + this.startY = 0; + this.finishY = 0; + + //Geometric neighbors of this node + this.surrounding = []; +} + +FDLayoutNode.prototype = Object.create(LNode.prototype); + +for (var prop in LNode) { + FDLayoutNode[prop] = LNode[prop]; +} + +FDLayoutNode.prototype.setGridCoordinates = function (_startX, _finishX, _startY, _finishY) { + this.startX = _startX; + this.finishX = _finishX; + this.startY = _startY; + this.finishY = _finishY; +}; + +module.exports = FDLayoutNode; + +/***/ }), +/* 21 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function DimensionD(width, height) { + this.width = 0; + this.height = 0; + if (width !== null && height !== null) { + this.height = height; + this.width = width; + } +} + +DimensionD.prototype.getWidth = function () { + return this.width; +}; + +DimensionD.prototype.setWidth = function (width) { + this.width = width; +}; + +DimensionD.prototype.getHeight = function () { + return this.height; +}; + +DimensionD.prototype.setHeight = function (height) { + this.height = height; +}; + +module.exports = DimensionD; + +/***/ }), +/* 22 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var UniqueIDGeneretor = __webpack_require__(14); + +function HashMap() { + this.map = {}; + this.keys = []; +} + +HashMap.prototype.put = function (key, value) { + var theId = UniqueIDGeneretor.createID(key); + if (!this.contains(theId)) { + this.map[theId] = value; + this.keys.push(key); + } +}; + +HashMap.prototype.contains = function (key) { + var theId = UniqueIDGeneretor.createID(key); + return this.map[key] != null; +}; + +HashMap.prototype.get = function (key) { + var theId = UniqueIDGeneretor.createID(key); + return this.map[theId]; +}; + +HashMap.prototype.keySet = function () { + return this.keys; +}; + +module.exports = HashMap; + +/***/ }), +/* 23 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var UniqueIDGeneretor = __webpack_require__(14); + +function HashSet() { + this.set = {}; +} +; + +HashSet.prototype.add = function (obj) { + var theId = UniqueIDGeneretor.createID(obj); + if (!this.contains(theId)) this.set[theId] = obj; +}; + +HashSet.prototype.remove = function (obj) { + delete this.set[UniqueIDGeneretor.createID(obj)]; +}; + +HashSet.prototype.clear = function () { + this.set = {}; +}; + +HashSet.prototype.contains = function (obj) { + return this.set[UniqueIDGeneretor.createID(obj)] == obj; +}; + +HashSet.prototype.isEmpty = function () { + return this.size() === 0; +}; + +HashSet.prototype.size = function () { + return Object.keys(this.set).length; +}; + +//concats this.set to the given list +HashSet.prototype.addAllTo = function (list) { + var keys = Object.keys(this.set); + var length = keys.length; + for (var i = 0; i < length; i++) { + list.push(this.set[keys[i]]); + } +}; + +HashSet.prototype.size = function () { + return Object.keys(this.set).length; +}; + +HashSet.prototype.addAll = function (list) { + var s = list.length; + for (var i = 0; i < s; i++) { + var v = list[i]; + this.add(v); + } +}; + +module.exports = HashSet; + +/***/ }), +/* 24 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +// Some matrix (1d and 2d array) operations +function Matrix() {} + +/** + * matrix multiplication + * array1, array2 and result are 2d arrays + */ +Matrix.multMat = function (array1, array2) { + var result = []; + + for (var i = 0; i < array1.length; i++) { + result[i] = []; + for (var j = 0; j < array2[0].length; j++) { + result[i][j] = 0; + for (var k = 0; k < array1[0].length; k++) { + result[i][j] += array1[i][k] * array2[k][j]; + } + } + } + return result; +}; + +/** + * matrix transpose + * array and result are 2d arrays + */ +Matrix.transpose = function (array) { + var result = []; + + for (var i = 0; i < array[0].length; i++) { + result[i] = []; + for (var j = 0; j < array.length; j++) { + result[i][j] = array[j][i]; + } + } + + return result; +}; + +/** + * multiply array with constant + * array and result are 1d arrays + */ +Matrix.multCons = function (array, constant) { + var result = []; + + for (var i = 0; i < array.length; i++) { + result[i] = array[i] * constant; + } + + return result; +}; + +/** + * substract two arrays + * array1, array2 and result are 1d arrays + */ +Matrix.minusOp = function (array1, array2) { + var result = []; + + for (var i = 0; i < array1.length; i++) { + result[i] = array1[i] - array2[i]; + } + + return result; +}; + +/** + * dot product of two arrays with same size + * array1 and array2 are 1d arrays + */ +Matrix.dotProduct = function (array1, array2) { + var product = 0; + + for (var i = 0; i < array1.length; i++) { + product += array1[i] * array2[i]; + } + + return product; +}; + +/** + * magnitude of an array + * array is 1d array + */ +Matrix.mag = function (array) { + return Math.sqrt(this.dotProduct(array, array)); +}; + +/** + * normalization of an array + * array and result are 1d array + */ +Matrix.normalize = function (array) { + var result = []; + var magnitude = this.mag(array); + + for (var i = 0; i < array.length; i++) { + result[i] = array[i] / magnitude; + } + + return result; +}; + +/** + * multiply an array with centering matrix + * array and result are 1d array + */ +Matrix.multGamma = function (array) { + var result = []; + var sum = 0; + + for (var i = 0; i < array.length; i++) { + sum += array[i]; + } + + sum *= -1 / array.length; + + for (var _i = 0; _i < array.length; _i++) { + result[_i] = sum + array[_i]; + } + return result; +}; + +/** + * a special matrix multiplication + * result = 0.5 * C * INV * C^T * array + * array and result are 1d, C and INV are 2d arrays + */ +Matrix.multL = function (array, C, INV) { + var result = []; + var temp1 = []; + var temp2 = []; + + // multiply by C^T + for (var i = 0; i < C[0].length; i++) { + var sum = 0; + for (var j = 0; j < C.length; j++) { + sum += -0.5 * C[j][i] * array[j]; + } + temp1[i] = sum; + } + // multiply the result by INV + for (var _i2 = 0; _i2 < INV.length; _i2++) { + var _sum = 0; + for (var _j = 0; _j < INV.length; _j++) { + _sum += INV[_i2][_j] * temp1[_j]; + } + temp2[_i2] = _sum; + } + // multiply the result by C + for (var _i3 = 0; _i3 < C.length; _i3++) { + var _sum2 = 0; + for (var _j2 = 0; _j2 < C[0].length; _j2++) { + _sum2 += C[_i3][_j2] * temp2[_j2]; + } + result[_i3] = _sum2; + } + + return result; +}; + +module.exports = Matrix; + +/***/ }), +/* 25 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +/** + * A classic Quicksort algorithm with Hoare's partition + * - Works also on LinkedList objects + * + * Copyright: i-Vis Research Group, Bilkent University, 2007 - present + */ + +var LinkedList = __webpack_require__(11); + +var Quicksort = function () { + function Quicksort(A, compareFunction) { + _classCallCheck(this, Quicksort); + + if (compareFunction !== null || compareFunction !== undefined) this.compareFunction = this._defaultCompareFunction; + + var length = void 0; + if (A instanceof LinkedList) length = A.size();else length = A.length; + + this._quicksort(A, 0, length - 1); + } + + _createClass(Quicksort, [{ + key: '_quicksort', + value: function _quicksort(A, p, r) { + if (p < r) { + var q = this._partition(A, p, r); + this._quicksort(A, p, q); + this._quicksort(A, q + 1, r); + } + } + }, { + key: '_partition', + value: function _partition(A, p, r) { + var x = this._get(A, p); + var i = p; + var j = r; + while (true) { + while (this.compareFunction(x, this._get(A, j))) { + j--; + }while (this.compareFunction(this._get(A, i), x)) { + i++; + }if (i < j) { + this._swap(A, i, j); + i++; + j--; + } else return j; + } + } + }, { + key: '_get', + value: function _get(object, index) { + if (object instanceof LinkedList) return object.get_object_at(index);else return object[index]; + } + }, { + key: '_set', + value: function _set(object, index, value) { + if (object instanceof LinkedList) object.set_object_at(index, value);else object[index] = value; + } + }, { + key: '_swap', + value: function _swap(A, i, j) { + var temp = this._get(A, i); + this._set(A, i, this._get(A, j)); + this._set(A, j, temp); + } + }, { + key: '_defaultCompareFunction', + value: function _defaultCompareFunction(a, b) { + return b > a; + } + }]); + + return Quicksort; +}(); + +module.exports = Quicksort; + +/***/ }), +/* 26 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +// Singular Value Decomposition implementation +function SVD() {}; + +/* Below singular value decomposition (svd) code including hypot function is adopted from https://github.com/dragonfly-ai/JamaJS + Some changes are applied to make the code compatible with the fcose code and to make it independent from Jama. + Input matrix is changed to a 2D array instead of Jama matrix. Matrix dimensions are taken according to 2D array instead of using Jama functions. + An object that includes singular value components is created for return. + The types of input parameters of the hypot function are removed. + let is used instead of var for the variable initialization. +*/ +/* + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +SVD.svd = function (A) { + this.U = null; + this.V = null; + this.s = null; + this.m = 0; + this.n = 0; + this.m = A.length; + this.n = A[0].length; + var nu = Math.min(this.m, this.n); + this.s = function (s) { + var a = []; + while (s-- > 0) { + a.push(0); + }return a; + }(Math.min(this.m + 1, this.n)); + this.U = function (dims) { + var allocate = function allocate(dims) { + if (dims.length == 0) { + return 0; + } else { + var array = []; + for (var i = 0; i < dims[0]; i++) { + array.push(allocate(dims.slice(1))); + } + return array; + } + }; + return allocate(dims); + }([this.m, nu]); + this.V = function (dims) { + var allocate = function allocate(dims) { + if (dims.length == 0) { + return 0; + } else { + var array = []; + for (var i = 0; i < dims[0]; i++) { + array.push(allocate(dims.slice(1))); + } + return array; + } + }; + return allocate(dims); + }([this.n, this.n]); + var e = function (s) { + var a = []; + while (s-- > 0) { + a.push(0); + }return a; + }(this.n); + var work = function (s) { + var a = []; + while (s-- > 0) { + a.push(0); + }return a; + }(this.m); + var wantu = true; + var wantv = true; + var nct = Math.min(this.m - 1, this.n); + var nrt = Math.max(0, Math.min(this.n - 2, this.m)); + for (var k = 0; k < Math.max(nct, nrt); k++) { + if (k < nct) { + this.s[k] = 0; + for (var i = k; i < this.m; i++) { + this.s[k] = SVD.hypot(this.s[k], A[i][k]); + } + ; + if (this.s[k] !== 0.0) { + if (A[k][k] < 0.0) { + this.s[k] = -this.s[k]; + } + for (var _i = k; _i < this.m; _i++) { + A[_i][k] /= this.s[k]; + } + ; + A[k][k] += 1.0; + } + this.s[k] = -this.s[k]; + } + for (var j = k + 1; j < this.n; j++) { + if (function (lhs, rhs) { + return lhs && rhs; + }(k < nct, this.s[k] !== 0.0)) { + var t = 0; + for (var _i2 = k; _i2 < this.m; _i2++) { + t += A[_i2][k] * A[_i2][j]; + } + ; + t = -t / A[k][k]; + for (var _i3 = k; _i3 < this.m; _i3++) { + A[_i3][j] += t * A[_i3][k]; + } + ; + } + e[j] = A[k][j]; + } + ; + if (function (lhs, rhs) { + return lhs && rhs; + }(wantu, k < nct)) { + for (var _i4 = k; _i4 < this.m; _i4++) { + this.U[_i4][k] = A[_i4][k]; + } + ; + } + if (k < nrt) { + e[k] = 0; + for (var _i5 = k + 1; _i5 < this.n; _i5++) { + e[k] = SVD.hypot(e[k], e[_i5]); + } + ; + if (e[k] !== 0.0) { + if (e[k + 1] < 0.0) { + e[k] = -e[k]; + } + for (var _i6 = k + 1; _i6 < this.n; _i6++) { + e[_i6] /= e[k]; + } + ; + e[k + 1] += 1.0; + } + e[k] = -e[k]; + if (function (lhs, rhs) { + return lhs && rhs; + }(k + 1 < this.m, e[k] !== 0.0)) { + for (var _i7 = k + 1; _i7 < this.m; _i7++) { + work[_i7] = 0.0; + } + ; + for (var _j = k + 1; _j < this.n; _j++) { + for (var _i8 = k + 1; _i8 < this.m; _i8++) { + work[_i8] += e[_j] * A[_i8][_j]; + } + ; + } + ; + for (var _j2 = k + 1; _j2 < this.n; _j2++) { + var _t = -e[_j2] / e[k + 1]; + for (var _i9 = k + 1; _i9 < this.m; _i9++) { + A[_i9][_j2] += _t * work[_i9]; + } + ; + } + ; + } + if (wantv) { + for (var _i10 = k + 1; _i10 < this.n; _i10++) { + this.V[_i10][k] = e[_i10]; + }; + } + } + }; + var p = Math.min(this.n, this.m + 1); + if (nct < this.n) { + this.s[nct] = A[nct][nct]; + } + if (this.m < p) { + this.s[p - 1] = 0.0; + } + if (nrt + 1 < p) { + e[nrt] = A[nrt][p - 1]; + } + e[p - 1] = 0.0; + if (wantu) { + for (var _j3 = nct; _j3 < nu; _j3++) { + for (var _i11 = 0; _i11 < this.m; _i11++) { + this.U[_i11][_j3] = 0.0; + } + ; + this.U[_j3][_j3] = 1.0; + }; + for (var _k = nct - 1; _k >= 0; _k--) { + if (this.s[_k] !== 0.0) { + for (var _j4 = _k + 1; _j4 < nu; _j4++) { + var _t2 = 0; + for (var _i12 = _k; _i12 < this.m; _i12++) { + _t2 += this.U[_i12][_k] * this.U[_i12][_j4]; + }; + _t2 = -_t2 / this.U[_k][_k]; + for (var _i13 = _k; _i13 < this.m; _i13++) { + this.U[_i13][_j4] += _t2 * this.U[_i13][_k]; + }; + }; + for (var _i14 = _k; _i14 < this.m; _i14++) { + this.U[_i14][_k] = -this.U[_i14][_k]; + }; + this.U[_k][_k] = 1.0 + this.U[_k][_k]; + for (var _i15 = 0; _i15 < _k - 1; _i15++) { + this.U[_i15][_k] = 0.0; + }; + } else { + for (var _i16 = 0; _i16 < this.m; _i16++) { + this.U[_i16][_k] = 0.0; + }; + this.U[_k][_k] = 1.0; + } + }; + } + if (wantv) { + for (var _k2 = this.n - 1; _k2 >= 0; _k2--) { + if (function (lhs, rhs) { + return lhs && rhs; + }(_k2 < nrt, e[_k2] !== 0.0)) { + for (var _j5 = _k2 + 1; _j5 < nu; _j5++) { + var _t3 = 0; + for (var _i17 = _k2 + 1; _i17 < this.n; _i17++) { + _t3 += this.V[_i17][_k2] * this.V[_i17][_j5]; + }; + _t3 = -_t3 / this.V[_k2 + 1][_k2]; + for (var _i18 = _k2 + 1; _i18 < this.n; _i18++) { + this.V[_i18][_j5] += _t3 * this.V[_i18][_k2]; + }; + }; + } + for (var _i19 = 0; _i19 < this.n; _i19++) { + this.V[_i19][_k2] = 0.0; + }; + this.V[_k2][_k2] = 1.0; + }; + } + var pp = p - 1; + var iter = 0; + var eps = Math.pow(2.0, -52.0); + var tiny = Math.pow(2.0, -966.0); + while (p > 0) { + var _k3 = void 0; + var kase = void 0; + for (_k3 = p - 2; _k3 >= -1; _k3--) { + if (_k3 === -1) { + break; + } + if (Math.abs(e[_k3]) <= tiny + eps * (Math.abs(this.s[_k3]) + Math.abs(this.s[_k3 + 1]))) { + e[_k3] = 0.0; + break; + } + }; + if (_k3 === p - 2) { + kase = 4; + } else { + var ks = void 0; + for (ks = p - 1; ks >= _k3; ks--) { + if (ks === _k3) { + break; + } + var _t4 = (ks !== p ? Math.abs(e[ks]) : 0.0) + (ks !== _k3 + 1 ? Math.abs(e[ks - 1]) : 0.0); + if (Math.abs(this.s[ks]) <= tiny + eps * _t4) { + this.s[ks] = 0.0; + break; + } + }; + if (ks === _k3) { + kase = 3; + } else if (ks === p - 1) { + kase = 1; + } else { + kase = 2; + _k3 = ks; + } + } + _k3++; + switch (kase) { + case 1: + { + var f = e[p - 2]; + e[p - 2] = 0.0; + for (var _j6 = p - 2; _j6 >= _k3; _j6--) { + var _t5 = SVD.hypot(this.s[_j6], f); + var cs = this.s[_j6] / _t5; + var sn = f / _t5; + this.s[_j6] = _t5; + if (_j6 !== _k3) { + f = -sn * e[_j6 - 1]; + e[_j6 - 1] = cs * e[_j6 - 1]; + } + if (wantv) { + for (var _i20 = 0; _i20 < this.n; _i20++) { + _t5 = cs * this.V[_i20][_j6] + sn * this.V[_i20][p - 1]; + this.V[_i20][p - 1] = -sn * this.V[_i20][_j6] + cs * this.V[_i20][p - 1]; + this.V[_i20][_j6] = _t5; + }; + } + }; + }; + break; + case 2: + { + var _f = e[_k3 - 1]; + e[_k3 - 1] = 0.0; + for (var _j7 = _k3; _j7 < p; _j7++) { + var _t6 = SVD.hypot(this.s[_j7], _f); + var _cs = this.s[_j7] / _t6; + var _sn = _f / _t6; + this.s[_j7] = _t6; + _f = -_sn * e[_j7]; + e[_j7] = _cs * e[_j7]; + if (wantu) { + for (var _i21 = 0; _i21 < this.m; _i21++) { + _t6 = _cs * this.U[_i21][_j7] + _sn * this.U[_i21][_k3 - 1]; + this.U[_i21][_k3 - 1] = -_sn * this.U[_i21][_j7] + _cs * this.U[_i21][_k3 - 1]; + this.U[_i21][_j7] = _t6; + }; + } + }; + }; + break; + case 3: + { + var scale = Math.max(Math.max(Math.max(Math.max(Math.abs(this.s[p - 1]), Math.abs(this.s[p - 2])), Math.abs(e[p - 2])), Math.abs(this.s[_k3])), Math.abs(e[_k3])); + var sp = this.s[p - 1] / scale; + var spm1 = this.s[p - 2] / scale; + var epm1 = e[p - 2] / scale; + var sk = this.s[_k3] / scale; + var ek = e[_k3] / scale; + var b = ((spm1 + sp) * (spm1 - sp) + epm1 * epm1) / 2.0; + var c = sp * epm1 * (sp * epm1); + var shift = 0.0; + if (function (lhs, rhs) { + return lhs || rhs; + }(b !== 0.0, c !== 0.0)) { + shift = Math.sqrt(b * b + c); + if (b < 0.0) { + shift = -shift; + } + shift = c / (b + shift); + } + var _f2 = (sk + sp) * (sk - sp) + shift; + var g = sk * ek; + for (var _j8 = _k3; _j8 < p - 1; _j8++) { + var _t7 = SVD.hypot(_f2, g); + var _cs2 = _f2 / _t7; + var _sn2 = g / _t7; + if (_j8 !== _k3) { + e[_j8 - 1] = _t7; + } + _f2 = _cs2 * this.s[_j8] + _sn2 * e[_j8]; + e[_j8] = _cs2 * e[_j8] - _sn2 * this.s[_j8]; + g = _sn2 * this.s[_j8 + 1]; + this.s[_j8 + 1] = _cs2 * this.s[_j8 + 1]; + if (wantv) { + for (var _i22 = 0; _i22 < this.n; _i22++) { + _t7 = _cs2 * this.V[_i22][_j8] + _sn2 * this.V[_i22][_j8 + 1]; + this.V[_i22][_j8 + 1] = -_sn2 * this.V[_i22][_j8] + _cs2 * this.V[_i22][_j8 + 1]; + this.V[_i22][_j8] = _t7; + }; + } + _t7 = SVD.hypot(_f2, g); + _cs2 = _f2 / _t7; + _sn2 = g / _t7; + this.s[_j8] = _t7; + _f2 = _cs2 * e[_j8] + _sn2 * this.s[_j8 + 1]; + this.s[_j8 + 1] = -_sn2 * e[_j8] + _cs2 * this.s[_j8 + 1]; + g = _sn2 * e[_j8 + 1]; + e[_j8 + 1] = _cs2 * e[_j8 + 1]; + if (wantu && _j8 < this.m - 1) { + for (var _i23 = 0; _i23 < this.m; _i23++) { + _t7 = _cs2 * this.U[_i23][_j8] + _sn2 * this.U[_i23][_j8 + 1]; + this.U[_i23][_j8 + 1] = -_sn2 * this.U[_i23][_j8] + _cs2 * this.U[_i23][_j8 + 1]; + this.U[_i23][_j8] = _t7; + }; + } + }; + e[p - 2] = _f2; + iter = iter + 1; + }; + break; + case 4: + { + if (this.s[_k3] <= 0.0) { + this.s[_k3] = this.s[_k3] < 0.0 ? -this.s[_k3] : 0.0; + if (wantv) { + for (var _i24 = 0; _i24 <= pp; _i24++) { + this.V[_i24][_k3] = -this.V[_i24][_k3]; + }; + } + } + while (_k3 < pp) { + if (this.s[_k3] >= this.s[_k3 + 1]) { + break; + } + var _t8 = this.s[_k3]; + this.s[_k3] = this.s[_k3 + 1]; + this.s[_k3 + 1] = _t8; + if (wantv && _k3 < this.n - 1) { + for (var _i25 = 0; _i25 < this.n; _i25++) { + _t8 = this.V[_i25][_k3 + 1]; + this.V[_i25][_k3 + 1] = this.V[_i25][_k3]; + this.V[_i25][_k3] = _t8; + }; + } + if (wantu && _k3 < this.m - 1) { + for (var _i26 = 0; _i26 < this.m; _i26++) { + _t8 = this.U[_i26][_k3 + 1]; + this.U[_i26][_k3 + 1] = this.U[_i26][_k3]; + this.U[_i26][_k3] = _t8; + }; + } + _k3++; + }; + iter = 0; + p--; + }; + break; + } + }; + var result = { U: this.U, V: this.V, S: this.s }; + return result; +}; + +// sqrt(a^2 + b^2) without under/overflow. +SVD.hypot = function (a, b) { + var r = void 0; + if (Math.abs(a) > Math.abs(b)) { + r = b / a; + r = Math.abs(a) * Math.sqrt(1 + r * r); + } else if (b != 0) { + r = a / b; + r = Math.abs(b) * Math.sqrt(1 + r * r); + } else { + r = 0.0; + } + return r; +}; + +module.exports = SVD; + +/***/ }), +/* 27 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +/** + * Needleman-Wunsch algorithm is an procedure to compute the optimal global alignment of two string + * sequences by S.B.Needleman and C.D.Wunsch (1970). + * + * Aside from the inputs, you can assign the scores for, + * - Match: The two characters at the current index are same. + * - Mismatch: The two characters at the current index are different. + * - Insertion/Deletion(gaps): The best alignment involves one letter aligning to a gap in the other string. + */ + +var NeedlemanWunsch = function () { + function NeedlemanWunsch(sequence1, sequence2) { + var match_score = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 1; + var mismatch_penalty = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : -1; + var gap_penalty = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : -1; + + _classCallCheck(this, NeedlemanWunsch); + + this.sequence1 = sequence1; + this.sequence2 = sequence2; + this.match_score = match_score; + this.mismatch_penalty = mismatch_penalty; + this.gap_penalty = gap_penalty; + + // Just the remove redundancy + this.iMax = sequence1.length + 1; + this.jMax = sequence2.length + 1; + + // Grid matrix of scores + this.grid = new Array(this.iMax); + for (var i = 0; i < this.iMax; i++) { + this.grid[i] = new Array(this.jMax); + + for (var j = 0; j < this.jMax; j++) { + this.grid[i][j] = 0; + } + } + + // Traceback matrix (2D array, each cell is an array of boolean values for [`Diag`, `Up`, `Left`] positions) + this.tracebackGrid = new Array(this.iMax); + for (var _i = 0; _i < this.iMax; _i++) { + this.tracebackGrid[_i] = new Array(this.jMax); + + for (var _j = 0; _j < this.jMax; _j++) { + this.tracebackGrid[_i][_j] = [null, null, null]; + } + } + + // The aligned sequences (return multiple possibilities) + this.alignments = []; + + // Final alignment score + this.score = -1; + + // Calculate scores and tracebacks + this.computeGrids(); + } + + _createClass(NeedlemanWunsch, [{ + key: "getScore", + value: function getScore() { + return this.score; + } + }, { + key: "getAlignments", + value: function getAlignments() { + return this.alignments; + } + + // Main dynamic programming procedure + + }, { + key: "computeGrids", + value: function computeGrids() { + // Fill in the first row + for (var j = 1; j < this.jMax; j++) { + this.grid[0][j] = this.grid[0][j - 1] + this.gap_penalty; + this.tracebackGrid[0][j] = [false, false, true]; + } + + // Fill in the first column + for (var i = 1; i < this.iMax; i++) { + this.grid[i][0] = this.grid[i - 1][0] + this.gap_penalty; + this.tracebackGrid[i][0] = [false, true, false]; + } + + // Fill the rest of the grid + for (var _i2 = 1; _i2 < this.iMax; _i2++) { + for (var _j2 = 1; _j2 < this.jMax; _j2++) { + // Find the max score(s) among [`Diag`, `Up`, `Left`] + var diag = void 0; + if (this.sequence1[_i2 - 1] === this.sequence2[_j2 - 1]) diag = this.grid[_i2 - 1][_j2 - 1] + this.match_score;else diag = this.grid[_i2 - 1][_j2 - 1] + this.mismatch_penalty; + + var up = this.grid[_i2 - 1][_j2] + this.gap_penalty; + var left = this.grid[_i2][_j2 - 1] + this.gap_penalty; + + // If there exists multiple max values, capture them for multiple paths + var maxOf = [diag, up, left]; + var indices = this.arrayAllMaxIndexes(maxOf); + + // Update Grids + this.grid[_i2][_j2] = maxOf[indices[0]]; + this.tracebackGrid[_i2][_j2] = [indices.includes(0), indices.includes(1), indices.includes(2)]; + } + } + + // Update alignment score + this.score = this.grid[this.iMax - 1][this.jMax - 1]; + } + + // Gets all possible valid sequence combinations + + }, { + key: "alignmentTraceback", + value: function alignmentTraceback() { + var inProcessAlignments = []; + + inProcessAlignments.push({ pos: [this.sequence1.length, this.sequence2.length], + seq1: "", + seq2: "" + }); + + while (inProcessAlignments[0]) { + var current = inProcessAlignments[0]; + var directions = this.tracebackGrid[current.pos[0]][current.pos[1]]; + + if (directions[0]) { + inProcessAlignments.push({ pos: [current.pos[0] - 1, current.pos[1] - 1], + seq1: this.sequence1[current.pos[0] - 1] + current.seq1, + seq2: this.sequence2[current.pos[1] - 1] + current.seq2 + }); + } + if (directions[1]) { + inProcessAlignments.push({ pos: [current.pos[0] - 1, current.pos[1]], + seq1: this.sequence1[current.pos[0] - 1] + current.seq1, + seq2: '-' + current.seq2 + }); + } + if (directions[2]) { + inProcessAlignments.push({ pos: [current.pos[0], current.pos[1] - 1], + seq1: '-' + current.seq1, + seq2: this.sequence2[current.pos[1] - 1] + current.seq2 + }); + } + + if (current.pos[0] === 0 && current.pos[1] === 0) this.alignments.push({ sequence1: current.seq1, + sequence2: current.seq2 + }); + + inProcessAlignments.shift(); + } + + return this.alignments; + } + + // Helper Functions + + }, { + key: "getAllIndexes", + value: function getAllIndexes(arr, val) { + var indexes = [], + i = -1; + while ((i = arr.indexOf(val, i + 1)) !== -1) { + indexes.push(i); + } + return indexes; + } + }, { + key: "arrayAllMaxIndexes", + value: function arrayAllMaxIndexes(array) { + return this.getAllIndexes(array, Math.max.apply(null, array)); + } + }]); + + return NeedlemanWunsch; +}(); + +module.exports = NeedlemanWunsch; + +/***/ }), +/* 28 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +var layoutBase = function layoutBase() { + return; +}; + +layoutBase.FDLayout = __webpack_require__(18); +layoutBase.FDLayoutConstants = __webpack_require__(4); +layoutBase.FDLayoutEdge = __webpack_require__(19); +layoutBase.FDLayoutNode = __webpack_require__(20); +layoutBase.DimensionD = __webpack_require__(21); +layoutBase.HashMap = __webpack_require__(22); +layoutBase.HashSet = __webpack_require__(23); +layoutBase.IGeometry = __webpack_require__(8); +layoutBase.IMath = __webpack_require__(9); +layoutBase.Integer = __webpack_require__(10); +layoutBase.Point = __webpack_require__(12); +layoutBase.PointD = __webpack_require__(5); +layoutBase.RandomSeed = __webpack_require__(16); +layoutBase.RectangleD = __webpack_require__(13); +layoutBase.Transform = __webpack_require__(17); +layoutBase.UniqueIDGeneretor = __webpack_require__(14); +layoutBase.Quicksort = __webpack_require__(25); +layoutBase.LinkedList = __webpack_require__(11); +layoutBase.LGraphObject = __webpack_require__(2); +layoutBase.LGraph = __webpack_require__(6); +layoutBase.LEdge = __webpack_require__(1); +layoutBase.LGraphManager = __webpack_require__(7); +layoutBase.LNode = __webpack_require__(3); +layoutBase.Layout = __webpack_require__(15); +layoutBase.LayoutConstants = __webpack_require__(0); +layoutBase.NeedlemanWunsch = __webpack_require__(27); +layoutBase.Matrix = __webpack_require__(24); +layoutBase.SVD = __webpack_require__(26); + +module.exports = layoutBase; + +/***/ }), +/* 29 */ +/***/ (function(module, exports, __webpack_require__) { + +"use strict"; + + +function Emitter() { + this.listeners = []; +} + +var p = Emitter.prototype; + +p.addListener = function (event, callback) { + this.listeners.push({ + event: event, + callback: callback + }); +}; + +p.removeListener = function (event, callback) { + for (var i = this.listeners.length; i >= 0; i--) { + var l = this.listeners[i]; + + if (l.event === event && l.callback === callback) { + this.listeners.splice(i, 1); + } + } +}; + +p.emit = function (event, data) { + for (var i = 0; i < this.listeners.length; i++) { + var l = this.listeners[i]; + + if (event === l.event) { + l.callback(data); + } + } +}; + +module.exports = Emitter; + +/***/ }) +/******/ ]); +}); \ No newline at end of file diff --git a/tools/rirPrettyGraph/interaction.js b/tools/rirPrettyGraph/interaction.js new file mode 100644 index 000000000..07778dcd3 --- /dev/null +++ b/tools/rirPrettyGraph/interaction.js @@ -0,0 +1,169 @@ +// Add option event handlers which affect the graph + +showRecordedCallsCheckmark.addEventListener("change", function () { + updateRecordedCallVisibility(); +}); +childrenInsideParentsCheckmark.addEventListener("change", function () { + regenerate(); +}); +lassoSelectionCheckmark.addEventListener("change", function () { + // noinspection JSUnresolvedReference + graph.lassoSelectionEnabled(lassoSelectionCheckmark.checked); +}); + +// Enable autopan-on-drag +// noinspection JSUnresolvedReference +graph.autopanOnDrag({ + enabled: true, + selector: "node", + speed: 10, +}); + +// We need custom scripting to make node bodies visible when you hover over or click on the nodes, +// and also click on edges to show their label, source, and target. +let focused = undefined; +let selected = undefined; + +const focus = function (elem) { + focused = elem; + if (elem.isNode()) { + details.style.display = ""; + addrDiv.textContent = elem.data("id"); + const name = elem.data("name"); + if (name) { + nameDiv.style.display = ""; + // We *want* to render HTML because the name is already escaped + nameDiv.innerHTML = name; + } else { + nameDiv.style.display = "none"; + nameDiv.innerHTML = ""; + } + const body = elem.data("body"); + if (body) { + bodyDiv.style.display = ""; + // We *want* to render HTML in the body + bodyDiv.innerHTML = body; + } else { + bodyDiv.style.display = "none"; + bodyDiv.innerHTML = ""; + } + } else if (elem.isEdge()) { + const source = elem.source(); + const target = elem.target(); + details.style.display = ""; + addrDiv.textContent = elem.data("label"); + nameDiv.style.display = ""; + nameDiv.textContent = elem.classes().filter(c => c !== "arrow").join(" "); + bodyDiv.style.display = ""; + const render = function (node) { + let text = node.id(); + if (node.data("name")) { + text += ` = ${node.data("name")}`; + } + return escapeHtml(text); + } + bodyDiv.innerHTML = ` +

Source: ${render(source)}

+

Target: ${render(target)}

+ `; + } else { + throw new Error(`Bad element: ${elem}`); + } +} + +const defocus = function () { + focused = undefined; + details.style.display = "none"; + addrDiv.innerHTML = ""; + nameDiv.innerHTML = ""; + bodyDiv.innerHTML = ""; +} + +const highlight = function (elem) { + focus(elem); +} + +const unhighlight = function () { + if (selected) { + focus(selected); + } else { + defocus(); + } +} + +const select = function (elem) { + selected = elem; + focus(elem); +} + +const deselect = function () { + if (focused === selected) { + defocus(); + } + selected = undefined; +} + +graph.on("mouseover", "node", function (event) { + highlight(event.target); +}); + +graph.on("mouseout", "node", function (event) { + unhighlight(event.target); +}); + +graph.on("tap", function (event) { + if (!event.target || !event.target.isNode || selected === event.target) { + deselect(); + } else { + select(event.target); + } +}); + +// Make it so Z-grab also grabs incoming edges, and X-grab also grabs outgoing edges +let isZPressed = false; +let isXPressed = false; +document.addEventListener("keydown", function (event) { + switch (event.key) { + case "z": + isZPressed = true; + break; + case "x": + isXPressed = true; + break; + default: + break; + } +}); +document.addEventListener("keyup", function (event) { + switch (event.key) { + case "z": + isZPressed = false; + break; + case "x": + isXPressed = false; + break; + default: + break; + } +}); +let dragAlongPosition; +let dragAlongNodes; +graph.on("grab", "node", function(event) { + const dragAlongSource = event.target; + const p = dragAlongSource.position(); + dragAlongPosition = { x: p.x, y: p.y }; + dragAlongNodes = graph.collection(); + if (isZPressed) { + dragAlongNodes.merge(dragAlongSource.incomers()); + } + if (isXPressed) { + dragAlongNodes.merge(dragAlongSource.outgoers()); + } + dragAlongNodes.unmerge(dragAlongSource); +}); +graph.on("drag", "node", function (event) { + const p = event.target.position(); + const dragAlongDelta = { x: p.x - dragAlongPosition.x, y: p.y - dragAlongPosition.y }; + dragAlongPosition = { x: p.x, y: p.y }; + dragAlongNodes.shift(dragAlongDelta); +}); \ No newline at end of file diff --git a/tools/rirPrettyGraph/main.js b/tools/rirPrettyGraph/main.js new file mode 100644 index 000000000..b80dd71c5 --- /dev/null +++ b/tools/rirPrettyGraph/main.js @@ -0,0 +1,184 @@ +// printPrettyGraph.cpp puts all elements as divs using HTML attributes as if they would be rendered in a pure HTML. +// However, we actually use cytoscape, which has its own rendering mechanisms. +// So we essentially use HTML as a fancy DSL to store our element data, +// and this script parses that fancy DSL using the DOM API and translates it into cytoscape's DSL. + +// Get fallback and input +const sourcesNeeded = document.getElementById("sources-needed"); +const input = document.getElementById("js-input"); + +// Create Cytoscape container and details +const container = document.createElement("main"); +container.id = "cy"; +const details = document.createElement("aside"); +details.id = "details"; +const addrDiv = document.createElement("div"); +addrDiv.id = "addr"; +const nameDiv = document.createElement("div"); +nameDiv.id = "name"; +const bodyDiv = document.createElement("div"); +bodyDiv.id = "body"; +details.appendChild(addrDiv); +details.appendChild(nameDiv); +details.appendChild(bodyDiv); +const options = document.createElement("aside"); +options.id = "options"; +function makeCheckmark(id, labelText) { + const label = document.createElement("label"); + label.htmlFor = id; + label.textContent = labelText + const checkmark = document.createElement("input"); + checkmark.type = "checkbox"; + checkmark.id = id + checkmark.checked = localStorage.getItem(id) === "true"; + checkmark.addEventListener("change", function () { + localStorage.setItem(id, checkmark.checked.toString()); + }); + const container = document.createElement("div"); + container.appendChild(label); + container.appendChild(checkmark); + options.appendChild(container); + return checkmark; +} +function makeTip(text) { + const tip = document.createElement("p"); + tip.className = "tip"; + tip.textContent = text; + options.appendChild(tip); +} +const showRecordedCallsCheckmark = makeCheckmark("showRecordedCalls", "Show recorded calls"); +const childrenInsideParentsCheckmark = makeCheckmark("childrenInsideParents", "Children inside parents"); +const lassoSelectionCheckmark = makeCheckmark("lassoSelection", "Lasso selection"); +makeTip("Hold Z before drag to also drag nodes from incoming edges, hold X to also drag outgoing"); + +// Remove fallback and input, add container and associated +document.body.removeChild(sourcesNeeded); +// Don't actually remove so we can inspect the source, just hide +input.display = "none"; +document.body.appendChild(container); +document.body.appendChild(options); +document.body.appendChild(details); + +// Translate input into output. +// Specifically, read options, then parse nodes from `input` and return then in cytoscape format. +// See https://js.cytoscape.org/#getting-started/including-cytoscape.js and https://js.cytoscape.org/#cy.add +// for the cytoscape DSL for nodes and edges (we return an array of nodes and edges, like `elements`). +function translate() { + // Options + const childrenInsideParents = childrenInsideParentsCheckmark.checked; + + const elements = []; + const elementsWithParents = new Map(); + let isFirst = true; + for (const child of input.children) { + elements.push({ + group: "nodes", + data: { + id: child.id, + name: child.getElementsByClassName("name").item(0)?.innerHTML, + body: child.getElementsByClassName("body").item(0)?.innerHTML, + isMain: isFirst, + }, + classes: child.className.replaceAll("node-", "") + (isFirst ? " main" : ""), + }); + for (const connected of child.getElementsByClassName("arrow")) { + const target = connected.getAttribute("data-connected"); + const isChild = connected.hasAttribute("data-is-child"); + if (childrenInsideParents && isChild) { + if (target in elementsWithParents) { + console.warn("Multiple parents for " + target + "!"); + } + // A bit confusing: child is actually the parent here, and target is its child + // `child` refers to input.children + elementsWithParents.set(target, child.id); + } else { + elements.push({ + group: "edges", + data: { + label: connected.innerHTML, + source: child.id, + target, + isChild, + isRecordedCall: connected.className.includes("arrow-Code-target"), + }, + classes: connected.className.replaceAll("arrow-", ""), + }) + } + } + isFirst = false; + } + for (const [element, parent] of elementsWithParents.entries()) { + const child = elements.find(e => e.data.id === element); + if (child) { + child.data.parent = parent; + } else { + console.error("Parent " + parent + " not found for " + element + "!"); + } + } + return elements; +} + +// Create cytoscape graph +// See https://js.cytoscape.org/#getting-started/including-cytoscape.js and the rest of the page +// for an overview of the cytoscape DSL +const layout = { + name: "fcose", + nodeRepulsion: 100000, + idealEdgeLength: edge => { + if (edge.data("isChild")) { + return 50; + } else if (edge.target().hasClass("node-other")) { + return 150; + } else { + return 450; + } + } +}; +// noinspection JSUnresolvedReference +const graph = cytoscape({ + container, + elements: translate(), + style, + layout, +}); + +// Recreate cytoscape graph without removing event listeners which get added after this script ends... +function regenerate() { + const newNodes = translate(); + graph.batch(() => { + graph.remove(graph.elements()); + graph.add(newNodes); + graph.layout(layout).run(); + }); +} + +const mainNode = graph.$("[?isMain]"); +// Defer selecting until after we register handlers in interaction.js +document.addEventListener("load", () => { + mainNode.select(); +}); +const recordedCallElems = (() => { + const connected = mainNode; + let oldSize; + do { + oldSize = connected.size(); + const newConnectedEdges = connected.connectedEdges("[!isRecordedCall]"); + const newConnectedNodes = newConnectedEdges.connectedNodes(); + const childNodes = connected.children(); + const parentNodes = connected.parent(); + connected.merge(newConnectedEdges); + connected.merge(newConnectedNodes); + connected.merge(childNodes); + connected.merge(parentNodes); + } while (connected.size() > oldSize); + return graph.elements().difference(connected); +})(); +function updateRecordedCallVisibility() { + const showRecordedCalls = showRecordedCallsCheckmark.checked; + if (!showRecordedCalls) { + recordedCallElems.remove(); + } else { + recordedCallElems.restore(); + } +} +updateRecordedCallVisibility(); \ No newline at end of file diff --git a/tools/rirPrettyGraph/style.css b/tools/rirPrettyGraph/style.css new file mode 100644 index 000000000..c99052ac5 --- /dev/null +++ b/tools/rirPrettyGraph/style.css @@ -0,0 +1,62 @@ +body { + margin: 0; + padding: 0; + width: 100vw; + height: 100vh; + font-family: sans-serif; + font-size: 14px; + line-height: 1.5; + color: #333; +} + +#cy { + position: absolute; + left: 0; + top: 0; + width: 100%; + height: 100%; +} + +#options, #details { + top: 0; + position: absolute; + padding: 1em; + background: #3338; + backdrop-filter: blur(10px); + color: #fff; + max-height: 100%; + overflow: scroll; +} + +#options { + left: 0; +} + +#options .tip { + width: 300px; +} + +#details { + right: 0; + width: 25%; +} + +#details #addr { + font-size: 2em; + font-weight: bold; + margin: 0; +} + +#details #name { + font-size: 1.5em; + font-weight: bold; + margin-bottom: 0.5em; +} + +#details #body { + margin: 1em 0; +} + +#details #body p { + margin: 0.5em 0; +} \ No newline at end of file diff --git a/tools/rirPrettyGraph/utils.js b/tools/rirPrettyGraph/utils.js new file mode 100644 index 000000000..cb8b825eb --- /dev/null +++ b/tools/rirPrettyGraph/utils.js @@ -0,0 +1,8 @@ +function escapeHtml(unsafe) { + return unsafe + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} \ No newline at end of file diff --git a/tools/source_all_tests.R b/tools/source_all_tests.R new file mode 100644 index 000000000..541ed830f --- /dev/null +++ b/tools/source_all_tests.R @@ -0,0 +1,19 @@ +quitEnv <- new.env(parent = globalenv()) +quitEnv$quit <- function(...) { + stop("quit called") +} +quitEnv$q <- quitEnv$quit + +# Typically you want to use bin/tests instead, since that runs the tests in parallel. +# This is for when you want to run tests all in R, or want to debug in gdb/lldb. +for (f in sort(list.files("../rir/tests", pattern = "*.[rR]$", full.names = TRUE))) { + print(paste("*** RUNNING ", basename(f))) + tryCatch(source(f, echo=TRUE, local=quitEnv), error = function(e) { + if (grepl("quit called", as.character(e), fixed = TRUE)) { + print(paste("*** QUIT ", basename(f))) + } else { + print(paste("*** ERROR in ", basename(f))) + print(e) + } + }) +} diff --git a/tools/test-compiler-client-and-server b/tools/test-compiler-client-and-server new file mode 100755 index 000000000..b8190bf4f --- /dev/null +++ b/tools/test-compiler-client-and-server @@ -0,0 +1,52 @@ +#!/bin/bash -e + +# region prelude +set -e +SCRIPTPATH=$(cd "$(dirname "$0")" && pwd) +if [ ! -d "$SCRIPTPATH" ]; then + echo "Could not determine absolute dir of $0" + echo "Maybe accessed with symlink" +fi +export SCRIPTPATH + +if [ -z "$RIR_BUILD" ]; then + RIR_BUILD=$(pwd) +fi +export RIR_BUILD +# shellcheck disable=SC2144 +if [ ! -f $RIR_BUILD/librir.* ]; then + echo "could not find librir. are you in the correct directory?" + exit 1 +fi + +. "${SCRIPTPATH}/script_include.sh" +# endregion + +export PORT="${PORT=5555}" + +# From https://stackoverflow.com/questions/71776455/stop-bash-if-any-of-the-functions-fail-in-parallel +# We run both the compiler server and client, but exit early if either of them fails +# Boilerplate +LOCKDIR=$(mktemp -d) || exit "$?" +trap 'rm -rf "$LOCKDIR"; wait' EXIT +diex() { + echo "!! $1 crashed" >&2; + + # kill the other process + if [ "$1" == "server" ]; then + [ ! -z "$client_pid" ] && kill "$client_pid" + else + [ ! -z "$server_pid" ] && kill "$server_pid" + fi + + exit 1 +} + +# Actually run compiler server and client, we delay the client a bit to ensure the server is started +{ LOG_PREFIX="(server) " "${SCRIPTPATH}/test-compiler-server-only" || diex "server"; } & +server_pid=$! +{ sleep 0.1; LOG_PREFIX="(client) " "${SCRIPTPATH}/test-compiler-client-only" || diex "client"; } & +client_pid=$! + +# Ensure the process keeps running until the children are actually done +wait diff --git a/tools/test-compiler-client-expected.out b/tools/test-compiler-client-expected.out new file mode 100644 index 000000000..30404ce4c --- /dev/null +++ b/tools/test-compiler-client-expected.out @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/tools/test-compiler-client-only b/tools/test-compiler-client-only new file mode 100755 index 000000000..970a20904 --- /dev/null +++ b/tools/test-compiler-client-only @@ -0,0 +1,38 @@ +#!/bin/bash -e + +# region prelude +SCRIPTPATH=$(cd "$(dirname "$0")" && pwd) +if [ ! -d "$SCRIPTPATH" ]; then + echo "${LOG_PREFIX}Could not determine absolute dir of $0" + echo "${LOG_PREFIX}Maybe accessed with symlink" +fi +export SCRIPTPATH + +if [ -z "$RIR_BUILD" ]; then + RIR_BUILD=$(pwd) +fi +export RIR_BUILD +if [ ! -f $RIR_BUILD/librir.* ]; then + echo "${LOG_PREFIX}could not find librir. are you in the correct directory?" + exit 1 +fi + +. "${SCRIPTPATH}/script_include.sh" +RIR_EXE="${RIR_BUILD}/bin/R" +export PIR_PRETTY_GRAPH_DEPENDENCY_LOCATION="$SCRIPTPATH/rirPrettyGraph" +# endregion + +export PIR_CLIENT_ADDR="${PIR_CLIENT_ADDR=tcp://localhost:${PORT=5555}}" + +EXPECTED_PATH="${SCRIPTPATH}/test-compiler-client-expected.out" +ACTUAL_PATH="/tmp/test-compiler-client-actual.out" + +echo "${LOG_PREFIX}-> Running compiler client test" +PIR_CLIENT_COMPILE_SIZE_TO_HASH_ONLY=1024 "${RIR_EXE}" -f "${SCRIPTPATH}/test-compiler-client.r" > "${ACTUAL_PATH}" 2>&1 +echo "${LOG_PREFIX}-> Comparing output" +if diff "${EXPECTED_PATH}" "${ACTUAL_PATH}"; then + echo "${LOG_PREFIX}-> Files are the same" +else + echo "${LOG_PREFIX}!! Files are different" + # exit 1 # TODO: Actuall set expected output and check, ensure the run is deterministic +fi diff --git a/tools/test-compiler-client.r b/tools/test-compiler-client.r new file mode 100644 index 000000000..6d4c32b26 --- /dev/null +++ b/tools/test-compiler-client.r @@ -0,0 +1,190 @@ +warnifnot <- function(x) { + text <- deparse(substitute(x)) + if (!x) warning(paste(text, "failed")) +} + +# Small closure (pir_regression.R) +f <- pir.compile(rir.compile(function(a) a(b=1, 2))) +# Memoized +f <- pir.compile(rir.compile(function(a) a(b=1, 2))) +# Memoized again +f <- pir.compile(rir.compile(function(a) a(b=1, 2))) + +# Another small closure with a promise +foo <- function(x) { + y <- x + function() { + y <- y + 1 + y + } +} + +warnifnot(pir.check(foo, NoExternalCalls, warmup=function(f) {f(1);f(2)})) + +# Medium closure with nested closures (pir_check.R) +mandelbrot <- function(size) { + size = size + sum = 0 + byteAcc = 0 + bitNum = 0 + y = 0 + while (y < size) { + ci = (2.0 * y / size) - 1.0 + x = 0 + while (x < size) { + zr = 0.0 + zrzr = 0.0 + zi = 0.0 + zizi = 0.0 + cr = (2.0 * x / size) - 1.5 + z = 0 + notDone = TRUE + escape = 0 + while (notDone && (z < 50)) { + zr = zrzr - zizi + cr + zi = 2.0 * zr * zi + ci + zrzr = zr * zr + zizi = zi * zi + if ((zrzr + zizi) > 4.0) { + notDone = FALSE + escape = 1 + } + z = z + 1 + } + byteAcc = bitwShiftL(byteAcc, 1) + escape + bitNum = bitNum + 1 + if (bitNum == 8) { + sum = bitwXor(sum, byteAcc) + byteAcc = 0 + bitNum = 0 + } else if (x == (size - 1)) { + byteAcc = bitwShiftL(byteAcc, 8 - bitNum) + sum = bitwXor(sum, byteAcc) + byteAcc = 0 + bitNum = 0 + } + x = x + 1 + } + y = y + 1 + } + return (sum) +} + +warnifnot(pir.check(mandelbrot, NoExternalCalls, NoPromise, warmup=function(f) {f(13);f(27)})) + +# Memoized +mandelbrot <- function(size) { + size = size + sum = 0 + byteAcc = 0 + bitNum = 0 + y = 0 + while (y < size) { + ci = (2.0 * y / size) - 1.0 + x = 0 + while (x < size) { + zr = 0.0 + zrzr = 0.0 + zi = 0.0 + zizi = 0.0 + cr = (2.0 * x / size) - 1.5 + z = 0 + notDone = TRUE + escape = 0 + while (notDone && (z < 50)) { + zr = zrzr - zizi + cr + zi = 2.0 * zr * zi + ci + zrzr = zr * zr + zizi = zi * zi + if ((zrzr + zizi) > 4.0) { + notDone = FALSE + escape = 1 + } + z = z + 1 + } + byteAcc = bitwShiftL(byteAcc, 1) + escape + bitNum = bitNum + 1 + if (bitNum == 8) { + sum = bitwXor(sum, byteAcc) + byteAcc = 0 + bitNum = 0 + } else if (x == (size - 1)) { + byteAcc = bitwShiftL(byteAcc, 8 - bitNum) + sum = bitwXor(sum, byteAcc) + byteAcc = 0 + bitNum = 0 + } + x = x + 1 + } + y = y + 1 + } + return (sum) +} +warnifnot(pir.check(mandelbrot, NoExternalCalls, NoPromise, warmup=function(f) {f(13);f(27)})) + +# Many closures (reg-tests-1c.R) +## merge.dendrogram(), PR#15648 +mkDend <- function(n, lab, method = "complete", + ## gives *ties* often: + rGen = function(n) 1+round(16*abs(rnorm(n)))) { + stopifnot(is.numeric(n), length(n) == 1, n >= 1, is.character(lab)) + a <- matrix(rGen(n*n), n, n) + colnames(a) <- rownames(a) <- paste0(lab, 1:n) + .HC. <<- hclust(as.dist(a + t(a)), method=method) + as.dendrogram(.HC.) +} + +## recursive dendrogram methods and deeply nested dendrograms +op <- options(expressions = 999)# , verbose = 2) # -> max. depth= 961 +set.seed(11); d <- mkDend(1500, "A", method="single") +rd <- reorder(d, nobs(d):1) +## Error: evaluation nested too deeply: infinite recursion .. in R <= 3.2.3 +stopifnot(is.leaf(r1 <- rd[[1]]), is.leaf(r2 <- rd[[2:1]]), + attr(r1, "label") == "A1458", attr(r2, "label") == "A1317") +options(op)# revert + +## recursive dendrogram methods and deeply nested dendrograms +op <- options(expressions = 999)# , verbose = 2) # -> max. depth= 961 +set.seed(11); d <- mkDend(1500, "A", method="single") +print(d[[1]]) +rd <- reorder(d, nobs(d):1) +print(rd[[1]]) +## Error: evaluation nested too deeply: infinite recursion .. in R <= 3.2.3 +stopifnot(is.leaf(r1 <- rd[[1]]), is.leaf(r2 <- rd[[2:1]]), + attr(r1, "label") == "A1458", attr(r2, "label") == "A1317") +options(op)# revert + +## recursive dendrogram methods and deeply nested dendrograms +op <- options(expressions = 999)# , verbose = 2) # -> max. depth= 961 +set.seed(11); d <- mkDend(1500, "A", method="single") +print(d[[1]]) +rd <- reorder(d, nobs(d):1) +print(rd[[1]]) +## Error: evaluation nested too deeply: infinite recursion .. in R <= 3.2.3 +stopifnot(is.leaf(r1 <- rd[[1]]), is.leaf(r2 <- rd[[2:1]]), + attr(r1, "label") == "A1458", attr(r2, "label") == "A1317") +options(op)# revert + +## recursive dendrogram methods and deeply nested dendrograms +op <- options(expressions = 999)# , verbose = 2) # -> max. depth= 961 +set.seed(11); d <- mkDend(1500, "A", method="single") +print(d[[1]]) +rd <- reorder(d, nobs(d):1) +print(rd[[1]]) +## Error: evaluation nested too deeply: infinite recursion .. in R <= 3.2.3 +stopifnot(is.leaf(r1 <- rd[[1]]), is.leaf(r2 <- rd[[2:1]]), + attr(r1, "label") == "A1458", attr(r2, "label") == "A1317") +options(op)# revert + +# Many more closures (pir_regression6.R) +lsNamespaceInfo <- function(ns, ...) { + ns <- asNamespace(ns, base.OK = FALSE) + ls(..., envir = get(".__NAMESPACE__.", envir = ns, inherits = FALSE)) +} +allinfoNS <- function(ns) sapply(lsNamespaceInfo(ns), getNamespaceInfo, ns=ns) +utils::str(allinfoNS("stats")) +utils::str(allinfoNS("stats4")) + +# Kill the server (named "servers" because it kills all connected servers, +# but there is only one in this case) +rir.killCompilerServers() diff --git a/tools/test-compiler-server-expected.out b/tools/test-compiler-server-expected.out new file mode 100644 index 000000000..30404ce4c --- /dev/null +++ b/tools/test-compiler-server-expected.out @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/tools/test-compiler-server-only b/tools/test-compiler-server-only new file mode 100755 index 000000000..b56234632 --- /dev/null +++ b/tools/test-compiler-server-only @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# region prelude +SCRIPTPATH=$(cd "$(dirname "$0")" && pwd) +if [ ! -d "$SCRIPTPATH" ]; then + echo "${LOG_PREFIX}Could not determine absolute dir of $0" + echo "${LOG_PREFIX}Maybe accessed with symlink" +fi +export SCRIPTPATH + +if [ -z "$RIR_BUILD" ]; then + RIR_BUILD=$(pwd) +fi +export RIR_BUILD +if [ ! -f $RIR_BUILD/librir.* ]; then + echo "${LOG_PREFIX}could not find librir. are you in the correct directory?" + exit 1 +fi + +. "${SCRIPTPATH}/script_include.sh" +RIR_EXE="${RIR_BUILD}/bin/R" +export PIR_PRETTY_GRAPH_DEPENDENCY_LOCATION="$SCRIPTPATH/rirPrettyGraph" +# endregion + +export PIR_SERVER_ADDR="${PIR_SERVER_ADDR=tcp://*:${PORT=5555}}" + +EXPECTED_PATH="${SCRIPTPATH}/test-compiler-server-expected.out" +ACTUAL_PATH="/tmp/test-compiler-server-actual.out" + +echo "${LOG_PREFIX}-> Running compiler server" +echo "${LOG_PREFIX} Note: the compiler client will kill the server when it exits, and the harness will kill if it fails, but otherwise this will run indefinitely" +"${RIR_EXE}" --no-save > "${ACTUAL_PATH}" 2>&1 +echo "${LOG_PREFIX}-> Comparing output" +if diff "${EXPECTED_PATH}" "${ACTUAL_PATH}"; then + echo "${LOG_PREFIX}-> Files are the same" +else + echo "${LOG_PREFIX}!! Files are different" + # exit 1 # TODO: Actuall set expected output and check, ensure the run is deterministic +fi diff --git a/tools/tests b/tools/tests index 24bc8bef7..00b704fc8 100755 --- a/tools/tests +++ b/tools/tests @@ -30,6 +30,7 @@ fi . "${SCRIPTPATH}/script_include.sh" export ROOT_DIR="${SCRIPTPATH}/.." +export PIR_PRETTY_GRAPH_DEPENDENCY_LOCATION="$SCRIPTPATH/rirPrettyGraph" STATUS=$(mktemp /tmp/r-test-status.XXXXXX) touch $STATUS @@ -84,7 +85,7 @@ function run_test { if [ $res -ne 0 ]; then echo -e "\n*************************************************************************************" echo "*************************************************************************************" - echo "*** failed test $name:" + echo "*** failed test $name (code $res):" echo "*** $R $VALGRIND -f $TEST" echo "*** log:" echo "*************************************************************************************"