Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -204,3 +204,11 @@ dmypy.json

# Pyre type checker
.pyre/

# Generated protobuf Python files (auto-generated by tests)
p-isa_tools/data_formats/python/heracles/proto/*_pb2.py
!p-isa_tools/data_formats/python/heracles/proto/__init__.py

# Test artifacts
p-isa_tools/data_formats/test/*.program_trace
p-isa_tools/data_formats/test/*.data_trace*
2 changes: 2 additions & 0 deletions .typos.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
# variation of params
parms = "parms"
bload = "bload"
ser = "ser"
SerType = "SerType"

[files]
extend-exclude = [
Expand Down
File renamed without changes.
11 changes: 10 additions & 1 deletion p-isa_tools/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,15 @@ else()
set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of Build" FORCE)
endif()

option(ENABLE_DATA_FORMATS "Enable support for the data formats library" OFF)
option(ENABLE_DATA_FORMATS "Enable support for the data formats library" ON)
message(ENABLE_DATA_FORMATS="${ENABLE_DATA_FORMATS}")

option(ENABLE_FUNCTIONAL_MODELER "Enable building of functional modeler" ON)
message(ENABLE_FUNCTIONAL_MODELER="${ENABLE_FUNCTIONAL_MODELER}")

option(ENABLE_KERNGEN "Enable kerngen (dependencies only)" ON)
message(ENABLE_KERNGEN="${ENABLE_KERNGEN}")

option(ENABLE_PROGRAM_MAPPER "Enable building of program mapper" ON)
message(ENABLE_PROGRAM_MAPPER="${ENABLE_PROGRAM_MAPPER}")

Expand Down Expand Up @@ -63,6 +66,12 @@ file(GLOB_RECURSE IDE_HEADERS program_mapper/*.h functional_modeler/*.h dependen

# Build sub-directories
add_subdirectory(common)
if(ENABLE_KERNGEN)
add_subdirectory(kerngen)
endif()
if(ENABLE_DATA_FORMATS)
add_subdirectory(data_formats)
endif()
if(ENABLE_FUNCTIONAL_MODELER)
add_subdirectory(functional_modeler)
endif()
Expand Down
12 changes: 0 additions & 12 deletions p-isa_tools/cmake/dependencies.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,3 @@ if (NOT snap_POPULATED)
include_directories(${snap_SOURCE_DIR}/snap-core ${snap_SOURCE_DIR}/glib-core)
message(STATUS "Finished building SNAP")
endif()

if(ENABLE_DATA_FORMATS)
find_package(HERACLES_DATA_FORMATS CONFIG)
if(NOT HERACLES_DATA_FORMATS_FOUND)
FetchContent_Declare(
heracles_data_formats
GIT_REPOSITORY git@github.com:IntelLabs/HERACLES-data-formats.git
GIT_TAG main
)
FetchContent_MakeAvailable(heracles_data_formats)
endif()
endif()
57 changes: 57 additions & 0 deletions p-isa_tools/data_formats/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
cmake_minimum_required(VERSION 3.15.0...3.24.0)

project(HERACLES_DATA_FORMATS VERSION 1.0.0)

set(CMAKE_CXX_STANDARD 17)
set(HERACLES_DATA_FORMATS_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
include(CMakePackageConfigHelpers)
include(GNUInstallDirs)

# Set default output directories
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}")

include(${PROJECT_SOURCE_DIR}/cmake/utils.cmake)
include(${PROJECT_SOURCE_DIR}/cmake/protobuf.cmake)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 -fPIC")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O3 -fPIC")
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_INSTALL_RPATH "$ORIGIN;$ORIGIN/${CMAKE_INSTALL_LIBDIR}")
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_INSTALL_LIBDIR})

find_package(OpenMP REQUIRED)

option(BUILD_TEST "Build c++/python tests with ctest" ON)
enable_testing()

add_subdirectory(proto)
add_subdirectory(cpp)
if(BUILD_TEST)
add_subdirectory(test)
endif()

# install python utility functions
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python/
DESTINATION python
# TODO(skmono): to be added afterwards
# FILES_MATCHING
# PATTERN "*.py"
)

# copy python utility functions to build/python
add_custom_target(HERACLES_DATA_FORMATS_COPY_PYTHON
ALL
DEPENDS HERACLES_data_proto
)
add_custom_command(
TARGET HERACLES_DATA_FORMATS_COPY_PYTHON
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/python ${PROJECT_BINARY_DIR}/python/
)

option(ENABLE_OPENFHE_TRACER "Build the OpenFHE tracer" ON)
if(ENABLE_OPENFHE_TRACER)
add_subdirectory(tracers/openfhe)
endif()
128 changes: 128 additions & 0 deletions p-isa_tools/data_formats/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
# HERACLES Data formatter interface

## CMake Configure and Build
```bash
cmake -S . -B build
cmake --build build --parallel
```
_Note: for now cmake will _not build with `ninja`_ and is only tested for
(default) `CMAKE_GENERATOR='Unix Makefiles`_


## Run test
```bash
cmake --build build --target test
```

Note: Python `[dev]` dependencies from the root `pyproject.toml` are required to run the Python test. They can be installed via
```bash
pip install -e ".[dev]" # from repository root
```

## C++

### Importing the **HERACLES-Data-Formats** Library

The C++ library be found and included with cmake by including
following statements in the cmakefile of the project depending on the
HERACLES data formats library:
```cmake
find_package(HERACLES_DATA_FORMATS 1.0.0 REQUIRED)
...
target_link_libraries(<YOUR_LIBRARY> PUBLIC HERACLES_DATA_FORMATS::heracles_data_formats)
```
Assuming you follow the convention of having all code
checked out in the same directory and named by their component name, you
can then build that project by executing the following:

```bash
# from project root
HERACLES_DATA_FORMATS_DIR=$(pwd)/../HERACLES-data-formats/build cmake -S . -B build
cmake --build build --parallel
```
Alternatively, you can also build and install HERACLES-data-formats
(with the destination chosen, e.g., using the
`-DCMAKE_INSTALL_PREFIX=/path/to/install` argument, and an `cmake
--build build --target install` after the build ). However, when
installing be careful in not forgetting to re-install
after each change and subsequent build or accidentally picking up
older versions installed elsewhere and earlier searched in CMAKE's
search paths.


### Usage example
The library can be used in the ```C++``` code, e.g., as followed:
```c++
// protobuf headers
#include "heracles/heracles_proto.h"
// cpp utility headers
#include "heracles/heracles_data_formats.h"

int main() {
heracles::fhe_trace::Trace trace;
heracles::data::InputPolynomials input_polys;

return 0;
}
```
Refer to the [heracles_test.cpp](src/data_formats/test/heracles_test.cpp) source
code for additional examples of using Heracles protobuf objects and
utility functions as well as [Protocol Buffer Basics:
C++](https://protobuf.dev/getting-started/cpptutorial/) for more
general information on using generated C++ protobuf code.


## Python


For the Python package to be used independently of CMake/C++ builds, the optional `dev` dependencies are required.

1. **Install dependencies**:
```bash
# For development (includes grpcio-tools for compiling protos, pytest for testing)
pip install -e ".[dev]"
```

2. **Compile Protocol Buffers**:
```bash
python p-isa_tools/data_formats/compile_protos.py
```

This generates the Python protobuf files in `p-isa_tools/data_formats/python/heracles/proto/`.

3. **Generate test traces** (if needed for testing):
```bash
python p-isa_tools/data_formats/test/generate_test_traces.py
```
Alternatively, you can simply run the `pytest` tests, which will create the protobuf files and/or test traces if they do not exist yet.

### Running Tests

From the repository root:
```bash
pytest p-isa_tools/data_formats/test/
```
(The path is optional, but avoids running unrelated tests)

### Usage example
The **HERACLES-Data-Formats** library can be imported via, e.g.,
```python
from heracles.proto.common_pb2 import Scheme
from heracles.proto.fhe_trace_pb2 import Trace, Instruction
import heracles.fhe_trace.io as hfi
import heracles.data.io as hdi

# Create and save a trace
trace = Trace()
trace.scheme = Scheme.SCHEME_BGV
hfi.store_trace("my_trace.bin", trace)

# Load a trace
loaded_trace = hfi.load_trace("my_trace.bin")
```

Refer to the [heracles_test.py](test/heracles_test.py) script for
examples of using Heracles protobuf objects and utility functions as
well as [Protocol Buffer Basics:
Python](https://protobuf.dev/getting-started/pythontutorial/) for more
general information on using generated python protobuf code.
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Copyright (C) 2025 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

@PACKAGE_INIT@

include(CMakeFindDependencyMacro)

include(${CMAKE_CURRENT_LIST_DIR}/HERACLES_DATA_FORMATSTargets.cmake)

if(TARGET HERACLES_DATA_FORMATS::heracles_data_formats)
set(HERACLES_DATA_FORMATS_FOUND TRUE)
message(STATUS "Heracles Data Formats Library found")
else()
message(STATUS "Heracles Data Formats Library not found")
endif()

# Requirement for protobuf
find_package(ZLIB REQUIRED)

set(HERACLES_DATA_FORMATS_VERSION "@HERACLES_DATA_FORMATS_VERSION")
set(HERACLES_DATA_FORMATS_VERSION_MAJOR "@HERACLES_DATA_FORMATS_VERSION_MAJOR")
set(HERACLES_DATA_FORMATS_VERSION_MINOR "@HERACLES_DATA_FORMATS_VERSION")
set(HERACLES_DATA_FORMATS_VERSION_PATCH "@HERACLES_DATA_FORMATS_VERSION")

set(HERACLES_DATA_FORMATS_DEBUG "@HERACLES_DATA_FORMATS_DEBUG")
Loading