diff --git a/.github/workflows/build_and_release.yml b/.github/workflows/build_and_release.yml new file mode 100644 index 0000000..3c34b16 --- /dev/null +++ b/.github/workflows/build_and_release.yml @@ -0,0 +1,165 @@ +name: Build and Release Pyraview + +on: + push: + branches: [ main ] + tags: [ 'v*' ] + pull_request: + branches: [ main ] + workflow_dispatch: # Allows manual triggering + +jobs: + build_and_test: + name: Build & Test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup CMake + uses: lukka/get-cmake@latest + + - name: Configure CMake + shell: bash + run: | + mkdir build + if [ "${{ runner.os }}" == "Windows" ]; then + cmake -S . -B build -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release + else + cmake -S . -B build -DCMAKE_BUILD_TYPE=Release + fi + + - name: Build All + run: cmake --build build --parallel + + - name: Run C Tests + shell: bash + run: | + cd build + ctest --output-on-failure + + # Zip binaries for release (naming by OS/Architecture) + - name: Package Binaries + if: startsWith(github.ref, 'refs/tags/') + shell: bash + run: | + mkdir -p dist + if [ "${{ runner.os }}" == "Windows" ]; then + # Windows CMake builds output to bin due to our CMakeLists.txt fix + zip -j dist/pyraview-win-x64.zip build/bin/*.dll build/bin/*.exe + elif [ "${{ runner.os }}" == "macOS" ]; then + zip -j dist/pyraview-mac-arm.zip build/bin/*.dylib build/bin/run_tests + else + zip -j dist/pyraview-linux-x64.zip build/bin/*.so build/bin/run_tests + fi + + - name: Upload Artifacts + if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v4 + with: + name: binaries-${{ matrix.os }} + path: dist/* + + build-matlab: + name: Build & Test Matlab (${{ matrix.os }}) + needs: build_and_test + runs-on: ${{ matrix.os }} + strategy: + matrix: + # Matlab actions support limited OS versions, check availability + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - uses: matlab-actions/setup-matlab@v2 + with: + release: 'R2024b' + + - name: Compile MEX + uses: matlab-actions/run-command@v2 + with: + # Enable OpenMP if supported by the platform (simple check or flag) + # For Ubuntu (GCC): -fopenmp + # For Windows (MSVC): -openmp (or implied via /openmp) + # For macOS (Clang): -Xpreprocessor -fopenmp -lomp (but requires libomp) + # To keep it simple and avoid linker errors on stock runners without libomp, we skip explicit OMP flags for now or use safe defaults. + # But pyraview.c has #include . If we don't link OMP, it might fail if _OPENMP is defined by default but library isn't linked. + # Let's try compiling WITHOUT flags first, relying on the source's #ifdef _OPENMP guards. + command: mex -v src/matlab/pyraview_mex.c src/c/pyraview.c -Iinclude -output src/matlab/pyraview + + - name: Run Matlab Tests + uses: matlab-actions/run-tests@v2 + with: + select-by-folder: src/matlab + + - name: Upload MEX Artifact + if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v4 + with: + name: mex-${{ matrix.os }} + path: src/matlab/pyraview.* # Matches pyraview.mexw64, .mexa64, etc. + if-no-files-found: error + + package-matlab: + name: Package Matlab Toolbox + needs: build-matlab + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: matlab-actions/setup-matlab@v2 + with: + release: 'R2024b' + + # Download all platform-specific MEX files we just built + - name: Download all MEX artifacts + uses: actions/download-artifact@v4 + with: + path: src/matlab + pattern: mex-* + merge-multiple: true + + # Run the packaging command + - name: Package Toolbox + uses: matlab-actions/run-command@v2 + with: + command: | + opts = matlab.addons.toolbox.ToolboxOptions('toolboxPackaging.prj'); + matlab.addons.toolbox.packageToolbox(opts); + + # Upload the .mltbx as an artifact so the release job can pick it up + - name: Upload Toolbox Artifact + uses: actions/upload-artifact@v4 + with: + name: matlab-toolbox + path: Pyraview.mltbx + + release: + name: Create GitHub Release + needs: [build_and_test, package-matlab] + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + permissions: + contents: write # Required to create releases + + steps: + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + path: release-assets + merge-multiple: true + + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + files: release-assets/* + name: Release ${{ github.ref_name }} + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5fb3d2a --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Ignore build artifacts +build/ +__pycache__/ +*.pyc +*.o +*.so +*.dylib +*.dll +*.a +*.exe +CMakeFiles/ +CMakeCache.txt +cmake_install.cmake +Makefile +CTestTestfile.cmake +Testing/ + +# Ignore test output +test_output* +test_matlab* +test_c* + +# Ignore generated files +*.bin diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..c6e2aa1 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.10) +project(Pyraview) + +enable_testing() + +# Set global output directories to verify consistent locations +# On Windows, DLLs and EXEs must be in the same folder for runtime linking +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") + +# Ensure Windows (MSVC) doesn't append Debug/Release subdirectories +foreach(config ${CMAKE_CONFIGURATION_TYPES} "Release" "Debug" "RelWithDebInfo" "MinSizeRel") + string(TOUPPER "${config}" CONFIG) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG} "${CMAKE_BINARY_DIR}/bin") + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CONFIG} "${CMAKE_BINARY_DIR}/bin") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CONFIG} "${CMAKE_BINARY_DIR}/lib") +endforeach() + +add_subdirectory(src/c) diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..c5a4852 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,70 @@ +# Pyraview API Reference + +## C API (`src/c/pyraview.c`, `include/pyraview_header.h`) + +### `pyraview_process_chunk` +Processes a chunk of raw data and updates decimation pyramids. + +Arguments: +- `dataArray`: Pointer to raw data. +- `numRows`: Number of samples per channel. +- `numCols`: Number of channels. +- `dataType`: + - 0: `int8` + - 1: `uint8` + - 2: `int16` + - 3: `uint16` + - 4: `int32` + - 5: `uint32` + - 6: `int64` + - 7: `uint64` + - 8: `float32` (Single) + - 9: `float64` (Double) +- `layout`: 0=SxC (Sample-Major), 1=CxS (Channel-Major). +- `filePrefix`: Base name for output files (e.g., `data/myrecording`). +- `append`: 1=Append, 0=Create/Overwrite. +- `levelSteps`: Pointer to array of decimation factors (e.g., `[100, 10, 10]`). +- `numLevels`: Number of elements in `levelSteps`. +- `nativeRate`: Original sampling rate (Hz). +- `numThreads`: Number of OpenMP threads (0 for auto). + +Returns: +- 0 on success. +- Negative values on error (e.g., -2 mismatch, -1 I/O error). + +--- + +## Python API (`src/python/pyraview.py`) + +### `process_chunk(data, file_prefix, level_steps, native_rate, append=False, layout='SxC', num_threads=0)` +Wrapper for the C function. + +Arguments: +- `data`: Numpy array (2D). Rows=Samples (if SxC), Rows=Channels (if CxS). + - Supported dtypes: `int8`, `uint8`, `int16`, `uint16`, `int32`, `uint32`, `int64`, `uint64`, `float32`, `float64`. +- `file_prefix`: String path prefix. +- `level_steps`: List of integers. +- `native_rate`: Float. +- `append`: Boolean. +- `layout`: 'SxC' or 'CxS'. +- `num_threads`: Int. + +Returns: +- 0 on success. Raises `RuntimeError` on failure. + +--- + +## Matlab API (`src/matlab/pyraview_mex.c`) + +### `status = pyraview_mex(data, prefix, steps, nativeRate, [append], [numThreads])` + +Arguments: +- `data`: Numeric matrix. Supports: `int8`, `uint8`, `int16`, `uint16`, `int32`, `uint32`, `int64`, `uint64`, `single`, `double`. +- `prefix`: String. +- `steps`: Vector of integers. +- `nativeRate`: Scalar double. +- `append`: Logical/Scalar (optional). +- `numThreads`: Scalar (optional). + +Returns: +- `status`: 0 on success. Throws error on failure. diff --git a/docs/BINARY_FORMAT.md b/docs/BINARY_FORMAT.md new file mode 100644 index 0000000..4fcc6c7 --- /dev/null +++ b/docs/BINARY_FORMAT.md @@ -0,0 +1,60 @@ +# Pyraview Binary Format + +Pyraview uses a simple, efficient binary format for storing multi-resolution time-series data. Each level of the pyramid is stored in a separate file (e.g., `data_L1.bin`, `data_L2.bin`). + +## File Structure + +| Section | Size | Description | +|---|---|---| +| Header | 1024 bytes | Metadata about the file. | +| Data | Variable | Interleaved Min/Max pairs for all channels. | + +## Header (1024 bytes) + +The header is fixed-size and little-endian. + +| Field | Type | Size | Description | +|---|---|---|---| +| `magic` | char[4] | 4 | "PYRA" magic string. | +| `version` | uint32 | 4 | Format version (currently 1). | +| `dataType` | uint32 | 4 | Enum for data precision (see below). | +| `channelCount` | uint32 | 4 | Number of channels. | +| `sampleRate` | double | 8 | Sample rate of *this level*. | +| `nativeRate` | double | 8 | Original recording rate. | +| `decimationFactor` | uint32 | 4 | Cumulative decimation factor from raw. | +| `reserved` | uint8 | 988 | Padding (zeros). | + +### Data Types (Enum) + +| Value | Type | Description | +|---|---|---| +| 0 | `int8` | Signed 8-bit integer. | +| 1 | `uint8` | Unsigned 8-bit integer. | +| 2 | `int16` | Signed 16-bit integer. | +| 3 | `uint16` | Unsigned 16-bit integer. | +| 4 | `int32` | Signed 32-bit integer. | +| 5 | `uint32` | Unsigned 32-bit integer. | +| 6 | `int64` | Signed 64-bit integer. | +| 7 | `uint64` | Unsigned 64-bit integer. | +| 8 | `float32` | 32-bit floating point (Single). | +| 9 | `float64` | 64-bit floating point (Double). | + +## Data Layout + +The data section follows the header immediately (byte 1024). + +The layout consists of **Time Chunks**. Each time chunk contains the Min/Max values for all channels for a specific time interval. + +Within a chunk, data is organized as: + +`[Ch0_MinMax][Ch1_MinMax]...[ChN_MinMax]` + +Where `ChX_MinMax` is a sequence of `(Min, Max)` pairs for that channel in that time chunk. + +Note: Since files are often appended to, the file consists of a sequence of these chunks. The chunk size depends on the processing block size used during generation. + +### Values + +Each logical sample in the decimated file consists of TWO values: `Min` and `Max`. They are stored interleaved: `Min, Max, Min, Max...`. + +If the file contains `N` logical samples for `C` channels, the total number of values is `N * C * 2`. diff --git a/include/pyraview_header.h b/include/pyraview_header.h new file mode 100644 index 0000000..0097803 --- /dev/null +++ b/include/pyraview_header.h @@ -0,0 +1,70 @@ +#ifndef PYRAVIEW_HEADER_H +#define PYRAVIEW_HEADER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Constants +#define PYRA_MAGIC "PYRA" +#define PYRA_HEADER_SIZE 1024 + +// Enum for Data Types +typedef enum { + PV_INT8 = 0, + PV_UINT8 = 1, + PV_INT16 = 2, + PV_UINT16 = 3, + PV_INT32 = 4, + PV_UINT32 = 5, + PV_INT64 = 6, + PV_UINT64 = 7, + PV_FLOAT32 = 8, + PV_FLOAT64 = 9 +} PvDataType; + +// Alignment Macros +#if defined(_MSC_VER) + #define PV_ALIGN_PREFIX(n) __declspec(align(n)) + #define PV_ALIGN_SUFFIX(n) +#else + #define PV_ALIGN_PREFIX(n) + #define PV_ALIGN_SUFFIX(n) __attribute__((aligned(n))) +#endif + +// Header Structure (1024-byte fixed, 64-byte aligned) +typedef PV_ALIGN_PREFIX(64) struct { + char magic[4]; // "PYRA" + uint32_t version; // 1 + uint32_t dataType; // PvDataType + uint32_t channelCount; // Number of channels + double sampleRate; // Sample rate of this level + double nativeRate; // Original recording rate + uint32_t decimationFactor; // Cumulative decimation from raw + uint8_t reserved[988]; // Padding to 1024 bytes +} PV_ALIGN_SUFFIX(64) PyraviewHeader; + +// API Function +// Returns 0 on success, negative values for errors +// layout: 0=SxC (Sample-Major), 1=CxS (Channel-Major) +int pyraview_process_chunk( + const void* dataArray, // Pointer to raw data + int64_t numRows, // Number of samples per channel + int64_t numCols, // Number of channels + int dataType, // PvDataType + int layout, // 0=SxC, 1=CxS + const char* filePrefix, // Base name for output files + int append, // Boolean flag + const int* levelSteps, // Array of decimation factors [100, 10, 10] + int numLevels, // Size of levelSteps array + double nativeRate, // Original recording rate (required for header/validation) + int numThreads // 0 for auto +); + +#ifdef __cplusplus +} +#endif + +#endif // PYRAVIEW_HEADER_H diff --git a/src/c/CMakeLists.txt b/src/c/CMakeLists.txt new file mode 100644 index 0000000..f1c1be9 --- /dev/null +++ b/src/c/CMakeLists.txt @@ -0,0 +1,36 @@ +cmake_minimum_required(VERSION 3.10) +project(pyraview C) + +find_package(OpenMP) + +# Use absolute path for include to avoid relative path headaches +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../include) + +# Standard output directories - works perfectly with MinGW/GCC/Clang +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +add_library(pyraview SHARED pyraview.c) + +if(OpenMP_C_FOUND) + target_link_libraries(pyraview PUBLIC OpenMP::OpenMP_C) +else() + message(WARNING "OpenMP not found. Compiling without parallel support.") +endif() + +if(NOT WIN32) + target_link_libraries(pyraview PRIVATE m) +endif() + +# Tests +add_executable(run_tests tests/test_main.c) +target_link_libraries(run_tests PRIVATE pyraview) + +if(NOT WIN32) + target_link_libraries(run_tests PRIVATE m) +endif() + +# Enable testing must be called before add_test +enable_testing() +add_test(NAME run_tests COMMAND run_tests) diff --git a/src/c/pyraview.c b/src/c/pyraview.c new file mode 100644 index 0000000..9e532eb --- /dev/null +++ b/src/c/pyraview.c @@ -0,0 +1,276 @@ +#define _FILE_OFFSET_BITS 64 +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include + +#ifdef _WIN32 + #include + #define pv_fseek _fseeki64 + #define pv_ftell _ftelli64 +#else + #include + #define pv_fseek fseeko + #define pv_ftell ftello +#endif + +#ifdef _OPENMP +#include +#else +#define omp_get_max_threads() 1 +#endif + +#include + +// Utility: Write header +static void pv_write_header(FILE* f, int channels, int type, double sampleRate, double nativeRate, int decimation) { + PyraviewHeader h; + memset(&h, 0, sizeof(h)); + memcpy(h.magic, "PYRA", 4); + h.version = 1; + h.dataType = type; + h.channelCount = channels; + h.sampleRate = sampleRate; + h.nativeRate = nativeRate; + h.decimationFactor = decimation; + fwrite(&h, sizeof(h), 1, f); +} + +// Utility: Validate header +// Returns 1 if valid (or created), 0 if mismatch, -1 if error +static int pv_validate_or_create(FILE** f_out, const char* filename, int channels, int type, double sampleRate, double nativeRate, int decimation, int append) { + FILE* f = NULL; + if (append) { + f = fopen(filename, "r+b"); // Try open existing for read/write + if (f) { + PyraviewHeader h; + if (fread(&h, sizeof(h), 1, f) != 1) { + fclose(f); + return -1; // File too short + } + if (memcmp(h.magic, "PYRA", 4) != 0) { + fclose(f); + return -1; // Not a PYRA file + } + // Strict match check + if (h.channelCount != (uint32_t)channels || + h.dataType != (uint32_t)type || + h.decimationFactor != (uint32_t)decimation || + fabs(h.sampleRate - sampleRate) > 1e-5) { + fclose(f); + return 0; // Mismatch + } + // Seek to end + pv_fseek(f, 0, SEEK_END); + *f_out = f; + return 1; + } + // If append is requested but file doesn't exist, create it (fall through) + } + + f = fopen(filename, "wb"); // Write new + if (!f) return -1; + pv_write_header(f, channels, type, sampleRate, nativeRate, decimation); + *f_out = f; + return 1; +} + +// Template macro for typed worker +#define DEFINE_WORKER(T, SUFFIX) \ +static int pv_internal_execute_##SUFFIX( \ + const T* data, \ + int64_t R, \ + int64_t C, \ + int layout, \ + const char* prefix, \ + int append, \ + const int* steps, \ + int nLevels, \ + double nativeRate, \ + int dataType, \ + int nThreads \ +) { \ + /* Open files */ \ + FILE* files[16]; \ + double currentRate = nativeRate; \ + int currentDecimation = 1; \ + int ret = 0; \ + \ + /* Pre-calculate rates and decimations */ \ + double rates[16]; \ + int decimations[16]; \ + for (int i = 0; i < nLevels; i++) { \ + currentDecimation *= steps[i]; \ + currentRate /= steps[i]; \ + rates[i] = currentRate; \ + decimations[i] = currentDecimation; \ + \ + char filename[512]; \ + snprintf(filename, sizeof(filename), "%s_L%d.bin", prefix, i+1); \ + int status = pv_validate_or_create(&files[i], filename, (int)C, dataType, rates[i], nativeRate, decimations[i], append); \ + if (status <= 0) { \ + /* Cleanup previous opens */ \ + for (int j = 0; j < i; j++) fclose(files[j]); \ + return (status == 0) ? -2 : -1; \ + } \ + } \ + \ + /* Stride logic */ \ + int64_t stride_ch = (layout == 1) ? R : 1; /* Distance between samples of same channel */ \ + int64_t stride_sample = (layout == 1) ? 1 : C; /* Distance between channels at same sample */ \ + \ + int64_t input_stride = (layout == 0) ? C : 1; \ + int64_t channel_step = (layout == 0) ? 1 : R; \ + \ + /* Determine max threads */ \ + int max_threads = (nThreads > 0) ? nThreads : omp_get_max_threads(); \ + \ + /* Parallel Loop */ \ + int64_t ch; \ + _Pragma("omp parallel for ordered num_threads(max_threads)") \ + for (ch = 0; ch < C; ch++) { \ + const T* ch_data = data + (ch * channel_step); \ + \ + /* Allocate buffers for this channel's output */ \ + /* Using malloc for buffers to avoid stack overflow */ \ + T* buffers[16]; \ + for(int i=0; i<16; i++) buffers[i] = NULL; \ + int64_t sizes[16]; \ + int64_t prev_len = R; \ + int alloc_failed = 0; \ + \ + for (int i = 0; i < nLevels; i++) { \ + int64_t out_len = prev_len / steps[i]; \ + /* We output Min/Max pairs, so 2 * out_len */ \ + sizes[i] = out_len; \ + if (out_len > 0) { \ + buffers[i] = (T*)malloc(out_len * 2 * sizeof(T)); \ + if (!buffers[i]) { alloc_failed = 1; break; } \ + } \ + prev_len = out_len; \ + } \ + \ + if (!alloc_failed) { \ + /* Compute L1 from Raw */ \ + if (sizes[0] > 0) { \ + int step = steps[0]; \ + T* out = buffers[0]; \ + int64_t count = sizes[0]; \ + for (int64_t i = 0; i < count; i++) { \ + T min_val = ch_data[i * step * input_stride]; \ + T max_val = min_val; \ + for (int j = 1; j < step; j++) { \ + T val = ch_data[(i * step + j) * input_stride]; \ + if (val < min_val) min_val = val; \ + if (val > max_val) max_val = val; \ + } \ + out[2*i] = min_val; \ + out[2*i+1] = max_val; \ + } \ + } \ + \ + /* Compute L2..Ln from previous level */ \ + for (int lvl = 1; lvl < nLevels; lvl++) { \ + if (sizes[lvl] > 0) { \ + int step = steps[lvl]; \ + T* prev_buf = buffers[lvl-1]; \ + T* out = buffers[lvl]; \ + int64_t count = sizes[lvl]; \ + for (int64_t i = 0; i < count; i++) { \ + T min_val = prev_buf[i * step * 2]; \ + T max_val = prev_buf[i * step * 2 + 1]; \ + for (int j = 1; j < step; j++) { \ + T p_min = prev_buf[(i * step + j) * 2]; \ + T p_max = prev_buf[(i * step + j) * 2 + 1]; \ + if (p_min < min_val) min_val = p_min; \ + if (p_max > max_val) max_val = p_max; \ + } \ + out[2*i] = min_val; \ + out[2*i+1] = max_val; \ + } \ + } \ + } \ + \ + /* Write to files sequentially */ \ + _Pragma("omp ordered") \ + { \ + for (int i = 0; i < nLevels; i++) { \ + if (sizes[i] > 0 && buffers[i]) { \ + fwrite(buffers[i], sizeof(T), sizes[i] * 2, files[i]); \ + } \ + } \ + } \ + } \ + \ + /* Cleanup buffers */ \ + for (int i = 0; i < nLevels; i++) { \ + if(buffers[i]) free(buffers[i]); \ + } \ + } \ + \ + /* Close files */ \ + for (int i = 0; i < nLevels; i++) fclose(files[i]); \ + return ret; \ +} + +// Instantiate workers +DEFINE_WORKER(int8_t, i8) +DEFINE_WORKER(uint8_t, u8) +DEFINE_WORKER(int16_t, i16) +DEFINE_WORKER(uint16_t, u16) +DEFINE_WORKER(int32_t, i32) +DEFINE_WORKER(uint32_t, u32) +DEFINE_WORKER(int64_t, i64) +DEFINE_WORKER(uint64_t, u64) +DEFINE_WORKER(float, f32) +DEFINE_WORKER(double, f64) + +// Master Dispatcher +int pyraview_process_chunk( + const void* dataArray, + int64_t numRows, + int64_t numCols, + int dataType, + int layout, + const char* filePrefix, + int append, + const int* levelSteps, + int numLevels, + double nativeRate, + int numThreads +) { + // 1. Validate inputs (basic) + if (!dataArray || !filePrefix || !levelSteps || numLevels <= 0 || numLevels > 16) return -1; + + // Validate levelSteps + for (int i=0; i +#include +#include +#include +#include + +// Macros for testing +#define ASSERT(cond, msg) if(!(cond)) { printf("FAIL: %s\n", msg); return 1; } + +// Test function +int run_test(int type, int layout, int channels, int threads) { + printf("Testing Type=%d, Layout=%d, Channels=%d, Threads=%d...", type, layout, channels, threads); + + int rows = 1000; + int cols = channels; + int data_size = rows * cols * 8; // Max size (double) + void* data = malloc(data_size); + memset(data, 0, data_size); + + // Fill with dummy data (linear ramp for verification if needed) + // For now just check execution success + + char prefix[256]; + sprintf(prefix, "test_c_T%d_L%d_C%d_Th%d", type, layout, channels, threads); + + // Cleanup previous + char fname[512]; + sprintf(fname, "%s_L1.bin", prefix); + remove(fname); + + int steps[] = {10}; + int ret = pyraview_process_chunk( + data, rows, cols, type, layout, + prefix, 0, steps, 1, 100.0, threads + ); + + free(data); + + if (ret != 0) { + printf("FAILED (ret=%d)\n", ret); + return 1; + } + + FILE* f = fopen(fname, "rb"); + if (!f) { + printf("FAILED (no file)\n"); + return 1; + } + + // Check header + PyraviewHeader h; + fread(&h, sizeof(h), 1, f); + fclose(f); + + if (h.dataType != (uint32_t)type) { + printf("FAILED (type mismatch)\n"); + return 1; + } + if (h.channelCount != (uint32_t)channels) { + printf("FAILED (channel count mismatch)\n"); + return 1; + } + + printf("OK\n"); + return 0; +} + +int main() { + printf("Running Comprehensive C Tests...\n"); + + int failures = 0; + + // Types: 0..9 + for (int t = 0; t <= 9; t++) { + // Layouts: 0 (SxC), 1 (CxS) + for (int l = 0; l <= 1; l++) { + // Channels: 1, 10 + int channels[] = {1, 10}; + for (int c_idx = 0; c_idx < 2; c_idx++) { + int c = channels[c_idx]; + // Threads: 0 (Auto), 2 + int threads[] = {0, 2}; + for (int th_idx = 0; th_idx < 2; th_idx++) { + int th = threads[th_idx]; + if (run_test(t, l, c, th) != 0) failures++; + } + } + } + } + + if (failures > 0) { + printf("\nTOTAL FAILURES: %d\n", failures); + return 1; + } + + printf("\nALL TESTS PASSED\n"); + return 0; +} diff --git a/src/matlab/README.md b/src/matlab/README.md new file mode 100644 index 0000000..2bd2a24 --- /dev/null +++ b/src/matlab/README.md @@ -0,0 +1,18 @@ +# Pyraview Matlab MEX + +## Compilation +To compile the MEX file: +1. Open Matlab and `cd` to this directory. +2. Run `build_pyraview`. + +## Usage +`status = pyraview_mex(data, prefix, steps, nativeRate, [append], [numThreads])` + +* `data`: Samples x Channels matrix (Single, Double, Int16, Uint8). +* `prefix`: Base file name (e.g. 'data/mydata'). +* `steps`: Vector of decimation factors (e.g. [100, 10, 10]). +* `nativeRate`: Original sampling rate. +* `append`: (Optional) Append to existing files. + +## Testing +Run `test_pyraview` to verify functionality. diff --git a/src/matlab/build_pyraview.m b/src/matlab/build_pyraview.m new file mode 100644 index 0000000..a08acb5 --- /dev/null +++ b/src/matlab/build_pyraview.m @@ -0,0 +1,23 @@ +% build_pyraview.m +% Build script for Pyraview MEX + +src_path = '../../src/c/pyraview.c'; +mex_src = 'pyraview_mex.c'; +include_path = '-I../../include'; + +% OpenMP flags (adjust for OS/Compiler) +if ispc + % Windows MSVC usually supports /openmp + omp_flags = 'COMPFLAGS="$COMPFLAGS /openmp"'; +else + % GCC/Clang + omp_flags = 'CFLAGS="$CFLAGS -fopenmp" LDFLAGS="$LDFLAGS -fopenmp"'; +end + +fprintf('Building Pyraview MEX...\n'); +try + mex('-v', include_path, src_path, mex_src, omp_flags); + fprintf('Build successful.\n'); +catch e + fprintf('Build failed: %s\n', e.message); +end diff --git a/src/matlab/pyraview_mex.c b/src/matlab/pyraview_mex.c new file mode 100644 index 0000000..91b1b79 --- /dev/null +++ b/src/matlab/pyraview_mex.c @@ -0,0 +1,139 @@ +#include "mex.h" +#include "../../include/pyraview_header.h" +#include + +/* + * pyraview_mex.c + * Gateway for Pyraview C Engine + * + * Usage: + * status = pyraview_mex(data, prefix, steps, nativeRate, [append], [numThreads]) + * + * Inputs: + * data: (Samples x Channels) matrix. uint8, int16, single, or double. + * prefix: char array (string). + * steps: double array of decimation factors (e.g. [100, 10]). + * nativeRate: double scalar. + * append: (optional) logical/scalar. Default false. + * numThreads: (optional) scalar. Default 0 (auto). + * + * Outputs: + * status: 0 on success. + */ + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + // Check inputs + if (nrhs < 4) { + mexErrMsgIdAndTxt("Pyraview:InvalidInput", "Usage: pyraview_mex(data, prefix, steps, nativeRate, [append], [numThreads])"); + } + + // 1. Data + const mxArray *mxData = prhs[0]; + void *dataPtr = mxGetData(mxData); + mwSize numRows = mxGetM(mxData); + mwSize numCols = mxGetN(mxData); + + // Determine type + mxClassID classID = mxGetClassID(mxData); + int dataType = -1; + switch (classID) { + case mxINT8_CLASS: dataType = PV_INT8; break; // 0 + case mxUINT8_CLASS: dataType = PV_UINT8; break; // 1 + case mxINT16_CLASS: dataType = PV_INT16; break; // 2 + case mxUINT16_CLASS: dataType = PV_UINT16; break; // 3 + case mxINT32_CLASS: dataType = PV_INT32; break; // 4 + case mxUINT32_CLASS: dataType = PV_UINT32; break; // 5 + case mxINT64_CLASS: dataType = PV_INT64; break; // 6 + case mxUINT64_CLASS: dataType = PV_UINT64; break; // 7 + case mxSINGLE_CLASS: dataType = PV_FLOAT32; break;// 8 + case mxDOUBLE_CLASS: dataType = PV_FLOAT64; break;// 9 + default: + mexErrMsgIdAndTxt("Pyraview:InvalidType", "Data type must be int8, uint8, int16, uint16, int32, uint32, int64, uint64, single, or double."); + } + + // Layout: Matlab is Column-Major. If input is Samples x Channels, then it is CxS. + // Layout code for CxS is 1. + int layout = 1; + + // 2. Prefix + if (!mxIsChar(prhs[1])) { + mexErrMsgIdAndTxt("Pyraview:InvalidInput", "Prefix must be a string."); + } + char *prefix = mxArrayToString(prhs[1]); + + // 3. Steps + if (!mxIsNumeric(prhs[2])) { + mexErrMsgIdAndTxt("Pyraview:InvalidInput", "Steps must be numeric."); + } + mwSize numSteps = mxGetNumberOfElements(prhs[2]); + int *levelSteps = (int*)mxMalloc(numSteps * sizeof(int)); + + if (mxGetClassID(prhs[2]) == mxDOUBLE_CLASS) { + double *dPtr = mxGetPr(prhs[2]); + for (size_t i = 0; i < numSteps; i++) levelSteps[i] = (int)dPtr[i]; + } else if (mxGetClassID(prhs[2]) == mxINT32_CLASS) { + int *iPtr = (int*)mxGetData(prhs[2]); + for (size_t i = 0; i < numSteps; i++) levelSteps[i] = iPtr[i]; + } else { + mxFree(levelSteps); + mxFree(prefix); + mexErrMsgIdAndTxt("Pyraview:InvalidInput", "Steps must be double or int32 array."); + } + + // 4. Native Rate + if (!mxIsDouble(prhs[3]) || mxGetNumberOfElements(prhs[3]) != 1) { + mxFree(levelSteps); + mxFree(prefix); + mexErrMsgIdAndTxt("Pyraview:InvalidInput", "NativeRate must be scalar double."); + } + double nativeRate = mxGetScalar(prhs[3]); + + // 5. Append (optional) + int append = 0; + if (nrhs >= 5) { + if (mxIsLogical(prhs[4]) || mxIsNumeric(prhs[4])) { + append = (int)mxGetScalar(prhs[4]); + } + } + + // 6. NumThreads (optional) + int numThreads = 0; + if (nrhs >= 6) { + numThreads = (int)mxGetScalar(prhs[5]); + } + + // Call Engine + // Note: We need to link against the engine. + int ret = pyraview_process_chunk( + dataPtr, + (int64_t)numRows, + (int64_t)numCols, + dataType, + layout, + prefix, + append, + levelSteps, + (int)numSteps, + nativeRate, + numThreads + ); + + // Cleanup + mxFree(prefix); + mxFree(levelSteps); + + // Return status + if (nlhs > 0) { + plhs[0] = mxCreateDoubleScalar((double)ret); + } + + if (ret < 0) { + // We might want to warn or error? + // The prompt says "return a specific error code". + // But throwing an error in MEX stops execution. + // It's better to return the code if the user wants to handle it, + // OR throw error. + // I'll throw error for now as it's safer. + mexErrMsgIdAndTxt("Pyraview:ExecutionError", "Engine returned error code %d", ret); + } +} diff --git a/src/matlab/test_pyraview.m b/src/matlab/test_pyraview.m new file mode 100644 index 0000000..076c2f3 --- /dev/null +++ b/src/matlab/test_pyraview.m @@ -0,0 +1,70 @@ +function tests = test_pyraview + tests = functiontests(localfunctions); +end + +function setupOnce(testCase) + % Verify MEX file exists + [~, mexName] = fileparts('pyraview'); + mexExt = mexext; + fullMexPath = fullfile(pwd, 'src', 'matlab', ['pyraview.' mexExt]); + + % If run via run-tests action, current folder might be repo root. + if ~exist(fullMexPath, 'file') + % Try relative to where this file is? + currentFileDir = fileparts(mfilename('fullpath')); + fullMexPath = fullfile(currentFileDir, ['pyraview.' mexExt]); + if ~exist(fullMexPath, 'file') + error('MEX file not found: %s', fullMexPath); + end + addpath(currentFileDir); + else + addpath(fileparts(fullMexPath)); + end + + fprintf('Using MEX: %s\n', fullMexPath); +end + +function test_comprehensive(testCase) + types = {'int8', 'uint8', 'int16', 'uint16', 'int32', 'uint32', 'int64', 'uint64', 'single', 'double'}; + bytes_per_type = [1, 1, 2, 2, 4, 4, 8, 8, 4, 8]; + channels_list = [1, 10]; + + for t = 1:length(types) + for c = 1:length(channels_list) + type_str = types{t}; + n_ch = channels_list(c); + bpt = bytes_per_type(t); + + fprintf('Testing %s, %d channels...', type_str, n_ch); + + data = cast(zeros(1000, n_ch), type_str); + prefix = sprintf('test_matlab_%s_%d', type_str, n_ch); + outfile = [prefix '_L1.bin']; + + % Ensure cleanup happens even on failure + c = onCleanup(@() cleanupFile(outfile)); + + try + status = pyraview(data, prefix, [10], 1000.0); + testCase.verifyEqual(status, 0, 'Status should be 0'); + + testCase.verifyTrue(exist(outfile, 'file') == 2, 'Output file should exist'); + + d = dir(outfile); + expected_size = 1024 + (1000/10) * 2 * n_ch * bpt; + testCase.verifyEqual(d.bytes, expected_size, 'File size mismatch'); + + fprintf('OK\n'); + catch e + fprintf('FAILED: %s\n', e.message); + rethrow(e); + end + end + end +end + +function cleanupFile(filename) + if exist(filename, 'file') + delete(filename); + end +end diff --git a/src/python/pyproject.toml b/src/python/pyproject.toml new file mode 100644 index 0000000..7d52f33 --- /dev/null +++ b/src/python/pyproject.toml @@ -0,0 +1,11 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pyraview" +version = "0.1.0" +description = "Pyraview Wrapper" +authors = [{name = "Jules"}] +dependencies = ["numpy"] +requires-python = ">=3.7" diff --git a/src/python/pyraview.py b/src/python/pyraview.py new file mode 100644 index 0000000..28badd0 --- /dev/null +++ b/src/python/pyraview.py @@ -0,0 +1,137 @@ +import os +import ctypes +import numpy as np +import sys + +# Try to find the shared library +def _find_library(): + # Priority: + # 1. Environment variable PYRAVIEW_LIB + # 2. Relative to this file: ../c/libpyraview.so (dev structure) + # 3. Current working directory: ./libpyraview.so + + lib_name = "libpyraview.so" + if sys.platform == "win32": + lib_name = "pyraview.dll" + elif sys.platform == "darwin": + lib_name = "libpyraview.dylib" + + env_path = os.environ.get("PYRAVIEW_LIB") + if env_path and os.path.exists(env_path): + return env_path + + # Relative to this file + this_dir = os.path.dirname(os.path.abspath(__file__)) + rel_path = os.path.join(this_dir, "..", "c", lib_name) + if os.path.exists(rel_path): + return rel_path + + cwd_path = os.path.join(os.getcwd(), lib_name) + if os.path.exists(cwd_path): + return cwd_path + + # If not found, try loading by name (if in system path) + return lib_name + +_lib_path = _find_library() +try: + _lib = ctypes.CDLL(_lib_path) +except OSError: + raise ImportError(f"Could not load Pyraview library at {_lib_path}") + +# Define types +_lib.pyraview_process_chunk.argtypes = [ + ctypes.c_void_p, # dataArray + ctypes.c_int64, # numRows + ctypes.c_int64, # numCols + ctypes.c_int, # dataType + ctypes.c_int, # layout + ctypes.c_char_p, # filePrefix + ctypes.c_int, # append + ctypes.POINTER(ctypes.c_int), # levelSteps + ctypes.c_int, # numLevels + ctypes.c_double, # nativeRate + ctypes.c_int # numThreads +] +_lib.pyraview_process_chunk.restype = ctypes.c_int + +def process_chunk(data, file_prefix, level_steps, native_rate, append=False, layout='SxC', num_threads=0): + """ + Process a chunk of data and append to pyramid files. + + Args: + data (np.ndarray): Input data (2D). Rows=Samples, Cols=Channels (if SxC). + file_prefix (str): Base name for output files (e.g. "data/myfile"). + level_steps (list[int]): Decimation factors for each level (e.g. [100, 10, 10]). + native_rate (float): Original sampling rate. + append (bool): If True, append to existing files. If False, create new. + layout (str): 'SxC' (Sample-Major) or 'CxS' (Channel-Major). Default 'SxC'. + num_threads (int): Number of threads (0 for auto). + + Returns: + int: 0 on success, negative on error. + """ + if not isinstance(data, np.ndarray): + raise TypeError("Data must be a numpy array") + + if data.ndim != 2: + raise ValueError("Data must be 2D") + + # Determine layout + layout_code = 0 + if layout == 'SxC': + layout_code = 0 + num_rows, num_cols = data.shape + elif layout == 'CxS': + layout_code = 1 + num_cols, num_rows = data.shape + else: + raise ValueError("Layout must be 'SxC' or 'CxS'") + + # Determine data type code + dtype_map = { + np.dtype('int8'): 0, + np.dtype('uint8'): 1, + np.dtype('int16'): 2, + np.dtype('uint16'): 3, + np.dtype('int32'): 4, + np.dtype('uint32'): 5, + np.dtype('int64'): 6, + np.dtype('uint64'): 7, + np.dtype('float32'): 8, + np.dtype('float64'): 9 + } + + if data.dtype not in dtype_map: + raise TypeError(f"Unsupported data type: {data.dtype}. Supported: int8, uint8, int16, uint16, int32, uint32, int64, uint64, float32, float64") + + data_type_code = dtype_map[data.dtype] + + # Prepare C arguments + c_level_steps = (ctypes.c_int * len(level_steps))(*level_steps) + c_prefix = file_prefix.encode('utf-8') + + # Ensure data is contiguous in memory + if not data.flags['C_CONTIGUOUS']: + data = np.ascontiguousarray(data) + + data_ptr = data.ctypes.data_as(ctypes.c_void_p) + + ret = _lib.pyraview_process_chunk( + data_ptr, + ctypes.c_int64(num_rows), + ctypes.c_int64(num_cols), + ctypes.c_int(data_type_code), + ctypes.c_int(layout_code), + c_prefix, + ctypes.c_int(1 if append else 0), + c_level_steps, + ctypes.c_int(len(level_steps)), + ctypes.c_double(native_rate), + ctypes.c_int(num_threads) + ) + + if ret < 0: + raise RuntimeError(f"Pyraview processing failed with code {ret}") + + return ret diff --git a/src/python/tests/test_header_types.py b/src/python/tests/test_header_types.py new file mode 100644 index 0000000..df7787f --- /dev/null +++ b/src/python/tests/test_header_types.py @@ -0,0 +1,50 @@ +import numpy as np +import os +import sys +import struct + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import pyraview + +def test_header_type(): + prefix = "test_header" + + # Test cases: (numpy_dtype, expected_enum_value) + cases = [ + (np.int8, 0), + (np.uint8, 1), + (np.int16, 2), + (np.uint16, 3), + (np.int32, 4), + (np.uint32, 5), + (np.int64, 6), + (np.uint64, 7), + (np.float32, 8), + (np.float64, 9), + ] + + for dtype, expected_code in cases: + print(f"Testing {dtype} -> {expected_code}") + data = np.zeros((100, 1), dtype=dtype) + # Cleanup + outfile = f"{prefix}_L1.bin" + if os.path.exists(outfile): + os.remove(outfile) + + pyraview.process_chunk(data, prefix, [10], 100.0) + + with open(outfile, 'rb') as f: + # magic: 4, version: 4, dataType: 4 + f.seek(8) + code_bytes = f.read(4) + code = struct.unpack(' 200 values per channel. + # Total 400 values in file. + data = np.random.rand(1000, 2).astype(np.float32) + steps = [10] + ret = pyraview.process_chunk(data, self.prefix, steps, 1000.0) + self.assertEqual(ret, 0) + + outfile = f"{self.prefix}_L1.bin" + self.assertTrue(os.path.exists(outfile)) + + # Verify size + # Header 1024 + 2 channels * 100 samples * 2 (min/max) * 4 bytes + # 1024 + 1600 = 2624 bytes. + size = os.path.getsize(outfile) + self.assertEqual(size, 2624) + + def test_values_correctness(self): + # Create predictable data + # Ch0: 0..99. Min/Max of 0..9 is 0,9. 10..19 is 10,19. + data = np.zeros((100, 1), dtype=np.int16) + for i in range(100): + data[i, 0] = i + + steps = [10] + pyraview.process_chunk(data, self.prefix, steps, 100.0) + + # Read back + outfile = f"{self.prefix}_L1.bin" + with open(outfile, 'rb') as f: + f.seek(1024) + raw = f.read() + arr = np.frombuffer(raw, dtype=np.int16) + + # Should be 10 pairs. Total 20 values. + self.assertEqual(len(arr), 20) + # 0: 0, 9 + # 1: 10, 19 + self.assertEqual(arr[0], 0) + self.assertEqual(arr[1], 9) + self.assertEqual(arr[2], 10) + self.assertEqual(arr[3], 19) + + def test_append(self): + data = np.zeros((100, 1), dtype=np.uint8) + steps = [10] + # First chunk + ret1 = pyraview.process_chunk(data, self.prefix, steps, 100.0, append=False) + self.assertEqual(ret1, 0) + + # Second chunk + ret2 = pyraview.process_chunk(data, self.prefix, steps, 100.0, append=True) + self.assertEqual(ret2, 0) + + outfile = f"{self.prefix}_L1.bin" + size = os.path.getsize(outfile) + # Header + 2 chunks * (100/10 * 2 values * 1 byte) = 1024 + 40 = 1064 + self.assertEqual(size, 1064) + +if __name__ == '__main__': + unittest.main() diff --git a/toolboxPackaging.prj b/toolboxPackaging.prj new file mode 100644 index 0000000..ce2d8e4 --- /dev/null +++ b/toolboxPackaging.prj @@ -0,0 +1,117 @@ + + + Pyraview + Jules + jules@example.com + + High-performance Pyraview engine wrapper. + Pyraview toolbox for generating multi-resolution min/max pyramids. + + 1.0 + ${PROJECT_ROOT}/Pyraview.mltbx + + + + + + + true + + + + + + + + + false + + + + + + false + true + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${PROJECT_ROOT}/src/matlab + + + ${PROJECT_ROOT}/src/matlab/README.md + ${PROJECT_ROOT}/src/matlab/pyraview_mex.c + + + + + + + /home/runner/work/Pyraview/Pyraview/Pyraview.mltbx + + + + /usr/local/MATLAB/R2024b + + + + true + false + false + false + false + false + true + false + 6.5.0-1014-azure + false + true + glnxa64 + true + + +