Improve Channelised Data Quality (lofar_cli_stokes.c)

CMakeLists.txt

@@ -1,5 +1,5 @@
 # Based on what I learnt from https://cliutils.gitlab.io/modern-cmake
-cmake_minimum_required(VERSION 3.14...3.19)
+cmake_minimum_required(VERSION 3.28)
 
 # Create a release output by default
 set(default_build_type "Release")
@@ -33,8 +33,14 @@ set(UPM_CLI_VERSION 0.9)
 
 message("Building udpPacketManager Version ${PROJECT_VERSION}\n\n")
 
-set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -march=native -O2 -ffast-math")
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -march=native -O2 -ffast-math")
+if (NOT "$ENV{CPU_OPTS}" STREQUAL "")
+       set(CPU_OPTS "$ENV{CPU_OPTS}" CACHE STRING "Set from ENV")
+else()
+       set(CPU_OPTS "-march=native" CACHE STRING "Set by default")
+endif()
+
+set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${CPU_OPTS} -O2 -ffast-math")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CPU_OPTS} -O2 -ffast-math")
 
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Wstrict-prototypes -Wmissing-prototypes -Wframe-larger-than=4095 -Wshadow -Wconversion -Wno-sign-conversion -Wall -Wextra -g3")
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wframe-larger-than=4095 -Wshadow -Wconversion -Wno-sign-conversion -Wall -Wextra -g3")
@@ -77,9 +83,14 @@ add_custom_target(install_python_requirements
 message("Configuring OpenMP...")
 find_package(OpenMP REQUIRED)
 
-execute_process(COMMAND bash -c "cat /proc/cpuinfo | uniq | grep -m 2 \"siblings\" | cut -d \":\" -f 2 | sort --numeric --unique | xargs echo"
+execute_process(COMMAND bash -c "cat /proc/cpuinfo | uniq | grep -m 2 \"siblings\" | cut -d \":\" -f 2 | sort --numeric --unique | head -n 1"
                 OUTPUT_VARIABLE OMP_THREADS
+                RESULT_VARIABLE CPU_COUNT_RET
 )
+if ("${OMP_THREADS}" LESS_EQUAL 1 OR "${OMP_THREADS} " STREQUAL " ")
+	   message("WARNING: Failed to automatically determine a reasonable number of default threads, setting default to 8")
+       set(OMP_THREADS 8)
+endif()
 
 #if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
 if (OpenMP_C_LIB_NAMES MATCHES ".*(gomp).*")
@@ -99,7 +110,8 @@ set(ZSTD_BUILD_STATIC ON CACHE INTERNAL "")
 set(ZSTD_BUILD_SHARED OFF CACHE INTERNAL "")
 FetchContent_Declare(zstd
                      GIT_REPOSITORY https://github.com/facebook/zstd.git
-                     GIT_TAG v1.5.5
+                     GIT_TAG v1.5.7
+					 EXCLUDE_FROM_ALL
 )
 FetchContent_MakeAvailable(zstd)
 # MakeAvailable doesn't allow for libzstd_static to be used as a target
@@ -125,8 +137,9 @@ ExternalProject_Add(internal_zlib
                     CONFIGURE_COMMAND ./configure --static --prefix=./
                     BUILD_COMMAND make test install -j8
                     BUILD_IN_SOURCE TRUE # Keep source beside install prefix
-                    INSTALL_COMMAND ""
+                    INSTALL_COMMAND cmake -E echo "Skipping zlib install step"
                     UPDATE_COMMAND ""
+					BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/internal_zlib-prefix/src/internal_zlib/libz.a
 )
 ExternalProject_Get_Property(internal_zlib INSTALL_DIR)
 set(zlib_INSTALL_DIR ${INSTALL_DIR})
@@ -135,14 +148,14 @@ set_property(TARGET libz PROPERTY IMPORTED_LOCATION ${zlib_INSTALL_DIR}/src/inte
 add_dependencies(libz internal_zlib)
 message("")
 ExternalProject_Add(internal_hdf5
-                    GIT_REPOSITORY https://github.com/HDFGroup/hdf5
-                    GIT_TAG hdf5-1_12_2
-                    CONFIGURE_COMMAND ./configure --disable-dependency-tracking --enable-optimization=high --disable-shared --disable-cxx --disable-hl --disable-hltools --disable-tools --enable-threadsafe --disable-fortran --enable-build-mode=production --with-zlib=${zlib_INSTALL_DIR}/src/internal_zlib/
+                    URL https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_12_3.zip
+                    CONFIGURE_COMMAND ./configure --disable-dependency-tracking --enable-optimization=high --disable-shared --disable-cxx --disable-hl --disable-tools --enable-threadsafe --disable-fortran --enable-build-mode=production --with-zlib=${zlib_INSTALL_DIR}/src/internal_zlib/
                     BUILD_COMMAND make all -j8
                     BUILD_IN_SOURCE TRUE # Header gets moved to incorrect folder otherwise
-                    INSTALL_COMMAND ""
+                    INSTALL_COMMAND cmake -E echo "Skipping hdf5 install step"
                     UPDATE_COMMAND ""
                     DEPENDS internal_zlib
+					BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/internal_hdf5-prefix/src/internal_hdf5/src/.libs/libhdf5.a
 )
 ExternalProject_Get_Property(internal_hdf5 INSTALL_DIR)
 set(hdf5_INSTALL_DIR ${INSTALL_DIR})
@@ -158,9 +171,10 @@ ExternalProject_Add(internal_bitshuffle
                     CONFIGURE_COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/src/misc/bitshuffleMakefile Makefile
                     BUILD_COMMAND CMAKE_BASE_DIR=${CMAKE_CURRENT_BINARY_DIR} make link -j4
                     BUILD_IN_SOURCE TRUE
-                    INSTALL_COMMAND ""
+                    INSTALL_COMMAND cmake -E echo "Skipping bitshuffle install step"
                     UPDATE_COMMAND ""
                     DEPENDS internal_hdf5 zstd internal_zlib
+					BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/internal_bitshuffle-prefix/src/internal_bitshuffle/libh5bshuf.a
 )
 ExternalProject_Get_Property(internal_bitshuffle INSTALL_DIR)
 set(bitshuffle_INSTALL_DIR ${INSTALL_DIR})
@@ -171,21 +185,13 @@ message("")
 
 
 message ("Configuring PSRDADA")
-# Find the PSRDADA libraries on the system (or set NODADA if not found)
-ExternalProject_Add(internal_PSRDADA
-                    GIT_REPOSITORY git://git.code.sf.net/p/psrdada/code
-                    GIT_TAG ba2b88
-                    BUILD_IN_SOURCE TRUE
-                    CONFIGURE_COMMAND ./bootstrap && ./configure --with-cuda-dir=no && cd 3rdparty && make libtimers.la # libtimers fails to compile during normal step with older Make versions
-                    BUILD_COMMAND cd src/ && make libpsrdada.la dada_db dada_dbmeminfo dada_dbmetric dada_dbmonitor -j4
-                    INSTALL_COMMAND ""
-                    UPDATE_COMMAND ""
+# Disable CUDA for the PSRDADA compile
+FetchContent_Declare(psrdada_src
+					 GIT_REPOSITORY https://github.com/mirror-psrdada/mirror-psrdada-code.git
+					 GIT_TAG 1e48b65e90b279683134f91395668b38e58f7646
+					 EXCLUDE_FROM_ALL
 )
-ExternalProject_Get_Property(internal_PSRDADA INSTALL_DIR)
-set(psrdada_INSTALL_DIR ${INSTALL_DIR})
-add_library(libpsrdada STATIC IMPORTED)
-set_property(TARGET libpsrdada PROPERTY IMPORTED_LOCATION ${psrdada_INSTALL_DIR}/src/internal_PSRDADA/src/.libs/libpsrdada.a)
-add_dependencies(libpsrdada internal_PSRDADA)
+FetchContent_MakeAvailable(psrdada_src)
 message("")
 
 
@@ -199,8 +205,9 @@ ExternalProject_ADD(internal_FFTW3F
                     CONFIGURE_COMMAND ./configure --disable-dependency-tracking --enable-float --enable-openmp --enable-static --enable-sse2 --enable-avx --enable-avx2 --enable-avx512 --enable-avx-128-fma
                     BUILD_COMMAND make -j4
                     BUILD_IN_SOURCE TRUE
-                    INSTALL_COMMAND ""
+                    INSTALL_COMMAND cmake -E echo "Skipping FFTW3 install step"
                     UPDATE_COMMAND ""
+					BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/internal_FFTW3F-prefix/src/internal_FFTW3F/threads/.libs/libfftw3f_omp.a ${CMAKE_CURRENT_BINARY_DIR}/internal_FFTW3F-prefix/src/internal_FFTW3F/.libs/libfftw3f.a
 )
 ExternalProject_Get_Property(internal_FFTW3F install_dir)
 set(fftw3f_INSTALL_DIR ${install_dir})
@@ -241,35 +248,35 @@ add_library(lofudpman STATIC
             src/lib/lofar_udp_io.c
             src/lib/lofar_udp_reader.c
             src/lib/lofar_udp_structs.c
-            src/CLI/lofar_cli_meta.c
             src/lib/lofar_udp_metadata.c
-            src/lib/lofar_udp_time.c)
+            src/lib/lofar_udp_time.c
+			src/CLI/lofar_cli_meta.c)
 
-add_dependencies(lofudpman libzstd_static libpsrdada libhdf5 libz libh5bshuf install_python_requirements) # libfftw3fomp) ##yaml) #CSpice::cspice)
+
+add_dependencies(lofudpman libzstd_static psrdada libhdf5 libz libh5bshuf install_python_requirements dada_db dada_dbmeminfo dada_dbmetric dada_dbmonitor) # libfftw3fomp) ##yaml) #CSpice::cspice)
 
 # Include all of our library headers
 target_include_directories(lofudpman PUBLIC
        ${CMAKE_CURRENT_SOURCE_DIR}/src/lib
        ${CMAKE_CURRENT_BINARY_DIR}/src/lib # Configured headers
-       ${CMAKE_CURRENT_SOURCE_DIR}/src/CLI
+#       ${CMAKE_CURRENT_SOURCE_DIR}/src/CLI
        ${CMAKE_CURRENT_BINARY_DIR}/src/CLI # Configured headers
        ${CMAKE_CURRENT_SOURCE_DIR}/src/metadata
-
 )
 # Include all of our dependency headers
 target_include_directories(lofudpman PUBLIC ${fftw3f_INSTALL_DIR}/src/internal_FFTW3F/api)
 target_include_directories(lofudpman PUBLIC ${hdf5_INSTALL_DIR}/src/internal_hdf5/src/ ${zlib_INSTALL_DIR}/src/internal_zlib/ ${bitshuffle_INSTALL_DIR}/src/internal_bitshuffle/src/)
-target_include_directories(lofudpman PUBLIC ${psrdada_INSTALL_DIR}/src/internal_PSRDADA/src/)
+target_include_directories(lofudpman PUBLIC ${psrdada_src_SOURCE_DIR}/src/)
 target_include_directories(lofudpman PUBLIC ${zstd_SOURCE_DIR}/lib/)
 #target_include_directories(lofudpman PUBLIC ${yaml_SOURCE_DIR}/include/)
 
-file(GLOB_RECURSE UDP_INCLUDE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/*/*.h" "${CMAKE_CURRENT_SOURCE_DIR}/src/*/*.hpp" "${CMAKE_CURRENT_BINARY_DIR}/src/*/*.h" "${CMAKE_CURRENT_BINARY_DIR}/src/*/*.hpp")
+file(GLOB_RECURSE UDP_INCLUDE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/lib/*.h" "${CMAKE_CURRENT_SOURCE_DIR}/src/lib/*.hpp" "${CMAKE_CURRENT_BINARY_DIR}/src/lib/*.h" "${CMAKE_CURRENT_BINARY_DIR}/src/lib/*.hpp" "${CMAKE_CURRENT_BINARY_DIR}/src/CLI/*h")
 
 # FFTW requires a libm (math) link
 find_library(LIB_M m REQUIRED)
 target_link_libraries(lofudpman PUBLIC libfftw3f_omp libfftw3f ${LIB_M})
 target_link_libraries(lofudpman PUBLIC libhdf5 libz libh5bshuf ${CMAKE_DL_LIBS}) # Some cases need libdl for HDF5
-target_link_libraries(lofudpman PUBLIC libpsrdada)
+target_link_libraries(lofudpman PUBLIC psrdada)
 target_link_libraries(lofudpman PUBLIC libzstd_static)
 
 find_library(LIB_RT rt REQUIRED) # Required for shmem functions
@@ -293,8 +300,8 @@ target_link_libraries(lofudpman PUBLIC OpenMP::OpenMP_CXX OpenMP::OpenMP_C)
 # Setup the CLIs
 add_executable(lofar_udp_extractor ${CMAKE_CURRENT_SOURCE_DIR}/src/CLI/lofar_cli_extractor.c)
 add_executable(lofar_stokes_extractor ${CMAKE_CURRENT_SOURCE_DIR}/src/CLI/lofar_cli_stokes.c)
-target_link_libraries(lofar_udp_extractor PUBLIC lofudpman)
-target_link_libraries(lofar_stokes_extractor PUBLIC lofudpman)
+target_link_libraries(lofar_udp_extractor PUBLIC lofudpman lofudpman)
+target_link_libraries(lofar_stokes_extractor PUBLIC lofudpman lofudpman)
 
 
 include(CMakePackageConfigHelpers)
@@ -312,8 +319,14 @@ install(TARGETS lofudpman lofar_udp_extractor lofar_stokes_extractor
 )
 install(FILES ${UDP_INCLUDE_FILES} DESTINATION include)
 install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/src/misc/dreamBeamJonesGenerator.py DESTINATION bin)
+install(PROGRAMS
+        ${psrdada_src_BINARY_DIR}/apps/dada_db
+        ${psrdada_src_BINARY_DIR}/apps/dada_dbmeminfo
+        ${psrdada_src_BINARY_DIR}/apps/dada_dbmetric
+        ${psrdada_src_BINARY_DIR}/apps/dada_dbmonitor
+        DESTINATION bin)
 
 # Add the tests directory
 enable_testing()
 set(TEST_ENV_PATH ${CMAKE_CURRENT_SOURCE_DIR}/tests/lib_tests/)
-add_subdirectory(tests)
+add_subdirectory(tests)

build.sh

@@ -1,13 +1,6 @@
-rm -rf ./build
-mkdir build
-cd build
+#!/usr/bin/env bash
+set -e
 
-echo "Preparing build with CC=${CC} CXX=${CXX} LD=${LD}"
-cmake -DCMAKE_BUILD_TYPE=Release ..
-cmake --build . --target all --config Release
-
-cmake --install .
-
-if [[ $1 -eq 1 ]]; then
-  ctest -V .
-fi
+bash ./build_prep.sh "${1}"
+bash ./build_compile.sh "${1}"
+bash ./build_install.sh "${1}"

build_compile.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+cd build
+
+if [[ $1 -eq 2 ]]; then
+  export CPU_OPTS="-march=x86-64"
+fi
+
+cmake --build . --target all --config Release

build_docker.sh

@@ -14,7 +14,7 @@ echo "Building Docker image with tag lofar-upm:latest from $LOCAL_DIR"
 docker build -f src/docker/Dockerfile -t lofar-upm:latest .
 
 if [ "$#" -eq 2 ]; then
-  echo "Building singulairty image, outputting in directory $1"
+  echo "Building singularity image, outputting in directory $1"
   touch "$1"/test
 
   if [ "$?" -ne 0 ]; then

build_install.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -e
+
+cd build
+cmake --install .
+exitCode="$?"
+
+if [[ $1 -gt 0 ]]; then
+  ctest -V .
+  exitTest="$?"
+  if [[ $exitCode -eq 0 ]]; then
+    exitCode="${exitTest}"
+  fi
+fi
+
+exit ${exitCode}

build_prep.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+rm -rf ./build
+
+set -e
+
+mkdir build
+cd build
+
+if [[ $1 -eq 2 ]]; then
+  export CPU_OPTS="-march=x86-64"
+fi
+
+echo "Preparing build with CC=${CC} CXX=${CXX}"
+cmake -DCMAKE_BUILD_TYPE=Release ..

docs/ARCHITECTURE.md

@@ -0,0 +1,53 @@
+Architecture
+============
+
+This document is a high-level overview of the udpPacketManager library's internal workings, call tree and some of the unexpected design choices in parts of the library.
+
+The repo consists of four main components:
+- [The library itself](#the-main-library)
+- A mini [library of metadata writers](#metadata-writers)
+- An [external Python executable](#python---dreambeam-wrapper) for interfacing with [dreamBeam](https://github.com/2baOrNot2ba/dreamBeam) to generate Jones matrices for calibration
+- [A set of reference CLIs](#the-clis), which can also be used to process data
+
+# The Main Library
+Interfacing with the library from a user perspective should only require three function calls, to setup a library for processing, to processed the input and then finally to cleanup any memory allocated for processing. 
+
+These are the functions
+- lofar_udp_reader_setup()
+- lofar_udp_reader_step() or lofar_udp_reader_step_timed()
+- lofar_udp_reader_cleanup()
+
+All of these functions can be found in the [lofar_udp_reader.c](../src/lib/lofar_udp_reader.c) file.
+
+
+
+To save on compute time, while the library does have a verbose output, it is only enabled in debug builds. It can be forcibly enabled in all builds by appending `-DVERBOSE` to your `CFLAGS` enviroment variable.
+
+### lofar_udp_reader_setup()
+The *lofar_udp_reader_setup* function takes in a configuration struct defined in [lofar_udp_structs.c](../src/lib/lofar_udp_structs.h), opens the input datastreams, reads in data and populates as much metadata as it can from the input packet stream. It returns the pointer to a *lofar_udp_reader* struct which contains all of the informaiton and alocated buffers required to read in new data and re-order it into a set output.
+
+Calling this function will read in a standard gulp of data from the set number of ports, find the lowest common packet after a given starting point that is present on all ports of data and align them. This function does not re-order the data into an output, this will only occur after the first *lofar_udp_reader_step*-like function is called.
+
+
+### lofar_udp_reader_step()
+This function performs a combined call to the read function in [`lofar_udp_io.c`](../src/lib/lofar_udp_io.c) to get new input data and then passes that input data to the C++ function in [lofar_udp_backends.cpp](../src/lib/lofar_udp_backends.cpp), which handles re-ordering the data to the desired output.
+
+In the case that there is large packet loss, a significant number of out-of-order packets or the input returns an end of stream marker, this function will return a non-zero value.
+
+### lofar_udp_reader_cleanup()
+This function calls `free` on all dynamically allocated memory, closes input data streams and generals cleans up after the library.
+
+
+### Implementation details
+The input read buffers are padded in two cases:
+- They are always extended by 2 packets to hold both a reference packet from a previous iteration, and a blank packet for the case that packets are not replayed when packets are dropped
+- If the input is from a zstd compressed file, the decompression system expects a given output size, so we need to extend the input buffer size to align with the expected size (only needed in cases where the packets per iteration isn't a multiple of 2)
+
+# Metadata Writers
+
+
+
+# Python - dreamBeam Wrapper
+
+# The CLIs
+The CLIs act as reference on how to use the library, as well as an interface to process an observation from raw CEP packets to (un)calibrated voltages or Stokes outputs, with optional metadata attached.

docs/README_CLI.md

@@ -334,3 +334,22 @@ has an example implementation of extended downsampling for modes (10, 20).
 
 - Take the input data, apply (10, 20 or 30) and (\*\*0) to form a Stokes \* sample, and sum it with the next sample
 - N input files -> 1 output file (16x less output samples)
+
+
+### Split Polarisation Power Modes
+
+An output of 2 files, containing the power for the X and Y polarisations has been added to test beam models.
+
+#### \*70: "Split Polarisation Powers"
+
+- Take the input data, apply (10, 20 or 30), and then combine the polarisation real/imaging values to form 2 output 
+32-bit floating point (power of X, Y) filterbanks for each frequency sample
+- N input files -> 2 output files
+
+#### \*7\*: "Split Polarisation Powers with Nx downsampling"
+
+- Follows the same rules as the Stokes downsampling described above, using 1, 2, 3, 4 as the final value of the
+processing mode to give 2^n downsampling.
+- Take the input data, apply (10, 20 or 30), and then combine the polarisation real/imaging values, sum 2^n times to
+generate 2 downsampled filterbanks.
+- N input files -> 2 output files

docs/README_RAW_DATA_HANDLING.md

@@ -0,0 +1,16 @@
+Using the CLI and Plotting the Outputs
+======================================
+
+This will be a summary of how to process the outputs of this program in one of two ways in Python: raw data handling with numpy (applies to all processing modes), or via [sigpyproc](https://github.com/pravirkr/sigpyproc). The numpy approach works best for modes below 100 (non-stokes outputs), though these modes can also be handled through sigpyproc if you are familiar with the indiexing you are dealing with.
+
+While all input, intermediate and output arrays are internally handed as chars (8-bit words), you will need to be aware of your data type when processing the outputs of the CLI. For modes below 100, the the output word size is the same as the input word size. However, for modes above 100 the output will be a float that is 4 times wider than the input, and the type is changed to a float. So an 8-bit input will generate a 32-bit floating point value (float) and 16-bit will generate a 64-bit floating point value (double).
+
+Using the CLI
+=============
+
+The Numpy Approach
+==================
+
+
+The SigPyProc Approach
+======================