From 81bad82c117e122077be532d26985f0c4b1342a5 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 22 Aug 2024 16:02:23 +0200 Subject: [PATCH 01/65] First draft of the skycleaver implementation --- cmake/compiler_settings.cmake | 4 +- cpp/skyweaver/CMakeLists.txt | 12 + cpp/skyweaver/DescribedVector.hpp | 32 +- cpp/skyweaver/MultiBeamWriter.hpp | 43 -- cpp/skyweaver/MultiFileReader.cuh | 5 + cpp/skyweaver/MultiFileWriter.cuh | 50 ++- cpp/skyweaver/ObservationHeader.hpp | 50 ++- cpp/skyweaver/SigprocFileWriter.hpp | 2 +- cpp/skyweaver/SigprocHeader.hpp | 32 +- cpp/skyweaver/SkyCleaver.cuh | 77 ++++ cpp/skyweaver/SkyCleaverConfig.hpp | 71 ++++ cpp/skyweaver/detail/MultiFileWriter.cu | 173 +++----- cpp/skyweaver/detail/SigprocHeader.cpp | 2 +- .../detail/file_writer_callbacks.cpp | 301 +++++++++++++ cpp/skyweaver/src/CoherentDedisperser.cu | 2 +- cpp/skyweaver/src/IncoherentDedisperser.cu | 2 +- cpp/skyweaver/src/MultiFileReader.cu | 18 + cpp/skyweaver/src/ObservationHeader.cpp | 72 ++-- cpp/skyweaver/src/SigprocHeader.cpp | 80 +++- cpp/skyweaver/src/SkyCleaver.cu | 399 ++++++++++++++++++ cpp/skyweaver/src/skycleaver_cli.cu | 228 ++++++++++ cpp/skyweaver/src/skyweaver_cli.cu | 58 ++- .../test/src/BeamformerPipelineTester.cu | 6 +- .../test/src/MultiFileWriterTester.cu | 8 +- 24 files changed, 1469 insertions(+), 258 deletions(-) delete mode 100644 cpp/skyweaver/MultiBeamWriter.hpp create mode 100644 cpp/skyweaver/SkyCleaver.cuh create mode 100644 cpp/skyweaver/SkyCleaverConfig.hpp create mode 100644 cpp/skyweaver/detail/file_writer_callbacks.cpp create mode 100644 cpp/skyweaver/src/SkyCleaver.cu create mode 100644 cpp/skyweaver/src/skycleaver_cli.cu diff --git a/cmake/compiler_settings.cmake b/cmake/compiler_settings.cmake index 410544e..879cf70 100644 --- a/cmake/compiler_settings.cmake +++ b/cmake/compiler_settings.cmake @@ -5,8 +5,10 @@ if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RELEASE") endif () +#set(CMAKE_VERBOSE_MAKEFILE 1) + # Set compiler flags -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fopenmp") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fopenmp -std=gnu++20") set(ARCH "broadwell" CACHE STRING "target architecture (-march=native, x86-64), defautls to broadwell") diff --git a/cpp/skyweaver/CMakeLists.txt b/cpp/skyweaver/CMakeLists.txt index 58d074f..0f49287 100644 --- a/cpp/skyweaver/CMakeLists.txt +++ b/cpp/skyweaver/CMakeLists.txt @@ -18,6 +18,9 @@ set(skyweaver_src src/Timer.cpp src/Transposer.cu src/WeightsManager.cu + src/SkyCleaver.cu + src/SigprocHeader.cpp + ) set(skyweaver_inc @@ -38,6 +41,8 @@ set(skyweaver_inc Timer.hpp Transposer.cuh WeightsManager.cuh + MultiFileWriter.cuh + SigprocHeader.hpp ) set(SKYWEAVER_LIBRARIES ${CMAKE_PROJECT_NAME} ${DEPENDENCY_LIBRARIES}) @@ -106,6 +111,13 @@ target_link_libraries(skyweavercpp OpenMP::OpenMP_CXX) install(TARGETS skyweavercpp DESTINATION bin) +cuda_add_executable(skycleaver src/skycleaver_cli.cu) +target_link_libraries(skycleaver + ${SKYWEAVER_LIBRARIES} + ${DEPENDENCY_LIBRARIES} + OpenMP::OpenMP_CXX) +install(TARGETS skycleaver DESTINATION bin) + #install (TARGETS ... DESTINATION bin) #install (TARGETS ${CMAKE_PROJECT_NAME} # RUNTIME DESTINATION bin diff --git a/cpp/skyweaver/DescribedVector.hpp b/cpp/skyweaver/DescribedVector.hpp index 6d4a8d5..6fcdd0a 100644 --- a/cpp/skyweaver/DescribedVector.hpp +++ b/cpp/skyweaver/DescribedVector.hpp @@ -127,6 +127,23 @@ struct DescribedVector { _vector.resize(calculate_nelements()); } + /** + * @brief Construct a new Described Vector object of specific size + * + * @param sizes The sizes of the dimensions (must match the number of + * dimensions) + */ + DescribedVector(std::initializer_list sizes, value_type default_value) + : _dms_stale(true), _frequencies_stale(true), _sizes(sizes), + _dims{dims...}, _tsamp(0.0) + { + if(_sizes.size() != sizeof...(dims)) { + throw std::invalid_argument( + "Number of sizes must match number of dimensions"); + } + _vector.resize(calculate_nelements(), default_value); + } + /** * @brief Destroy the Described Vector object * @@ -213,6 +230,12 @@ struct DescribedVector { */ auto const& operator[](std::size_t idx) const { return _vector[idx]; } + + // also the at() method + auto& at(std::size_t idx) { return _vector[idx]; } + auto const& at(std::size_t idx) const { return _vector[idx]; } + + /** * @brief Resize the dimensions of the vector * @@ -648,7 +671,7 @@ using BTFPowersH = DescribedVector>, template using BTFPowersD = DescribedVector, BeamDim, TimeDim, FreqDim>; -// Incoherent dedisperser outputs +// Incoherent dedisperser outputs, also used for skycleaver template using TDBPowersH = DescribedVector>, TimeDim, @@ -667,6 +690,13 @@ template using FPAStatsD = DescribedVector, FreqDim, PolnDim, AntennaDim>; +//skycleaver output vectors + +template +using TFPowersH = DescribedVector>, + TimeDim, + FreqDim>; + } // namespace skyweaver #endif // SKYWEAVER_DESCRIBEDVECTORS_HPP \ No newline at end of file diff --git a/cpp/skyweaver/MultiBeamWriter.hpp b/cpp/skyweaver/MultiBeamWriter.hpp deleted file mode 100644 index c22ae3d..0000000 --- a/cpp/skyweaver/MultiBeamWriter.hpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "SigprocFileWriter.hpp" -#include "boost/log/trivial.hpp" -#include "psrdada_cpp/psrdadaheader.hpp" -#include "psrdada_cpp/raw_bytes.hpp" -#include "skyweaver/ObservationHeader.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -namespace skyweaver -{ - -struct Bridge -{ - private: - using PowerType = std::vector; - public: - std::vector _tdb_filenames; - std::string freq; - -}; // class Bridge - -class MultiBeamWriter -{ - private: - using FreqType = unsigned int; // up to the nearest Hz - std::map> _bridges; - std::vector _beam_filenames; - - - public - : add_bridge(FreqType freq, std::vector tdb_filenames); - remove_bridge(FreqType freq); - - init(); - -} // class MultiFileWriter -} // namespace skyweaver \ No newline at end of file diff --git a/cpp/skyweaver/MultiFileReader.cuh b/cpp/skyweaver/MultiFileReader.cuh index da92207..53bc87a 100644 --- a/cpp/skyweaver/MultiFileReader.cuh +++ b/cpp/skyweaver/MultiFileReader.cuh @@ -36,6 +36,7 @@ class skyweaver::MultiFileReader std::size_t _dada_header_size; std::size_t _total_size; bool _is_open; + void read_header(std::vector& headerBytes); void check_contiguity(); @@ -49,14 +50,18 @@ class skyweaver::MultiFileReader void open_previous(); void close(); void seekg(long pos, std::ios_base::seekdir dir = std::ios_base::beg); + std::size_t tellg() const; bool eof() const; bool good() const; bool can_read(std::size_t bytes) const; std::streamsize read(char* raw_ptr, std::streamsize bytes); + + bool is_open() const; std::size_t safe_read(std::ifstream& input_stream, char* buffer, std::size_t nbytes); + std::vector const& get_dada_files() const; std::size_t get_total_size() const; diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index e5ea6f0..ceeb2d8 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -12,6 +12,37 @@ namespace skyweaver { + + + + +struct MultiFileWriterConfig{ + + std::size_t header_size; + std::size_t max_file_size; + std::string stokes_mode; + std::string output_dir; + std::string prefix; + std::string extension; + + + MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("test"), prefix("test"), extension("default"){}; + MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, std::string extension) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension){}; + MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), stokes_mode(other.stokes_mode), output_dir(other.output_dir), prefix(other.prefix), extension(other.extension){}; + MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other){ + header_size = other.header_size; + max_file_size = other.max_file_size; + stokes_mode = other.stokes_mode; + output_dir = other.output_dir; + prefix = other.prefix; + extension = other.extension; + return *this; + } + + std::string to_string(){ + return "header_size: " + std::to_string(header_size) + ", max_file_size: " + std::to_string(max_file_size) + ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + ", prefix: " + prefix + ", extension: " + extension; + } +}; /** * @brief A class for handling writing of DescribedVectors * @@ -19,6 +50,13 @@ namespace skyweaver template class MultiFileWriter { +public: + + using CreateStreamCallBackType = std::function(MultiFileWriterConfig const&, + ObservationHeader const&, + VectorType const&, + std::size_t)>; + public: /** * @brief Construct a new Multi File Writer object @@ -27,7 +65,9 @@ class MultiFileWriter * @param tag A string tag to be added to the file name * (used to avoid clashing file names). */ - MultiFileWriter(PipelineConfig const& config, std::string tag = ""); + // MultiFileWriter(PipelineConfig const& config, std::string tag = ""); + MultiFileWriter(PipelineConfig const& config, std::string tag, CreateStreamCallBackType create_stream_callback); + MultiFileWriter(MultiFileWriterConfig config, std::string tag, CreateStreamCallBackType create_stream_callback); MultiFileWriter(MultiFileWriter const&) = delete; /** @@ -62,6 +102,9 @@ class MultiFileWriter */ bool operator()(VectorType const& stream_data, std::size_t stream_idx = 0); + bool write(VectorType const& stream_data, + std::size_t stream_idx = 0); + private: bool has_stream(std::size_t stream_idx); FileOutputStream& create_stream(VectorType const& stream_data, @@ -71,8 +114,8 @@ class MultiFileWriter std::string get_basefilename(VectorType const& stream_data, std::size_t stream_idx); std::string get_extension(VectorType const& stream_data); - - PipelineConfig const& _config; + CreateStreamCallBackType _create_stream_callback; + MultiFileWriterConfig _config; std::string _tag; ObservationHeader _header; std::map> _file_streams; @@ -83,5 +126,6 @@ class MultiFileWriter } // namespace skyweaver #include "skyweaver/detail/MultiFileWriter.cu" +#include "skyweaver/detail/file_writer_callbacks.cpp" #endif // SKYWEAVER_MULTIFILEWRITER_CUH \ No newline at end of file diff --git a/cpp/skyweaver/ObservationHeader.hpp b/cpp/skyweaver/ObservationHeader.hpp index c8b6c1b..753ac2a 100644 --- a/cpp/skyweaver/ObservationHeader.hpp +++ b/cpp/skyweaver/ObservationHeader.hpp @@ -4,9 +4,12 @@ #include "psrdada_cpp/raw_bytes.hpp" #include "skyweaver/Header.hpp" #include "skyweaver/PipelineConfig.hpp" +#include namespace skyweaver { + + struct ObservationHeader { std::size_t nchans = 0; // Number of frequency channels in the subband std::size_t npol = 0; // Number of polarisations @@ -27,15 +30,46 @@ struct ObservationHeader { long double sync_time = 0.0; // The UNIX epoch of the sampler zero long double utc_start = 0.0; // The UTC start time of the data long double mjd_start = 0.0; // The MJD start time of the data - std::size_t obs_offset = 0; // The offset of the current file from UTC_START in bytes + std::size_t obs_offset = 0; // The offset of the current file from UTC_START in bytesß + long double refdm = 0.0; // Reference DM + std::size_t ibeam = 0.0; // Beam number + std::size_t nbeams = 0; // Number of beams std::string source_name; // Name of observation target std::string ra; // Right ascension std::string dec; // Declination std::string telescope; // Telescope name std::string instrument; // Name of the recording instrument - std::string to_string() const; + std::string order; // Order of the dimensions in the data + std::string to_string() const; // Convert the header to a string + + long double az; // Azimuth + long double za; // Zenith angle + std::size_t machineid = 0; // Machine ID + std::size_t nifs = 0; // Number of IFs + std::size_t telescopeid = 0; // Telescope ID + std::size_t datatype = 0; // Data type + std::size_t barycentric = 0; // Barycentric correction + std::string rawfile; // Raw file name + double fch1 = 0.0; // Centre frequency of the first channel + double foff = 0.0; // Frequency offset between channels + + bool sigproc_params = false; // Whether to include sigproc parameters + ObservationHeader() = default; + ObservationHeader(ObservationHeader const&) = default; + ObservationHeader& operator=(ObservationHeader const&) = default; + }; +// template for comparing two floating point objects + +template +typename std::enable_if::value, bool>::type +is_close(T a, T b, T tolerance = 1e-12) +{ + return std::fabs(a - b) < tolerance; +} + + /** * @brief Parse header information for a DADA header block * @@ -47,11 +81,21 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, void validate_header(ObservationHeader const& header, PipelineConfig const& config); + void update_config(PipelineConfig& config, ObservationHeader const& header); bool are_headers_similar(ObservationHeader const& header1, - ObservationHeader const& header2); + ObservationHeader const& header2); + + + + + + } // namespace skyweaver + + + #endif // SKYWEAVER_OBSERVATIONHEADER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SigprocFileWriter.hpp b/cpp/skyweaver/SigprocFileWriter.hpp index 77bc4d8..7206813 100644 --- a/cpp/skyweaver/SigprocFileWriter.hpp +++ b/cpp/skyweaver/SigprocFileWriter.hpp @@ -29,7 +29,7 @@ class SigprocFileWriter */ SigprocFileWriter(); SigprocFileWriter(SigprocFileWriter const&) = delete; - ~SigprocFileWriter(); + ~SigprocFileWriter()=default; /** * @brief Set a tag to be used in the filename of generated files diff --git a/cpp/skyweaver/SigprocHeader.hpp b/cpp/skyweaver/SigprocHeader.hpp index 912e83e..098beff 100644 --- a/cpp/skyweaver/SigprocHeader.hpp +++ b/cpp/skyweaver/SigprocHeader.hpp @@ -3,6 +3,7 @@ #define SKYWEAVER_SIGPROCHEADER_HPP #include "psrdada_cpp/raw_bytes.hpp" +#include "skyweaver/ObservationHeader.hpp" #include #include @@ -14,7 +15,22 @@ namespace skyweaver { -struct FilHead { + +class SigprocHeader +{ + public: + SigprocHeader(); + + SigprocHeader(ObservationHeader const& obs_header); + + SigprocHeader(SigprocHeader const&) = delete; + + ~SigprocHeader(); + + void write_header(std::ostream& stream); + void add_time_offset(double offset_mjd); + + private: std::string rawfile = "unset"; std::string source = "unset"; double az = 0.0; // azimuth angle in deg @@ -35,20 +51,7 @@ struct FilHead { uint32_t nchans = 0; // number of frequency channels uint32_t nifs = 0; // number of ifs (pols) uint32_t telescopeid = 0; // telescope ID -}; - -class SigprocHeader -{ - public: - SigprocHeader(); - SigprocHeader(SigprocHeader const&) = delete; - - ~SigprocHeader(); - - std::ostream& write_header(std::ostream& stream, FilHead const& header); - - private: /* * @brief write string to the header */ @@ -58,6 +61,7 @@ class SigprocHeader std::string const& str, std::string const& name); + /* * @brief write a value to the stream */ diff --git a/cpp/skyweaver/SkyCleaver.cuh b/cpp/skyweaver/SkyCleaver.cuh new file mode 100644 index 0000000..9e4b86e --- /dev/null +++ b/cpp/skyweaver/SkyCleaver.cuh @@ -0,0 +1,77 @@ +#ifndef SKYWEAVER_SKYCLEAVER_HPP +#define SKYWEAVER_SKYCLEAVER_HPP +#include "skyweaver/DescribedVector.hpp" +#include "boost/log/trivial.hpp" +#include "psrdada_cpp/psrdadaheader.hpp" +#include "psrdada_cpp/raw_bytes.hpp" +#include "skyweaver/ObservationHeader.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include "skyweaver/MultiFileReader.cuh" +#include "skyweaver/SkyCleaverConfig.hpp" +#include "skyweaver/MultiFileWriter.cuh" +#include "skyweaver/ObservationHeader.hpp" +#include "skyweaver/Timer.hpp" + +namespace skyweaver +{ + +struct BridgeReader +{ + public: + std::vector _tdb_filenames; + std::unique_ptr _tdb_reader; + std::string freq; + +}; // BridgeReader +class SkyCleaver +{ + public: + using InputType = int8_t; + using OutputType = int8_t; + using InputVectorType = TDBPowersH; + using OutputVectorType = TFPowersH; + using FreqType = std::size_t; // up to the nearest Hz + using BeamNumberType = std::size_t; + using DMNumberType = std::size_t; + + + private: + SkyCleaverConfig const& _config; + std::map> _bridge_readers; + std::map> _bridge_data; // shared ptr so that we can reuse zero data + std::vector _expected_freqs; + std::vector _available_freqs; + std::size_t _nsamples_to_read; + ObservationHeader _header; + + std::vector _beam_filenames; + std::map>>> _beam_writers; + std::map>> _beam_data; + std::size_t _total_beam_writers; + + + void init_readers(); + void init_writers(); + + Timer _timer; + + + public: + SkyCleaver(SkyCleaverConfig const& config); + SkyCleaver(SkyCleaver const&) = delete; + void operator=(SkyCleaver const&) = delete; + + void cleave(); + +}; // class SkyCleaver +} // namespace skyweaver + +#endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp new file mode 100644 index 0000000..a1ae8c7 --- /dev/null +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -0,0 +1,71 @@ +#ifndef SKYCLEAVERCONFIG_HPP +#define SKYCLEAVERCONFIG_HPP +namespace skyweaver { + +class SkyCleaverConfig +{ + std::string _output_dir; + std::string _root_dir; + std::string _root_prefix; + std::string _out_prefix; + std::size_t _nthreads; + std::size_t _nsamples_per_block; + std::size_t _nchans; + std::size_t _nbeams; + std::size_t _max_ram_gb; + std::size_t _max_output_filesize; + std::size_t _stream_id; + std::size_t _nbridges; + std::size_t _ndms; + std::string _stokes_mode; + std::size_t _dada_header_size; + + + + + + public: + + SkyCleaverConfig() : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096) {} + SkyCleaverConfig(SkyCleaverConfig const&) = delete; + + void output_dir(std::string output_dir) { _output_dir = output_dir; } + void root_dir(std::string root_dir) { _root_dir = root_dir; } + void root_prefix(std::string root_prefix) { _root_prefix = root_prefix; } + void out_prefix(std::string out_prefix) { _out_prefix = out_prefix; } + void nthreads(std::size_t nthreads) { _nthreads = nthreads; } + void nsamples_per_block(std::size_t nsamples_per_block) { _nsamples_per_block = nsamples_per_block; } + void nchans(std::size_t nchans) { _nchans = nchans; } + void nbeams(std::size_t nbeams) { _nbeams = nbeams; } + void max_ram_gb(std::size_t max_ram_gb) { _max_ram_gb = max_ram_gb; } + void max_output_filesize(std::size_t max_output_filesize) { _max_output_filesize = max_output_filesize; } + void stream_id(std::size_t stream_id) { _stream_id = stream_id; } + void nbridges(std::size_t nbridges) { _nbridges = nbridges; } + void ndms(std::size_t ndms) { _ndms = ndms; } + void stokes_mode(std::string stokes_mode) { _stokes_mode = stokes_mode; } + void dada_header_size(std::size_t dada_header_size) { _dada_header_size = dada_header_size; } + + + + std::string output_dir() const { return _output_dir; } + std::string root_dir() const { return _root_dir; } + std::string root_prefix() const { return _root_prefix; } + std::string out_prefix() const { return _out_prefix; } + std::size_t nthreads() const { return _nthreads; } + std::size_t nsamples_per_block() const { return _nsamples_per_block; } + std::size_t nchans() const { return _nchans; } + std::size_t nbeams() const { return _nbeams; } + std::size_t max_ram_gb() const { return _max_ram_gb; } + std::size_t max_output_filesize() const { return _max_output_filesize; } + std::size_t stream_id() const { return _stream_id; } + std::size_t nbridges() const { return _nbridges; } + std::size_t ndms() const { return _ndms; } + std::string stokes_mode() const { return _stokes_mode; } + std::size_t dada_header_size() const { return _dada_header_size; } + + + + +}; +} // namespace skyweaver +#endif // SKYCLEAVERCONFIG_HPP \ No newline at end of file diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 64b3941..0fdead7 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -7,6 +7,8 @@ #include #include + + /** * Now write a DADA file per DM * with optional time splitting @@ -18,51 +20,6 @@ namespace skyweaver namespace { -/** - * The expectation is that each file contains data in - * TBTF order. Need to explicitly update: - * INNER_T --> the number of timesamples per block that was processed - * NBEAMS --> the total number of beams in the file - * DM --> the dispersion measure of this data - * Freq --> the centre frequency of this subband (???) - * BW --> the bandwidth of the subband (???) - * TSAMP --> the sampling interval of the data - * NPOL --> normally 1 but could be 4 for full stokes - * CHAN0_IDX --> this comes from the obs header and uniquely identifies the - * bridge - */ -std::string default_dada_header = R"( -HEADER DADA -HDR_VERSION 1.0 -HDR_SIZE 4096 -DADA_VERSION 1.0 - -FILE_SIZE 100000000000 -FILE_NUMBER 0 - -UTC_START 1708082229.000020336 -MJD_START 60356.47024305579093 - -SOURCE J1644-4559 -RA 16:44:49.27 -DEC -45:59:09.7 -TELESCOPE MeerKAT -INSTRUMENT CBF-Feng -RECEIVER L-band -FREQ 1284000000.000000 -BW 856000000.000000 -TSAMP 4.7850467290 -STOKES I - -NBIT 8 -NDIM 1 -NPOL 1 -NCHAN 64 -NBEAM 800 -ORDER TFB - -CHAN0_IDX 2688 -)"; std::string get_formatted_time(long double unix_timestamp) { @@ -75,13 +32,36 @@ std::string get_formatted_time(long double unix_timestamp) } // namespace +// template +// MultiFileWriter::MultiFileWriter(PipelineConfig const& config, +// std::string tag) +// : _config(config), _tag(tag) +// { +// } + template MultiFileWriter::MultiFileWriter(PipelineConfig const& config, - std::string tag) - : _config(config), _tag(tag) + std::string tag, + CreateStreamCallBackType create_stream_callback) + : _tag(tag), _create_stream_callback(create_stream_callback) { + MultiFileWriterConfig writer_config; + writer_config.header_size = config.dada_header_size(); + writer_config.max_file_size = config.max_output_filesize(); + writer_config.stokes_mode = config.stokes_mode(); + _config = writer_config; } +template +MultiFileWriter::MultiFileWriter(MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback) + : _config(config), _tag(tag), _create_stream_callback(create_stream_callback) +{ +} + + + template MultiFileWriter::~MultiFileWriter(){}; @@ -99,80 +79,22 @@ bool MultiFileWriter::has_stream(std::size_t stream_idx) } template -FileOutputStream& +FileOutputStream& MultiFileWriter::create_stream(VectorType const& stream_data, std::size_t stream_idx) { - BOOST_LOG_TRIVIAL(debug) << "Creating stream based on stream prototype: " - << stream_data.describe(); - // Here we round the file size to a multiple of the stream prototype - std::size_t filesize = - std::max(1ul, - _config.max_output_filesize() / stream_data.size() / - sizeof(typename VectorType::value_type)) * - stream_data.size(); - BOOST_LOG_TRIVIAL(debug) - << "Maximum allowed file size = " << filesize << " bytes (+header)"; - - _file_streams[stream_idx] = std::make_unique( - get_output_dir(stream_data, stream_idx), - get_basefilename(stream_data, stream_idx), - get_extension(stream_data), - filesize, - [&, this, stream_data, stream_idx, filesize]( - std::size_t& header_size, - std::size_t bytes_written, - std::size_t file_idx) -> std::shared_ptr { - header_size = _config.dada_header_size(); - char* temp_header = new char[header_size]; - std::fill(temp_header, temp_header + header_size, 0); - std::memcpy(temp_header, - default_dada_header.c_str(), - default_dada_header.size()); - psrdada_cpp::RawBytes bytes(temp_header, - header_size, - header_size, - false); - Header header_writer(bytes); - header_writer.set("NBEAM", stream_data.nbeams()); - header_writer.set("NCHAN", stream_data.nchannels()); - header_writer.set("OBS_NCHAN", _header.obs_nchans); - header_writer.set("OBS_FREQUENCY", - _header.obs_frequency); - header_writer.set("OBS_BW", _header.obs_bandwidth); - header_writer.set("NSAMP", stream_data.nsamples()); - if(stream_data.ndms()) { - header_writer.set("NDMS", stream_data.ndms()); - header_writer.set("DMS", stream_data.dms(), 7); - } - header_writer.set( - "COHERENT_DM", - static_cast(stream_data.reference_dm())); - header_writer.set("FREQ", _header.frequency); - header_writer.set("BW", _header.bandwidth); - header_writer.set("TSAMP", stream_data.tsamp() * 1e6); - if(_config.stokes_mode() == "IQUV") { - header_writer.set("NPOL", 4); - } else { - header_writer.set("NPOL", 1); - } - header_writer.set("STOKES_MODE", - _config.stokes_mode()); - header_writer.set("ORDER", - stream_data.dims_as_string()); - header_writer.set("CHAN0_IDX", _header.chan0_idx); - header_writer.set("FILE_SIZE", filesize); - header_writer.set("FILE_NUMBER", file_idx); - header_writer.set("OBS_OFFSET", bytes_written); - header_writer.set("OBS_OVERLAP", 0); - header_writer.set("UTC_START", - _header.utc_start + - stream_data.utc_offset()); - std::shared_ptr header_ptr( - temp_header, - std::default_delete()); - return header_ptr; - }); + + // config.output_dir = get_output_dir(stream_data, stream_idx); + // config.prefix = get_basefilename(stream_data, stream_idx); + // config.extension = get_extension(stream_data); + + BOOST_LOG_TRIVIAL(info) << "Creating stream " << stream_idx << " in " << _config.output_dir; + BOOST_LOG_TRIVIAL(info) << "Prefix: " << _config.prefix; + BOOST_LOG_TRIVIAL(info) << "Extension: " << _config.extension; + BOOST_LOG_TRIVIAL(info) << "Output directory: " << _config.output_dir; + + _file_streams[stream_idx] = _create_stream_callback(_config, _header, stream_data, stream_idx); + return *_file_streams[stream_idx]; } @@ -184,7 +106,7 @@ MultiFileWriter::get_output_dir(VectorType const& stream_data, // Output directory format // // std::stringstream output_dir; - output_dir << _config.output_dir() << "/" + output_dir << _config.output_dir << "/" << get_formatted_time(_header.utc_start) << "/" << stream_idx << "/" << std::fixed << std::setfill('0') << std::setw(9) @@ -200,8 +122,8 @@ MultiFileWriter::get_basefilename(VectorType const& stream_data, // Output file format // ___. std::stringstream base_filename; - if(!_config.output_file_prefix().empty()) { - base_filename << _config.output_file_prefix() << "_"; + if(!_config.prefix.empty()) { + base_filename << _config.prefix << "_"; } base_filename << get_formatted_time(_header.utc_start) << "_" << stream_idx << "_" << std::fixed << std::setprecision(3) @@ -247,4 +169,13 @@ bool MultiFileWriter::operator()(VectorType const& stream_data, return false; } +template +bool MultiFileWriter::write(VectorType const& stream_data, + std::size_t stream_idx) +{ + + return this->operator()(stream_data, stream_idx); + +} + } // namespace skyweaver \ No newline at end of file diff --git a/cpp/skyweaver/detail/SigprocHeader.cpp b/cpp/skyweaver/detail/SigprocHeader.cpp index 99c9e28..eef42cd 100644 --- a/cpp/skyweaver/detail/SigprocHeader.cpp +++ b/cpp/skyweaver/detail/SigprocHeader.cpp @@ -1,4 +1,4 @@ -#include "skyweaver/SigProcHeader.hpp" +#include "skyweaver/SigprocHeader.hpp" #include diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp new file mode 100644 index 0000000..f16150b --- /dev/null +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -0,0 +1,301 @@ + + +#include "skyweaver/FileOutputStream.hpp" +#include "skyweaver/MultiFileWriter.cuh" +#include "skyweaver/ObservationHeader.hpp" +#include "skyweaver/SigprocHeader.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +namespace { + /** + * The expectation is that each file contains data in + * TBTF order. Need to explicitly update: + * INNER_T --> the number of timesamples per block that was processed + * NBEAMS --> the total number of beams in the file + * DM --> the dispersion measure of this data + * Freq --> the centre frequency of this subband (???) + * BW --> the bandwidth of the subband (???) + * TSAMP --> the sampling interval of the data + * NPOL --> normally 1 but could be 4 for full stokes + * CHAN0_IDX --> this comes from the obs header and uniquely identifies the + * bridge + */ +std::string default_dada_header = R"( + HEADER DADA + HDR_VERSION 1.0 + HDR_SIZE 4096 + DADA_VERSION 1.0 + + FILE_SIZE 100000000000 + FILE_NUMBER 0 + + UTC_START 1708082229.000020336 + MJD_START 60356.47024305579093 + + SOURCE J1644-4559 + RA 16:44:49.27 + DEC -45:59:09.7 + TELESCOPE MeerKAT + INSTRUMENT CBF-Feng + RECEIVER L-band + FREQ 1284000000.000000 + BW 856000000.000000 + TSAMP 4.7850467290 + STOKES I + + NBIT 8 + NDIM 1 + NPOL 1 + NCHAN 64 + NBEAM 800 + ORDER TFB + + CHAN0_IDX 2688 + )"; +} +namespace skyweaver +{ +namespace detail +{ +template inline +std::unique_ptr create_dada_file_stream(MultiFileWriterConfig const& config, + ObservationHeader const& header, + VectorType const& stream_data, + std::size_t stream_idx) +{ + BOOST_LOG_TRIVIAL(debug) << "Creating stream based on stream prototype: " + << stream_data.describe(); + // Here we round the file size to a multiple of the stream prototype + std::size_t filesize = + std::max(1ul, + config.max_file_size / stream_data.size() / + sizeof(typename VectorType::value_type)) * + stream_data.size(); + + BOOST_LOG_TRIVIAL(debug) + << "Maximum allowed file size = " << filesize << " bytes (+header)"; + + std::unique_ptr file_stream = std::make_unique( + config.output_dir, + config.prefix, + config.extension, + filesize, + [&, header, stream_data, stream_idx, filesize]( + std::size_t& header_size, + std::size_t bytes_written, + std::size_t file_idx) -> std::shared_ptr { + header_size = config.header_size; + char* temp_header = new char[header_size]; + std::fill(temp_header, temp_header + header_size, 0); + std::memcpy(temp_header, + default_dada_header.c_str(), + default_dada_header.size()); + psrdada_cpp::RawBytes bytes(temp_header, + header_size, + header_size, + false); + Header header_writer(bytes); + header_writer.set("NBEAM", stream_data.nbeams()); + header_writer.set("NCHAN", stream_data.nchannels()); + header_writer.set("OBS_NCHAN", header.obs_nchans); + header_writer.set("OBS_FREQUENCY", + header.obs_frequency); + header_writer.set("OBS_BW", header.obs_bandwidth); + header_writer.set("NSAMP", stream_data.nsamples()); + if(stream_data.ndms()) { + header_writer.set("NDMS", stream_data.ndms()); + header_writer.set("DMS", stream_data.dms(), 7); + } + header_writer.set( + "COHERENT_DM", + static_cast(stream_data.reference_dm())); + header_writer.set("FREQ", header.frequency); + header_writer.set("BW", header.bandwidth); + header_writer.set("TSAMP", stream_data.tsamp() * 1e6); + if(config.stokes_mode == "IQUV") { + header_writer.set("NPOL", 4); + } else { + header_writer.set("NPOL", 1); + } + header_writer.set("STOKES_MODE", + config.stokes_mode); + header_writer.set("ORDER", + stream_data.dims_as_string()); + header_writer.set("CHAN0_IDX", header.chan0_idx); + header_writer.set("FILE_SIZE", filesize); + header_writer.set("FILE_NUMBER", file_idx); + header_writer.set("OBS_OFFSET", bytes_written); + header_writer.set("OBS_OVERLAP", 0); + header_writer.set("UTC_START", + header.utc_start + + stream_data.utc_offset()); + std::shared_ptr header_ptr( + temp_header, + std::default_delete()); + return header_ptr; + }); + return file_stream; +} + +template inline +std::unique_ptr create_sigproc_file_stream(MultiFileWriterConfig const& config, + ObservationHeader const& obs_header, + VectorType const& stream_data, + std::size_t stream_idx) +{ + + + + BOOST_LOG_TRIVIAL(debug) << "Creating stream based on stream prototype: " + << stream_data.describe(); + + ObservationHeader header = obs_header; + + BOOST_LOG_TRIVIAL(debug) << "Header: " << header.to_string(); + + // Here we round the file size to a multiple of the stream prototype + std::size_t filesize = + std::max(1ul, + config.max_file_size / stream_data.size() / + sizeof(typename VectorType::value_type)) * + stream_data.size(); + BOOST_LOG_TRIVIAL(debug) + << "Maximum allowed file size = " << filesize << " bytes (+header)"; + + + double foff = -1* static_cast(header.obs_bandwidth / header.nchans)/1e6;// MHz + double fch1 = static_cast(header.obs_frequency + header.obs_bandwidth / 2.0)/1e6 + 0.5 * foff; // MHz + double tstart = static_cast(header.mjd_start); + uint32_t datatype = 0; + uint32_t barycentric = 0; + // uint32_t ibeam = 0; + double az = 0.0; + double za = 0.0; + uint32_t machineid = 0; + uint32_t nifs = header.npol; + uint32_t telescopeid = 0; + + header.sigproc_params = true; + header.rawfile = std::string("unset"); + header.fch1 = fch1; + header.foff = foff; + header.tsamp = header.tsamp/1e6; + header.az = az; + header.za = za; + header.datatype = datatype; + header.barycentric = barycentric; + header.nifs = nifs; + header.telescopeid = telescopeid; + + + + BOOST_LOG_TRIVIAL(info) << "Creating Sigproc file stream"; + BOOST_LOG_TRIVIAL(info) << "fch1: " << header.obs_frequency + header.obs_bandwidth / 2.0; + + BOOST_LOG_TRIVIAL(info) << "rawfile: " << header.rawfile; + BOOST_LOG_TRIVIAL(info) << "source: " << header.source_name; + BOOST_LOG_TRIVIAL(info) << "ra: " << header.ra; + BOOST_LOG_TRIVIAL(info) << "dec: " << header.dec; + BOOST_LOG_TRIVIAL(info) << "fch1: " << fch1; + BOOST_LOG_TRIVIAL(info) << "foff: " << foff; + BOOST_LOG_TRIVIAL(info) << "rdm: " << header.refdm; + BOOST_LOG_TRIVIAL(info) << "tsamp: " << header.tsamp; + BOOST_LOG_TRIVIAL(info) << "tstart: " << tstart; + BOOST_LOG_TRIVIAL(info) << "az: " << az; + BOOST_LOG_TRIVIAL(info) << "za: " << za; + BOOST_LOG_TRIVIAL(info) << "datatype: " << header.datatype; + BOOST_LOG_TRIVIAL(info) << "barycentric: " << header.barycentric; + BOOST_LOG_TRIVIAL(info) << "ibeam: " << header.ibeam; + BOOST_LOG_TRIVIAL(info) << "machineid: " << machineid; + BOOST_LOG_TRIVIAL(info) << "nbeams: " << header.nbeams; + BOOST_LOG_TRIVIAL(info) << "nbits: " << header.nbits; + BOOST_LOG_TRIVIAL(info) << "nchans: " << header.nchans; + BOOST_LOG_TRIVIAL(info) << "nifs: " << nifs; + BOOST_LOG_TRIVIAL(info) << "telescopeid: " << telescopeid; + BOOST_LOG_TRIVIAL(info) << "outdir: " << config.output_dir; + BOOST_LOG_TRIVIAL(info) << "prefix: " << config.prefix; + BOOST_LOG_TRIVIAL(info) << "extension: " << config.extension; + // // Here we should update the tstart of the default header to be the + // // start of the stream + // // reset the total bytes counter to keep the time tracked correctly + // // Generate the new base filename in _ format + // std::stringstream base_filename; + // // Get UTC time string + // std::time_t unix_time = + // static_cast(( header.mjd_start - 40587.0) * 86400.0); + // struct std::tm* ptm = std::gmtime(&unix_time); + + // // Swapped out put_time call for strftime due to put_time + // // causing compiler bugs prior to g++ 5.x + // char formatted_time[80]; + // strftime(formatted_time, 80, "%Y-%m-%d-%H:%M:%S", ptm); + // base_filename << formatted_time; + + + + std::unique_ptr file_stream = std::make_unique( + config.output_dir, + config.prefix, + config.extension, + filesize, + [header](std::size_t& header_size, + std::size_t bytes_written, + std::size_t file_idx) -> std::shared_ptr { + // We do not explicitly delete[] this array + // Cleanup is handled by the shared pointer + // created below + std::ostringstream header_stream; + // get ostream from temp_header + BOOST_LOG_TRIVIAL(info) << "Creating Sigproc header"; + BOOST_LOG_TRIVIAL(info) << "rawfile: " << header.rawfile; + BOOST_LOG_TRIVIAL(info) << "source: " << header.source_name; + BOOST_LOG_TRIVIAL(info) << "ra: " << header.ra; + BOOST_LOG_TRIVIAL(info) << "dec: " << header.dec; + BOOST_LOG_TRIVIAL(info) << "fch1: " << header.fch1; + BOOST_LOG_TRIVIAL(info) << "foff: " << header.foff; + BOOST_LOG_TRIVIAL(info) << "rdm: " << header.refdm; + BOOST_LOG_TRIVIAL(info) << "tsamp: " << header.tsamp; + BOOST_LOG_TRIVIAL(info) << "tstart: " << header.mjd_start; + BOOST_LOG_TRIVIAL(info) << "az: " << header.az; + BOOST_LOG_TRIVIAL(info) << "za: " << header.za; + BOOST_LOG_TRIVIAL(info) << "datatype: " << header.datatype; + BOOST_LOG_TRIVIAL(info) << "barycentric: " << header.barycentric; + BOOST_LOG_TRIVIAL(info) << "ibeam: " << header.ibeam; + BOOST_LOG_TRIVIAL(info) << "machineid: " << header.machineid; + BOOST_LOG_TRIVIAL(info) << "nbeams: " << header.nbeams; + BOOST_LOG_TRIVIAL(info) << "nbits: " << header.nbits; + BOOST_LOG_TRIVIAL(info) << "nchans: " << header.nchans; + BOOST_LOG_TRIVIAL(info) << "nifs: " << header.nifs; + BOOST_LOG_TRIVIAL(info) << "telescopeid: " << header.telescopeid; + + SigprocHeader sigproc_header(header); + double mjd_offset = (((bytes_written / (header.nbits / 8.0)) / (header.nchans)) * + header.tsamp) / + (86400.0); + sigproc_header.add_time_offset(mjd_offset); + sigproc_header.write_header(header_stream); + std::string header_str = header_stream.str(); + header_size = header_str.size(); + char* header_cstr = new char[header_size]; + std::copy(header_str.begin(), header_str.end(), header_cstr); + std::shared_ptr header_ptr( + header_cstr, + std::default_delete()); + return header_ptr; + }); + return file_stream; +} + +} // namespace detail +} // namespace skyweaver \ No newline at end of file diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index ea1ed8f..08d176e 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -53,7 +53,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, pipeline_config.coherent_dms()); } /* - * @brief Create a new CoherentDedisperser object, mostly used only for + * @brief Create a new CoherentDedis§rser object, mostly used only for * testing * * @param config The config reference diff --git a/cpp/skyweaver/src/IncoherentDedisperser.cu b/cpp/skyweaver/src/IncoherentDedisperser.cu index c1bd109..117e218 100644 --- a/cpp/skyweaver/src/IncoherentDedisperser.cu +++ b/cpp/skyweaver/src/IncoherentDedisperser.cu @@ -93,7 +93,7 @@ void IncoherentDedisperser::dedisperse( for(int t_idx = _max_sample_delay; t_idx < nsamples; t_idx += _tscrunch) { std::size_t t_output_offset = std::size_t(t_idx - _max_sample_delay) / _tscrunch * nbeams * ndms; for(int dm_idx = 0; dm_idx < ndms; ++dm_idx) { - int offset = nchans * dm_idx; + int offset = nchans * dm_idx; std::fill(powers.begin(), powers.end(), value_traits< diff --git a/cpp/skyweaver/src/MultiFileReader.cu b/cpp/skyweaver/src/MultiFileReader.cu index 47c1abe..3bd3dae 100644 --- a/cpp/skyweaver/src/MultiFileReader.cu +++ b/cpp/skyweaver/src/MultiFileReader.cu @@ -39,6 +39,7 @@ MultiFileReader::MultiFileReader(PipelineConfig const& config) MultiFileReader::MultiFileReader(std::vector dada_files,std::size_t dada_header_size, bool check_input_contiguity) : _current_file_idx(0), _current_position(0), _eof_flag(false) { + static_assert(sizeof(char2) == 2 * sizeof(char), "Size of char2 is not as expected"); @@ -47,6 +48,13 @@ MultiFileReader::MultiFileReader(std::vector dada_files,std::size_t this->_dada_header_size = dada_header_size; std::vector header_bytes(_dada_header_size); + + + if(_dada_files.size() == 0) { + throw std::runtime_error("No input files provided for MultiFileReader"); + } + + for(const auto& file: _dada_files) { std::ifstream f(file, std::ifstream::in | std::ifstream::binary); if(!f.is_open()) { @@ -56,6 +64,11 @@ MultiFileReader::MultiFileReader(std::vector dada_files,std::size_t read_header(header_bytes); f.seekg(0, std::ios::end); // Move to the end of the file std::size_t size = f.tellg(); + + if (size < _dada_header_size) { + throw std::runtime_error("File " + file + " is too small to contain a header"); + } + _total_size += (size - _dada_header_size); // skip header _sizes.push_back(size - _dada_header_size); BOOST_LOG_TRIVIAL(debug) @@ -99,6 +112,11 @@ MultiFileReader::~MultiFileReader() this->close(); } +std::vector const& MultiFileReader::get_dada_files() const +{ + return _dada_files; +} + void MultiFileReader::read_header(std::vector& header_bytes) { psrdada_cpp::RawBytes block(header_bytes.data(), diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index e233de3..1694deb 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -1,29 +1,44 @@ #include "skyweaver/ObservationHeader.hpp" -#include + #include +#include namespace skyweaver { void read_dada_header(psrdada_cpp::RawBytes& raw_header, ObservationHeader& header) { Header parser(raw_header); - header.nchans = parser.get("NCHAN"); + header.order = parser.get("ORDER"); + + if(header.order.find("A") != std::string::npos) { + header.nantennas = parser.get("NANT"); + header.sample_clock = + parser.get("SAMPLE_CLOCK"); + header.sample_clock_start = + parser.get( + "SAMPLE_CLOCK_START"); + + header.sync_time = parser.get("SYNC_TIME"); + } + else { + + header.refdm = parser.get("COHERENT_DM"); + + } + header.obs_frequency = + parser.get("OBS_FREQUENCY"); header.obs_nchans = parser.get("OBS_NCHAN"); header.npol = parser.get("NPOL"); header.nbits = parser.get("NBIT"); - header.nantennas = parser.get("NANT"); - header.sample_clock_start = - parser.get("SAMPLE_CLOCK_START"); + header.nchans = parser.get("NCHAN"); + header.bandwidth = parser.get("BW"); header.obs_bandwidth = parser.get("OBS_BW"); header.frequency = parser.get("FREQ"); - header.obs_frequency = - parser.get("OBS_FREQ"); + header.tsamp = parser.get("TSAMP"); - header.sample_clock = - parser.get("SAMPLE_CLOCK"); - header.sync_time = parser.get("SYNC_TIME"); + header.utc_start = parser.get("UTC_START"); header.mjd_start = parser.get("MJD_START"); header.source_name = parser.get("SOURCE"); @@ -34,7 +49,6 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.chan0_idx = parser.get("CHAN0_IDX"); header.obs_offset = parser.get("OBS_OFFSET"); } - void validate_header(ObservationHeader const& header, PipelineConfig const& config) { @@ -61,7 +75,6 @@ void validate_header(ObservationHeader const& header, "currently only 8-bit input supported"); } } - void update_config(PipelineConfig& config, ObservationHeader const& header) { config.bandwidth(header.bandwidth); @@ -69,18 +82,11 @@ void update_config(PipelineConfig& config, ObservationHeader const& header) // TO DO: might need to add other variables in the future. } -//template for comparing two floating point objects -template -typename std::enable_if::value, bool>::type -is_close(T a, T b, T tolerance = 1e-12) { - return std::fabs(a - b) < tolerance; -} bool are_headers_similar(ObservationHeader const& header1, - ObservationHeader const& header2) + ObservationHeader const& header2) { - return header1.nchans == header2.nchans && - header1.npol == header2.npol && + return header1.nchans == header2.nchans && header1.npol == header2.npol && header1.nbits == header2.nbits && header1.nantennas == header2.nantennas && header1.chan0_idx == header2.chan0_idx && @@ -95,12 +101,10 @@ bool are_headers_similar(ObservationHeader const& header1, is_close(header1.utc_start, header2.utc_start) && is_close(header1.mjd_start, header2.mjd_start) && header1.source_name == header2.source_name && - header1.ra == header2.ra && - header1.dec == header2.dec && + header1.ra == header2.ra && header1.dec == header2.dec && header1.telescope == header2.telescope && header1.instrument == header2.instrument; } - std::string ObservationHeader::to_string() const { std::ostringstream oss; @@ -124,8 +128,26 @@ std::string ObservationHeader::to_string() const << " instrument: " << instrument << "\n" << " chan0_idx: " << chan0_idx << "\n" << " obs_offset: " << obs_offset << "\n"; + if(sigproc_params) + { + + oss << " Sigproc parameters:\n" + << " az: " << az << "\n" + << " za: " << za << "\n" + << " machineid: " << machineid << "\n" + << " nifs: " << nifs << "\n" + << " telescopeid: " << telescopeid << "\n" + << " datatype: " << datatype << "\n" + << " barycentric: " << barycentric << "\n" + << " ibeam: " << ibeam << "\n" + << " nbeams: " << nbeams << "\n" + << " rawfile: " << rawfile << "\n" + << " fch1" << fch1 << "\n" + << " foff" << foff << "\n" + << " tsamp" << tsamp << "\n"; + } return oss.str(); } -} // namespace skyweaver \ No newline at end of file +} // namespace skyweaver diff --git a/cpp/skyweaver/src/SigprocHeader.cpp b/cpp/skyweaver/src/SigprocHeader.cpp index a5880c5..ef2f348 100644 --- a/cpp/skyweaver/src/SigprocHeader.cpp +++ b/cpp/skyweaver/src/SigprocHeader.cpp @@ -4,6 +4,13 @@ #include #include #include +#include +#include +#include +#include + + + namespace skyweaver { @@ -12,6 +19,41 @@ SigprocHeader::SigprocHeader() { } +SigprocHeader::SigprocHeader(ObservationHeader const& obs_header) +{ + source = obs_header.source_name; + auto ra_val = obs_header.ra; + auto dec_val = obs_header.dec; + std::vector ra_s; + std::vector dec_s; + boost::split(ra_s, ra_val, boost::is_any_of(":")); + boost::split(dec_s, dec_val, boost::is_any_of(":")); + ra = stod(boost::join(ra_s, "")); + dec = stod(boost::join(dec_s, "")); + fch1 = obs_header.fch1; + foff = obs_header.foff; + tsamp = obs_header.tsamp; + tstart = obs_header.mjd_start; + nbits = obs_header.nbits; + nifs = obs_header.nifs; + nchans = obs_header.nchans; + ibeam = obs_header.ibeam; + rawfile = obs_header.rawfile; + telescopeid = obs_header.telescopeid; + machineid = obs_header.machineid; + datatype = obs_header.datatype; + barycentric = obs_header.barycentric; + nbeams = obs_header.nbeams; + az = obs_header.az; + za = obs_header.za; +} + + +void SigprocHeader::add_time_offset(double offset_mjd) +{ + tstart += offset_mjd; +} + SigprocHeader::~SigprocHeader() { } @@ -25,33 +67,31 @@ void SigprocHeader::header_write(std::ostream& stream, std::string const& str) void SigprocHeader::header_write(std::ostream& stream, std::string const& str, - std::string const& name) + std::string const& val) { header_write(stream, str); - header_write(stream, name); + header_write(stream, val); } -std::ostream& SigprocHeader::write_header(std::ostream& stream, - FilHead const& params) +void SigprocHeader::write_header(std::ostream& stream) { header_write(stream, "HEADER_START"); - header_write(stream, "telescope_id", params.telescopeid); - header_write(stream, "machine_id", params.machineid); - header_write(stream, "data_type", params.datatype); - header_write(stream, "barycentric", params.barycentric); - header_write(stream, "source_name", params.source); - header_write(stream, "src_raj", params.ra); - header_write(stream, "src_dej", params.dec); - header_write(stream, "nbits", params.nbits); - header_write(stream, "nifs", params.nifs); - header_write(stream, "nchans", params.nchans); - header_write(stream, "ibeam", params.ibeam); - header_write(stream, "fch1", params.fch1); - header_write(stream, "foff", params.foff); - header_write(stream, "tstart", params.tstart); - header_write(stream, "tsamp", params.tsamp); + header_write(stream, "telescope_id",telescopeid); + header_write(stream, "machine_id",machineid); + header_write(stream, "data_type",datatype); + header_write(stream, "barycentric",barycentric); + header_write(stream, "source_name",source); + header_write(stream, "src_raj",ra); + header_write(stream, "src_dej",dec); + header_write(stream, "nbits",nbits); + header_write(stream, "nifs",nifs); + header_write(stream, "nchans",nchans); + header_write(stream, "ibeam",ibeam); + header_write(stream, "fch1",fch1); + header_write(stream, "foff",foff); + header_write(stream, "tstart",tstart); + header_write(stream, "tsamp",tsamp); header_write(stream, "HEADER_END"); - return stream; } } // namespace skyweaver diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu new file mode 100644 index 0000000..16a6280 --- /dev/null +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -0,0 +1,399 @@ + +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; + +#include "skyweaver/SkyCleaver.cuh" + +using FreqType = skyweaver::SkyCleaver::FreqType; +using BridgeReader = skyweaver::BridgeReader; +using MultiFileReader = skyweaver::MultiFileReader; +using SkyCleaver = skyweaver::SkyCleaver; +using OutputVectorType = skyweaver::SkyCleaver::OutputVectorType; +using InputVectorType = skyweaver::SkyCleaver::InputVectorType; + +namespace +{ +template +std::string to_string_with_padding(T num, int width, int precision = -1) +{ + std::ostringstream oss; + oss << std::setw(width) << std::setfill('0'); + if(precision >= + 0) { // Check if precision is specified for floating-point numbers + oss << std::fixed << std::setprecision(precision); + } + oss << num; + return oss.str(); +} +std::vector +get_subdirs(std::string directory_path, + std::regex numeric_regex = std::regex("^[0-9]+$")) +{ + std::vector subdirs; + try { + if(fs::exists(directory_path) && fs::is_directory(directory_path)) { + for(const auto& entry: fs::directory_iterator(directory_path)) { + if(fs::is_directory(entry.status())) { + std::string folder_name = entry.path().filename().string(); + if(std::regex_match(folder_name, numeric_regex)) { + BOOST_LOG_TRIVIAL(debug) + << "Found subdirectory: " << folder_name; + subdirs.push_back(folder_name); + } + } + } + } else { + std::runtime_error( + "Root directory does not exist or is not a directory."); + } + } catch(const fs::filesystem_error& e) { + std::cerr << "Filesystem error: " << e.what() << std::endl; + std::runtime_error("Error reading subdirectories in root directory: " + + directory_path); + } + + return subdirs; +} + +std::vector get_files(std::string directory_path, + std::string extension) +{ + std::vector files; + try { + if(fs::exists(directory_path) && fs::is_directory(directory_path)) { + for(const auto& entry: fs::directory_iterator(directory_path)) { + if(fs::is_regular_file(entry.status())) { + std::string file_name = entry.path().string(); + if(file_name.find(extension) != std::string::npos) { + files.push_back(file_name); + } + } + } + } else { + std::runtime_error("No files in bridge directory: " + + directory_path); + } + } catch(const fs::filesystem_error& e) { + std::cerr << "Filesystem error: " << e.what() << std::endl; + std::runtime_error("Error reading files in bridge directory: " + + directory_path); + } + + return files; +} + +} // namespace + +void SkyCleaver::init_readers() +{ + BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_readers") + + std::string root_dir = _config.root_dir(); + std::string root_prefix = _config.root_prefix(); + std::size_t stream_id = _config.stream_id(); + + // get the list of directories in root/stream_id(for the nex) + std::vector freq_dirs = + get_subdirs(root_dir + "/" + std::to_string(stream_id)); + + BOOST_LOG_TRIVIAL(info) + << "Found " << freq_dirs.size() + << " frequency directories in root directory: " << root_dir; + + std::size_t smallest_data_size = std::numeric_limits::max(); + + for(const auto& freq_dir: freq_dirs) { + std::vector tdb_files = get_files( + root_dir + "/" + std::to_string(stream_id) + "/" + freq_dir, + ".tdb"); + BOOST_LOG_TRIVIAL(info) << "Found " << tdb_files.size() + << " TDB files for frequency: " << freq_dir; + if(tdb_files.empty()) { + BOOST_LOG_TRIVIAL(warning) + << "No TDB files found for frequency: " << freq_dir; + continue; + } + + std::size_t freq = static_cast(std::stoul(freq_dir)); + + _bridge_readers[freq] = + std::make_unique(tdb_files, + _config.dada_header_size(), + false); + _available_freqs.push_back(freq); + std::size_t data_size = + _bridge_readers[freq]->get_total_size(); + BOOST_LOG_TRIVIAL(debug) + << "Data size for frequency: " << freq_dir << " is " << data_size; + if(data_size < smallest_data_size) { + smallest_data_size = data_size; + } + + BOOST_LOG_TRIVIAL(debug) + << "Added bridge reader for frequency: " << freq_dir; + } + + BOOST_LOG_TRIVIAL(debug) << "Smallest data size: " << smallest_data_size; + BOOST_LOG_TRIVIAL(debug) << "ndm: " << _config.ndms(); + BOOST_LOG_TRIVIAL(debug) << "nbeams: " << _config.nbeams(); + + if(smallest_data_size % (_config.ndms() * _config.nbeams()) != 0) { + std::runtime_error("Data size is not a multiple of ndms * nbeams"); + } + + std::size_t smallest_nsamples = + std::floor(smallest_data_size / _config.ndms() / _config.nbeams()); + + BOOST_LOG_TRIVIAL(info) + << "Smallest data size: " << smallest_data_size + << " Smallest number of samples: " << smallest_nsamples; + + if(smallest_nsamples < _config.nsamples_per_block()) { + std::runtime_error( + "Smallest data size is less than nsamples_per_block"); + } + + _nsamples_to_read = smallest_nsamples; + + BOOST_LOG_TRIVIAL(info) + << "Added " << _bridge_readers.size() << " bridges to SkyCleaver"; + + // read the first file of first bridge to get the header + + ObservationHeader header = (*_bridge_readers.begin()).second->get_header(); + BOOST_LOG_TRIVIAL(info) + << "Read header from first file: " << header.to_string(); + + float obs_centre_freq = header.obs_frequency; + if(obs_centre_freq == 0.0) { + obs_centre_freq = 1284000000; + } + float obs_bandwidth = header.obs_bandwidth; + int obs_nchans = header.obs_nchans; + + BOOST_LOG_TRIVIAL(info) << "Centre frequency: " << obs_centre_freq + << " Bandwidth: " << obs_bandwidth + << " Number of obs channels: " << obs_nchans; + int nbridges = _config.nbridges(); + + std::size_t gulp_size = + _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); + // std::unique_ptr zero_data = + // std::make_unique(gulp_size, 0); + + for(int i = 0; i < nbridges; i++) { + int ifreq = std::lround(obs_centre_freq - obs_bandwidth / 2 + + (i + 0.5) * obs_bandwidth / nbridges); + _expected_freqs.push_back(ifreq); + BOOST_LOG_TRIVIAL(info) + << "Expected frequency [" << i << "]: " << ifreq; + + if(_bridge_readers.find(ifreq) == _bridge_readers.end()) { + BOOST_LOG_TRIVIAL(warning) + << "Frequency " << ifreq + << " not found in bridge readers, will write zeros"; + } + _bridge_data[ifreq] = std::make_unique( + std::initializer_list{_config.nsamples_per_block(), + _config.ndms(), + _config.nbeams()}, + 0); + } + + // at this point, all non-existed frequencies have been added with zero data + // now check if there are any unexpected frequencies in the bridge readers + + for(const auto& [freq, reader]: _bridge_readers) { + if(std::find(_expected_freqs.begin(), _expected_freqs.end(), freq) == + _expected_freqs.end()) { + throw std::runtime_error("Frequency " + std::to_string(freq) + + " not found in expected frequencies"); + } + } + + BOOST_LOG_TRIVIAL(info) + << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; + + BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver"; + BOOST_LOG_TRIVIAL(info) << "Header: " << header.to_string(); + + _header = header; + _header.nchans = _header.nchans * _config.nbridges(); + _header.nbeams = _config.nbeams(); + +} + +void SkyCleaver::init_writers() +{ + BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_writers") + BOOST_LOG_TRIVIAL(debug) + << "_config.output_dir(); " << _config.output_dir(); + if(!fs::exists(_config.output_dir())) { + fs::create_directories(_config.output_dir()); + } + + for(int idm = 0; idm < _config.ndms(); idm++) { + for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + std::string out_prefix = _config.out_prefix(); + std::string output_dir = _config.output_dir(); + std::string prefix = out_prefix + "_dm" + + to_string_with_padding(idm, 3) + "_cfbf" + + to_string_with_padding(ibeam, 5); + std::string beam_filename = prefix + ".fil"; + _beam_filenames.push_back(beam_filename); + + std::string beam_filepath = output_dir + "/" + beam_filename; + BOOST_LOG_TRIVIAL(info) << "Beam file path: " << beam_filepath; + MultiFileWriterConfig writer_config; + + writer_config.header_size = _config.dada_header_size(); + writer_config.max_file_size = _config.max_output_filesize(); + writer_config.stokes_mode = _config.stokes_mode(); + writer_config.output_dir = output_dir; + writer_config.prefix = prefix; + writer_config.extension = ".fil"; + + BOOST_LOG_TRIVIAL(info) + << "Writer config: " << writer_config.to_string(); + + typename MultiFileWriter::CreateStreamCallBackType + create_stream_callback_sigproc = + skyweaver::detail::create_sigproc_file_stream< + OutputVectorType>; + _beam_writers[idm][ibeam] = + std::make_unique>( + writer_config, + "", + create_stream_callback_sigproc); + _header.ibeam = ibeam; + _beam_writers[idm][ibeam]->init(_header); + + _beam_data[idm][ibeam] = std::make_shared( + std::initializer_list{_config.nsamples_per_block(), + _config.nbridges()}, + 0); + + _total_beam_writers++; + } + } + + BOOST_LOG_TRIVIAL(info) + << "Added " << _total_beam_writers << " beam writers to SkyCleaver"; +} + +SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) +{ + _timer.start("skycleaver::init_readers"); + init_readers(); + _timer.stop("skycleaver::init_readers"); + _timer.start("skycleaver::init_writers"); + init_writers(); + _timer.stop("skycleaver::init_writers"); +} + +void SkyCleaver::cleave() +{ + BOOST_LOG_NAMED_SCOPE("SkyCleaver::cleave") + + for(std::size_t nsamples_read = 0; nsamples_read < _nsamples_to_read; + nsamples_read += _config.nsamples_per_block()) { + std::size_t gulp_samples = _nsamples_to_read - nsamples_read < _config.nsamples_per_block() + ? _nsamples_to_read - nsamples_read + : _config.nsamples_per_block(); + + BOOST_LOG_TRIVIAL(info) << "Cleaving samples: " << nsamples_read + << " to " << nsamples_read + gulp_samples; + + std::size_t gulp_size = + gulp_samples * _config.ndms() * _config.nbeams(); + + int nthreads_read = _config.nthreads() > _config.nbridges() + ? _config.nbridges() + : _config.nthreads(); + + omp_set_num_threads(nthreads_read); + + _timer.start("skyweaver::read_data"); + + std::vector read_status(_available_freqs.size(), false); // since we cannot throw exceptions in parallel regions +#pragma omp parallel for + for(std::size_t i = 0; i < _available_freqs.size(); i++) { + FreqType freq = _available_freqs[i]; + if(_bridge_readers.find(freq) == _bridge_readers.end()) { + read_status[i] = true; + } + const auto& reader = _bridge_readers[freq]; + if(reader->eof()) { + read_status[i] = true; + } + + std::streamsize read_size = + reader->read(reinterpret_cast(thrust::raw_pointer_cast( + _bridge_data[freq]->data())), + gulp_size); // read a big chunk of data + BOOST_LOG_TRIVIAL(info) << "Read " << read_size << " bytes from bridge" << freq; + if(read_size < gulp_size * sizeof(InputVectorType::value_type)) { + BOOST_LOG_TRIVIAL(warning) + << "Read less data than expected from bridge " << freq; + read_status[i] = true; + + } + } + + if(std::any_of(read_status.begin(), read_status.end(), + [](bool status) { return status; })) { + std::runtime_error("Some bridges have had unexpected reads"); + } + + BOOST_LOG_TRIVIAL(info) << "Read data from bridge readers"; + + _timer.stop("skyweaver::read_data"); + _timer.start("skyweaver::process_data"); + + omp_set_num_threads(_config.nthreads()); + + std::size_t nbridges = _config.nbridges(); + std::size_t nsamples_per_block = _config.nsamples_per_block(); + std::size_t ndms = _config.ndms(); + std::size_t nbeams = _config.nbeams(); + +#pragma omp parallel for schedule(static) collapse(2) + for(std::size_t idm = 0; idm < ndms; idm++) { + for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { +#pragma omp simd + for(std::size_t isample = 0; isample < nsamples_per_block; + isample++) { + const std::size_t base_index = + isample * ndms * nbeams + idm * nbeams + ibeam; + + std::size_t ifreq = 0; + std::size_t out_offset = isample * nbridges; + for(const auto& [freq, ifreq_data]: _bridge_data) { + _beam_data[idm][ibeam]->at(out_offset + nbridges - 1 -ifreq++) = + ifreq_data->at(base_index); + } + } + } + } + BOOST_LOG_TRIVIAL(info) << "Processed data"; + _timer.stop("skyweaver::process_data"); + + _timer.start("skyweaver::write_data"); + +#pragma omp parallel for schedule(static) collapse(2) + for(int idm = 0; idm < _config.ndms(); idm++) { + for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], 0); + } + } + + _timer.stop("skyweaver::write_data"); + } + _timer.show_all_timings(); + +} diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cu new file mode 100644 index 0000000..ff7ad8f --- /dev/null +++ b/cpp/skyweaver/src/skycleaver_cli.cu @@ -0,0 +1,228 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "skyweaver/logging.hpp" +#include "skyweaver/SkyCleaverConfig.hpp" +#include "skyweaver/SkyCleaver.cuh" +#define BOOST_LOG_DYN_LINK 1 + + +namespace +{ + + +std::string skycleaver_splash=R"( + __ __ +.-----.| |--.--.--.----.| |.-----.---.-.--.--.-----.----. +|__ --|| <| | | __|| || -__| _ | | | -__| _| +|_____||__|__|___ |____||__||_____|___._|\___/|_____|__| + |_____| + +)"; +const size_t ERROR_IN_COMMAND_LINE = 1; +const size_t SUCCESS = 0; +const size_t ERROR_UNHANDLED_EXCEPTION = 2; + +const char* build_time = __DATE__ " " __TIME__; + + +} + +namespace std +{ +std::ostream& operator<<(std::ostream& os, const std::vector& vec) +{ + for(auto item: vec) { os << item << " "; } + return os; +} +} // namespace std + + + + +int main(int argc, char** argv) +{ + std::cout << skycleaver_splash; + std::cout << "Build time: " << build_time << std::endl; + // skyweaver::init_logging("warning"); + + skyweaver::SkyCleaverConfig config; + + + namespace po = boost::program_options; + + po::options_description generic("Generic options"); + generic.add_options()("cfg,c", + po::value()->default_value(""), + "Skycleaver configuration file"); + + po::options_description main_options("Main options"); + main_options.add_options() + ("help,h", "Produce help message") + ("root-dir,r", + po::value() + -> required() + ->notifier( + [&config](std::string key) { config.root_dir(key); }), + "The output directory for all results") + ("output-dir", + po::value() + ->default_value(config.output_dir()) + ->notifier( + [&config](std::string key) { config.output_dir(key); }), + "The output directory for all results") + ("root-prefix", + po::value() + ->default_value(config.root_prefix()) + ->notifier( + [&config](std::string key) { config.root_prefix(key); }), + "The prefix for all output files") + ("out-prefix", + po::value() + ->default_value(config.out_prefix()) + ->notifier( + [&config](std::string key) { config.out_prefix(key); }), + "The prefix for all output files") + ("nthreads", + po::value() + ->default_value(config.nthreads()) + ->notifier( + [&config](unsigned int key) { config.nthreads(key); }), + "The number of threads to use for processing") + ("nsamples-per-block", + po::value() + ->default_value(config.nsamples_per_block()) + ->notifier( + [&config](std::size_t key) { config.nsamples_per_block(key); }), + "The number of samples per block") + ("nchans", + po::value() + ->default_value(config.nchans()) + ->notifier( + [&config](std::size_t key) { config.nchans(key); }), + "The number of channels") + ("nbridges", + po::value() + ->default_value(config.nbridges()) + ->notifier( + [&config](std::size_t key) { config.nbridges(key); }), + "The number of bridges") + ("nbeams", + po::value() + ->default_value(config.nbeams()) + ->notifier( + [&config](std::size_t key) { config.nbeams(key); }), + "The number of beams") + ("ndms", + po::value() + ->default_value(config.ndms()) + ->notifier( + [&config](std::size_t key) { config.ndms(key); }), + "The number of DMs") + ("stokes-mode", + po::value() + ->default_value(config.stokes_mode()) + ->notifier( + [&config](std::string key) { config.stokes_mode(key); }), + "The stokes mode") + ("stream-id", + po::value() + ->default_value(config.stream_id()) + ->notifier( + [&config](std::size_t key) { config.stream_id(key); }), + "The stream id") + ("max-ram-gb", + po::value() + ->default_value(config.max_ram_gb()) + ->notifier( + [&config](std::size_t key) { config.max_ram_gb(key); }), + "The maximum amount of RAM to use in GB") + ("max-output-filesize", + po::value() + ->default_value(config.max_output_filesize()) + ->notifier( + [&config](std::size_t key) { config.max_output_filesize(key); }), + "The maximum output file size in bytes") + ("dada-header-size", + po::value() + ->default_value(config.dada_header_size()) + ->notifier( + [&config](std::size_t key) { config.dada_header_size(key); }), + "The size of the DADA header") + ("log-level", + po::value()->default_value("info")->notifier( + [](std::string level) { skyweaver::init_logging(level); }), + "The logging level to use (debug, info, warning, error)"); + + po::options_description cmdline_options; + cmdline_options.add(generic).add(main_options); + + // set options allowed in config file + po::options_description config_file_options; + config_file_options.add(main_options); + po::variables_map variable_map; + try { + po::store(po::command_line_parser(argc, argv) + .options(cmdline_options) + .run(), + variable_map); + if(variable_map.count("help")) { + std::cout << "skycleaver -- A pipeline that cleaves input TDB files, " + "and cleaves them to form output Sigproc Filterbank files." + << std::endl + << cmdline_options << std::endl; + return SUCCESS; + } + } catch(po::error& e) { + std::cerr << "ERROR: " << e.what() << std::endl << std::endl; + return ERROR_IN_COMMAND_LINE; + } + + auto config_file = variable_map.at("cfg").as(); + + if(config_file != "") { + std::ifstream config_fs(config_file.c_str()); + if(!config_fs.is_open()) { + std::cerr << "Unable to open configuration file: " + << config_file << " (" << std::strerror(errno) + << ")\n"; + return ERROR_UNHANDLED_EXCEPTION; + } else { + po::store(po::parse_config_file(config_fs, config_file_options), + variable_map); + } + } + po::notify(variable_map); + BOOST_LOG_NAMED_SCOPE("skycleaver_cli"); + BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; + BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); + BOOST_LOG_TRIVIAL(info) << "output_dir: " << config.output_dir(); + BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); + BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); + BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); + BOOST_LOG_TRIVIAL(info) << "nsamples_per_block: " << config.nsamples_per_block(); + BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); + BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); + BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); + BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); + + + skyweaver::SkyCleaver skycleaver(config); + skycleaver.cleave(); + + +} \ No newline at end of file diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 456aa69..e168f63 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -52,7 +52,7 @@ class NullHandler { public: template - void init(Args... args) {}; + void init(Args... args){}; template bool operator()(Args... args) @@ -254,14 +254,30 @@ void setup_pipeline(skyweaver::PipelineConfig& config) update_config(config, header); BOOST_LOG_TRIVIAL(debug) << "Creating pipeline handlers"; using OutputType = typename BfTraits::QuantisedPowerType; - skyweaver::MultiFileWriter> ib_handler( - config, - "ib"); - skyweaver::MultiFileWriter> - stats_handler(config, "stats"); + + using IBWriterType = + skyweaver::MultiFileWriter>; + typename IBWriterType::CreateStreamCallBackType create_stream_callback_ib = + skyweaver::detail::create_dada_file_stream< + skyweaver::BTFPowersH>; + IBWriterType ib_handler(config, "ib", create_stream_callback_ib); + + using StatsWriterType = + skyweaver::MultiFileWriter>; + typename StatsWriterType::CreateStreamCallBackType + create_stream_callback_stats = + skyweaver::detail::create_dada_file_stream< + skyweaver::FPAStatsD>; + StatsWriterType stats_handler(config, "stats", create_stream_callback_stats); + + if constexpr(enable_incoherent_dedispersion) { + using CBWriterType = skyweaver::MultiFileWriter>; + typename CBWriterType::CreateStreamCallBackType + create_stream_callback_cb = + skyweaver::detail::create_dada_file_stream>; skyweaver::MultiFileWriter> - cb_file_writer(config, "cb"); + cb_file_writer(config, "cb", create_stream_callback_cb); skyweaver::IncoherentDedispersionPipeline @@ -276,8 +292,11 @@ void setup_pipeline(skyweaver::PipelineConfig& config) stats_handler); run_pipeline(pipeline, config, file_reader, header); } else { - skyweaver::MultiFileWriter> - cb_file_writer(config, "cb"); + using CBWriterType = skyweaver::MultiFileWriter>; + typename CBWriterType::CreateStreamCallBackType + create_stream_callback_cb = + skyweaver::detail::create_dada_file_stream>; + CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb); skyweaver::BeamformerPipelinedefault_value(config.gulp_length_samps()) ->notifier([&config](std::size_t const& gulp_size) { // Round off to next multiple of 256 - if(gulp_size % config.nsamples_per_heap() != 0 || //256 - gulp_size % config.nsamples_per_block() != 0) { // 32* tscrunch - - std::size_t larger_value = std::max( - config.nsamples_per_heap(), - config.nsamples_per_block()); + if(gulp_size % config.nsamples_per_heap() != 0 || // 256 + gulp_size % config.nsamples_per_block() != + 0) { // 32* tscrunch + + std::size_t larger_value = + std::max(config.nsamples_per_heap(), + config.nsamples_per_block()); BOOST_LOG_TRIVIAL(warning) - << "Rounding up gulp-size to next multiple of NSAMPLES PER HEAP and NSAMPLES PER BLOCK"; + << "Rounding up gulp-size to next multiple of " + "NSAMPLES PER HEAP and NSAMPLES PER BLOCK"; - config.gulp_length_samps( - (gulp_size / larger_value) * - larger_value); + config.gulp_length_samps((gulp_size / larger_value) * + larger_value); } else { config.gulp_length_samps(gulp_size); } diff --git a/cpp/skyweaver/test/src/BeamformerPipelineTester.cu b/cpp/skyweaver/test/src/BeamformerPipelineTester.cu index df21a32..44720b9 100644 --- a/cpp/skyweaver/test/src/BeamformerPipelineTester.cu +++ b/cpp/skyweaver/test/src/BeamformerPipelineTester.cu @@ -146,9 +146,9 @@ TYPED_TEST(BeamformerPipelineTester, full_pipeline_test) read_dada_header(raw_header, header); validate_header(header, this->_config); update_config(this->_config, header); - - MultiFileWriter> - cb_handler(this->_config, "cb"); + using WriterType = MultiFileWriter>; + typename WriterType::CreateStreamCallBackType create_stream_callback = detail::create_dada_file_stream>; + WriterType cb_handler(_config, "cb", create_stream_callback); NullHandler ib_handler; NullHandler stats_handler; using IDPipelineType = diff --git a/cpp/skyweaver/test/src/MultiFileWriterTester.cu b/cpp/skyweaver/test/src/MultiFileWriterTester.cu index 32ee1aa..2b797f8 100644 --- a/cpp/skyweaver/test/src/MultiFileWriterTester.cu +++ b/cpp/skyweaver/test/src/MultiFileWriterTester.cu @@ -1,5 +1,6 @@ #include "skyweaver/DescribedVector.hpp" #include "skyweaver/test/MultiFileWriterTester.cuh" +#include "skyweaver/MultiFileWriter.cuh" #include #include @@ -43,7 +44,12 @@ TEST_F(MultiFileWriterTester, simple_updating_write) BOOST_LOG_TRIVIAL(debug) << "Testing in tmp directory: " << _config.output_dir(); - MultiFileWriter mfw(_config); + + using WriterType = MultiFileWriter; + typename WriterType::CreateStreamCallBackType create_stream_callback = detail::create_dada_file_stream; + + + WriterType mfw(_config, "test", create_stream_callback); _config.max_output_filesize(1000); InputType powers({_config.nsamples_per_block(), 64, _config.nbeams()}); powers.dms({0.0}); From efeffb7592694b01ff83f3de1e638c58b3f387b6 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 22 Aug 2024 17:33:59 +0200 Subject: [PATCH 02/65] Change source name and position from default values in header --- cpp/skyweaver/detail/file_writer_callbacks.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index f16150b..70069ad 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -106,6 +106,9 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig header_size, false); Header header_writer(bytes); + header_writer.set("SOURCE", header.source_name); + header_writer.set("RA", header.ra); + header_writer.set("DEC", header.dec); header_writer.set("NBEAM", stream_data.nbeams()); header_writer.set("NCHAN", stream_data.nchannels()); header_writer.set("OBS_NCHAN", header.obs_nchans); @@ -184,7 +187,7 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon double za = 0.0; uint32_t machineid = 0; uint32_t nifs = header.npol; - uint32_t telescopeid = 0; + uint32_t telescopeid = 64; header.sigproc_params = true; header.rawfile = std::string("unset"); From 63c854b5b77cffaef7b89c7379e3e53d1af68599 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Tue, 3 Sep 2024 10:56:24 +0200 Subject: [PATCH 03/65] seek to the correct timestamp --- cpp/skyweaver/SkyCleaver.cuh | 1 + cpp/skyweaver/src/BufferedDispenser.cu | 2 +- cpp/skyweaver/src/CoherentDedisperser.cu | 2 + cpp/skyweaver/src/SkyCleaver.cu | 164 +++++++++++++---------- 4 files changed, 98 insertions(+), 71 deletions(-) diff --git a/cpp/skyweaver/SkyCleaver.cuh b/cpp/skyweaver/SkyCleaver.cuh index 9e4b86e..6db9237 100644 --- a/cpp/skyweaver/SkyCleaver.cuh +++ b/cpp/skyweaver/SkyCleaver.cuh @@ -47,6 +47,7 @@ class SkyCleaver SkyCleaverConfig const& _config; std::map> _bridge_readers; std::map> _bridge_data; // shared ptr so that we can reuse zero data + std::vector _expected_freqs; std::vector _available_freqs; std::size_t _nsamples_to_read; diff --git a/cpp/skyweaver/src/BufferedDispenser.cu b/cpp/skyweaver/src/BufferedDispenser.cu index bb5eebf..ffa6d32 100644 --- a/cpp/skyweaver/src/BufferedDispenser.cu +++ b/cpp/skyweaver/src/BufferedDispenser.cu @@ -21,7 +21,7 @@ BufferedDispenser::BufferedDispenser(PipelineConfig const& config, for(std::size_t i = 0; i < _config.nchans(); i++) { _d_channeled_tpa_voltages[i].resize( - {_config.gulp_length_samps() + dedisp_config.overlap_samps, + { _config.gulp_length_samps() + dedisp_config.overlap_samps, _config.npol(), _config.nantennas()}); _d_prev_channeled_tpa_voltages[i].resize( diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 08d176e..6025313 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -26,6 +26,8 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, pipeline_config.centre_frequency() + pipeline_config.bandwidth() / 2.0f; float tsamp = pipeline_config.nchans() / pipeline_config.bandwidth(); + + if(pipeline_config.coherent_dms().empty()) { throw std::runtime_error("No coherent DMs specified"); } diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index 16a6280..b40b1f4 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -101,11 +101,12 @@ void SkyCleaver::init_readers() std::vector freq_dirs = get_subdirs(root_dir + "/" + std::to_string(stream_id)); - BOOST_LOG_TRIVIAL(info) - << "Found " << freq_dirs.size() - << " frequency directories in root directory: " << root_dir; + BOOST_LOG_TRIVIAL(info) << "Found " << freq_dirs.size() << " frequency directories in root directory: " << root_dir; + + + std::map bridge_timestamps; + long double latest_timestamp = 0.0; - std::size_t smallest_data_size = std::numeric_limits::max(); for(const auto& freq_dir: freq_dirs) { std::vector tdb_files = get_files( @@ -125,47 +126,23 @@ void SkyCleaver::init_readers() std::make_unique(tdb_files, _config.dada_header_size(), false); - _available_freqs.push_back(freq); - std::size_t data_size = - _bridge_readers[freq]->get_total_size(); - BOOST_LOG_TRIVIAL(debug) - << "Data size for frequency: " << freq_dir << " is " << data_size; - if(data_size < smallest_data_size) { - smallest_data_size = data_size; + long double timestamp = _bridge_readers[freq]->get_header().utc_start; + bridge_timestamps.insert({freq, timestamp}); + if(timestamp > latest_timestamp) { + latest_timestamp = timestamp; } + _available_freqs.push_back(freq); BOOST_LOG_TRIVIAL(debug) << "Added bridge reader for frequency: " << freq_dir; } - BOOST_LOG_TRIVIAL(debug) << "Smallest data size: " << smallest_data_size; - BOOST_LOG_TRIVIAL(debug) << "ndm: " << _config.ndms(); - BOOST_LOG_TRIVIAL(debug) << "nbeams: " << _config.nbeams(); - - if(smallest_data_size % (_config.ndms() * _config.nbeams()) != 0) { - std::runtime_error("Data size is not a multiple of ndms * nbeams"); - } - - std::size_t smallest_nsamples = - std::floor(smallest_data_size / _config.ndms() / _config.nbeams()); - - BOOST_LOG_TRIVIAL(info) - << "Smallest data size: " << smallest_data_size - << " Smallest number of samples: " << smallest_nsamples; - - if(smallest_nsamples < _config.nsamples_per_block()) { - std::runtime_error( - "Smallest data size is less than nsamples_per_block"); - } - - _nsamples_to_read = smallest_nsamples; - - BOOST_LOG_TRIVIAL(info) - << "Added " << _bridge_readers.size() << " bridges to SkyCleaver"; + int nbridges = _config.nbridges(); - // read the first file of first bridge to get the header + std::size_t gulp_size = + _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); - ObservationHeader header = (*_bridge_readers.begin()).second->get_header(); + ObservationHeader const& header = (*_bridge_readers.begin()).second->get_header(); BOOST_LOG_TRIVIAL(info) << "Read header from first file: " << header.to_string(); @@ -174,18 +151,6 @@ void SkyCleaver::init_readers() obs_centre_freq = 1284000000; } float obs_bandwidth = header.obs_bandwidth; - int obs_nchans = header.obs_nchans; - - BOOST_LOG_TRIVIAL(info) << "Centre frequency: " << obs_centre_freq - << " Bandwidth: " << obs_bandwidth - << " Number of obs channels: " << obs_nchans; - int nbridges = _config.nbridges(); - - std::size_t gulp_size = - _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); - // std::unique_ptr zero_data = - // std::make_unique(gulp_size, 0); - for(int i = 0; i < nbridges; i++) { int ifreq = std::lround(obs_centre_freq - obs_bandwidth / 2 + (i + 0.5) * obs_bandwidth / nbridges); @@ -205,27 +170,80 @@ void SkyCleaver::init_readers() 0); } - // at this point, all non-existed frequencies have been added with zero data - // now check if there are any unexpected frequencies in the bridge readers + std::size_t smallest_data_size = std::numeric_limits::max(); for(const auto& [freq, reader]: _bridge_readers) { + + // at this point, all non-existed frequencies have been added with zero data + // now check if there are any unexpected frequencies in the bridge readers. if(std::find(_expected_freqs.begin(), _expected_freqs.end(), freq) == _expected_freqs.end()) { throw std::runtime_error("Frequency " + std::to_string(freq) + " not found in expected frequencies"); } + + // now time align all the bridges to the latest timestamp + long double timestamp = bridge_timestamps[freq]; + long double time_diff = latest_timestamp - timestamp; + long double tsamp = reader->get_header().tsamp * 1e-6; // Header has it in microseconds, converting to seconds + std::size_t nsamples = std::floor(time_diff / tsamp); + + BOOST_LOG_TRIVIAL(info) + << "Frequency: " << freq << " Timestamp: " << timestamp << "tsamp: " << tsamp + << " Latest timestamp: " << latest_timestamp + << " Time difference: " << time_diff + << " Number of samples to skip: " << nsamples; + + BOOST_LOG_TRIVIAL(info) + << "Seeking " << nsamples * _config.ndms() * _config.nbeams() + << " bytes in bridge reader for frequency: " << freq; + + std::size_t bytes_seeking = nsamples * _config.ndms() * _config.nbeams() * + sizeof(InputVectorType::value_type); + + _bridge_readers[freq]->seekg(nsamples * _config.ndms() * _config.nbeams() * sizeof(InputVectorType::value_type), + std::ios_base::beg); + + std::size_t data_size = _bridge_readers[freq]->get_total_size() - bytes_seeking; + BOOST_LOG_TRIVIAL(debug) + << "Data size for frequency: " << freq << " is " << data_size; + if(data_size < smallest_data_size) { + smallest_data_size = data_size; + } + + } + + BOOST_LOG_TRIVIAL(debug) << "Smallest data size: " << smallest_data_size; + BOOST_LOG_TRIVIAL(debug) << "ndm: " << _config.ndms(); + BOOST_LOG_TRIVIAL(debug) << "nbeams: " << _config.nbeams(); + + if(smallest_data_size % (_config.ndms() * _config.nbeams()) != 0) { + std::runtime_error("Data size is not a multiple of ndms * nbeams"); } + std::size_t smallest_nsamples = + std::floor(smallest_data_size / _config.ndms() / _config.nbeams()); + + BOOST_LOG_TRIVIAL(info) + << "Smallest data size: " << smallest_data_size + << " Smallest number of samples: " << smallest_nsamples; + + if(smallest_nsamples < _config.nsamples_per_block()) { + std::runtime_error( + "Smallest data size is less than nsamples_per_block"); + } + + _nsamples_to_read = smallest_nsamples; + BOOST_LOG_TRIVIAL(info) << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; - BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver"; - BOOST_LOG_TRIVIAL(info) << "Header: " << header.to_string(); - _header = header; + _header = _bridge_readers[_available_freqs[0]]->get_header(); + BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver"; + BOOST_LOG_TRIVIAL(info) << "Header: " << _header.to_string(); _header.nchans = _header.nchans * _config.nbridges(); _header.nbeams = _config.nbeams(); - } void SkyCleaver::init_writers() @@ -302,9 +320,10 @@ void SkyCleaver::cleave() for(std::size_t nsamples_read = 0; nsamples_read < _nsamples_to_read; nsamples_read += _config.nsamples_per_block()) { - std::size_t gulp_samples = _nsamples_to_read - nsamples_read < _config.nsamples_per_block() - ? _nsamples_to_read - nsamples_read - : _config.nsamples_per_block(); + std::size_t gulp_samples = + _nsamples_to_read - nsamples_read < _config.nsamples_per_block() + ? _nsamples_to_read - nsamples_read + : _config.nsamples_per_block(); BOOST_LOG_TRIVIAL(info) << "Cleaving samples: " << nsamples_read << " to " << nsamples_read + gulp_samples; @@ -320,7 +339,9 @@ void SkyCleaver::cleave() _timer.start("skyweaver::read_data"); - std::vector read_status(_available_freqs.size(), false); // since we cannot throw exceptions in parallel regions + std::vector read_status( + _available_freqs.size(), + false); // since we cannot throw exceptions in parallel regions #pragma omp parallel for for(std::size_t i = 0; i < _available_freqs.size(); i++) { FreqType freq = _available_freqs[i]; @@ -336,17 +357,18 @@ void SkyCleaver::cleave() reader->read(reinterpret_cast(thrust::raw_pointer_cast( _bridge_data[freq]->data())), gulp_size); // read a big chunk of data - BOOST_LOG_TRIVIAL(info) << "Read " << read_size << " bytes from bridge" << freq; + BOOST_LOG_TRIVIAL(info) + << "Read " << read_size << " bytes from bridge" << freq; if(read_size < gulp_size * sizeof(InputVectorType::value_type)) { BOOST_LOG_TRIVIAL(warning) << "Read less data than expected from bridge " << freq; read_status[i] = true; - } } - - if(std::any_of(read_status.begin(), read_status.end(), - [](bool status) { return status; })) { + + if(std::any_of(read_status.begin(), read_status.end(), [](bool status) { + return status; + })) { std::runtime_error("Some bridges have had unexpected reads"); } @@ -368,15 +390,18 @@ void SkyCleaver::cleave() #pragma omp simd for(std::size_t isample = 0; isample < nsamples_per_block; isample++) { - const std::size_t base_index = - isample * ndms * nbeams + idm * nbeams + ibeam; + const std::size_t base_index = isample * ndms * nbeams + + idm * nbeams + ibeam; std::size_t ifreq = 0; std::size_t out_offset = isample * nbridges; - for(const auto& [freq, ifreq_data]: _bridge_data) { - _beam_data[idm][ibeam]->at(out_offset + nbridges - 1 -ifreq++) = + for(const auto& [freq, ifreq_data]: _bridge_data) { // for each frequency + _beam_data[idm][ibeam]->at(out_offset + nbridges - 1 - + ifreq++) = ifreq_data->at(base_index); } + + } } } @@ -388,12 +413,11 @@ void SkyCleaver::cleave() #pragma omp parallel for schedule(static) collapse(2) for(int idm = 0; idm < _config.ndms(); idm++) { for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], 0); + _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], _config.stream_id()); } } _timer.stop("skyweaver::write_data"); } _timer.show_all_timings(); - } From 0201f7ee42062bec3f4f535da812705bc07f3364 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 4 Sep 2024 17:42:10 +0200 Subject: [PATCH 04/65] use uint8 for filterbank output --- cpp/skyweaver/SkyCleaver.cuh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/skyweaver/SkyCleaver.cuh b/cpp/skyweaver/SkyCleaver.cuh index 6db9237..8c91290 100644 --- a/cpp/skyweaver/SkyCleaver.cuh +++ b/cpp/skyweaver/SkyCleaver.cuh @@ -35,7 +35,7 @@ class SkyCleaver { public: using InputType = int8_t; - using OutputType = int8_t; + using OutputType = uint8_t; using InputVectorType = TDBPowersH; using OutputVectorType = TFPowersH; using FreqType = std::size_t; // up to the nearest Hz @@ -46,7 +46,7 @@ class SkyCleaver private: SkyCleaverConfig const& _config; std::map> _bridge_readers; - std::map> _bridge_data; // shared ptr so that we can reuse zero data + std::map> _bridge_data; std::vector _expected_freqs; std::vector _available_freqs; From 331c1cffdc1d530877a643243c7de5fdb389feed Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 4 Sep 2024 17:42:36 +0200 Subject: [PATCH 05/65] Correct frequency tagging of tdb outputs --- .../detail/IncoherentDedispersionPipeline.cu | 2 +- cpp/skyweaver/detail/file_writer_callbacks.cpp | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index 824f1ad..2c664ce 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -73,7 +73,7 @@ void IncoherentDedispersionPipeline:: // Set the correct DMs on the block _output_buffers[ref_dm_idx].dms(plan[ref_dm_idx].incoherent_dms); _output_buffers[ref_dm_idx].reference_dm(plan[ref_dm_idx].coherent_dm); - _output_buffers[ref_dm_idx].frequencies({_config.centre_frequency()}); + _output_buffers[ref_dm_idx].frequencies({_config.centre_frequency() - _config.bandwidth() / 2.0}); BOOST_LOG_TRIVIAL(debug) << "Passing output buffer to handler: " << _output_buffers[ref_dm_idx].describe(); diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 70069ad..da380ff 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -123,7 +123,16 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig header_writer.set( "COHERENT_DM", static_cast(stream_data.reference_dm())); - header_writer.set("FREQ", header.frequency); + try{ + header_writer.set("FREQ", std::accumulate( + stream_data.frequencies().begin(), + stream_data.frequencies().end(), + 0.0)/stream_data.frequencies().size()); + } catch(std::runtime_error& ){ + BOOST_LOG_TRIVIAL(warning) << "Warning: Frequencies array was stale, using the centre frequency from the header"; + header_writer.set("FREQ", header.frequency); + } + header_writer.set("BW", header.bandwidth); header_writer.set("TSAMP", stream_data.tsamp() * 1e6); if(config.stokes_mode == "IQUV") { From 5dc0bf425c74ca13d8817da7920f469f35d81e2a Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 4 Sep 2024 17:43:52 +0200 Subject: [PATCH 06/65] swap beam and dm loops, convert signed to unsigned by rescaling data --- cpp/skyweaver/src/SkyCleaver.cu | 70 ++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index b40b1f4..6ec7574 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -5,14 +5,16 @@ #include #include #include +#include "skyweaver/types.cuh" +#include "skyweaver/SkyCleaver.cuh" + namespace fs = std::filesystem; +using SkyCleaver = skyweaver::SkyCleaver; -#include "skyweaver/SkyCleaver.cuh" using FreqType = skyweaver::SkyCleaver::FreqType; using BridgeReader = skyweaver::BridgeReader; using MultiFileReader = skyweaver::MultiFileReader; -using SkyCleaver = skyweaver::SkyCleaver; using OutputVectorType = skyweaver::SkyCleaver::OutputVectorType; using InputVectorType = skyweaver::SkyCleaver::InputVectorType; @@ -101,13 +103,13 @@ void SkyCleaver::init_readers() std::vector freq_dirs = get_subdirs(root_dir + "/" + std::to_string(stream_id)); - BOOST_LOG_TRIVIAL(info) << "Found " << freq_dirs.size() << " frequency directories in root directory: " << root_dir; - + BOOST_LOG_TRIVIAL(info) + << "Found " << freq_dirs.size() + << " frequency directories in root directory: " << root_dir; std::map bridge_timestamps; long double latest_timestamp = 0.0; - for(const auto& freq_dir: freq_dirs) { std::vector tdb_files = get_files( root_dir + "/" + std::to_string(stream_id) + "/" + freq_dir, @@ -142,7 +144,8 @@ void SkyCleaver::init_readers() std::size_t gulp_size = _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); - ObservationHeader const& header = (*_bridge_readers.begin()).second->get_header(); + ObservationHeader const& header = + (*_bridge_readers.begin()).second->get_header(); BOOST_LOG_TRIVIAL(info) << "Read header from first file: " << header.to_string(); @@ -173,9 +176,9 @@ void SkyCleaver::init_readers() std::size_t smallest_data_size = std::numeric_limits::max(); for(const auto& [freq, reader]: _bridge_readers) { - - // at this point, all non-existed frequencies have been added with zero data - // now check if there are any unexpected frequencies in the bridge readers. + // at this point, all non-existed frequencies have been added with zero + // data now check if there are any unexpected frequencies in the bridge + // readers. if(std::find(_expected_freqs.begin(), _expected_freqs.end(), freq) == _expected_freqs.end()) { throw std::runtime_error("Frequency " + std::to_string(freq) + @@ -185,12 +188,14 @@ void SkyCleaver::init_readers() // now time align all the bridges to the latest timestamp long double timestamp = bridge_timestamps[freq]; long double time_diff = latest_timestamp - timestamp; - long double tsamp = reader->get_header().tsamp * 1e-6; // Header has it in microseconds, converting to seconds + long double tsamp = + reader->get_header().tsamp * + 1e-6; // Header has it in microseconds, converting to seconds std::size_t nsamples = std::floor(time_diff / tsamp); BOOST_LOG_TRIVIAL(info) - << "Frequency: " << freq << " Timestamp: " << timestamp << "tsamp: " << tsamp - << " Latest timestamp: " << latest_timestamp + << "Frequency: " << freq << " Timestamp: " << timestamp + << "tsamp: " << tsamp << " Latest timestamp: " << latest_timestamp << " Time difference: " << time_diff << " Number of samples to skip: " << nsamples; @@ -198,19 +203,22 @@ void SkyCleaver::init_readers() << "Seeking " << nsamples * _config.ndms() * _config.nbeams() << " bytes in bridge reader for frequency: " << freq; - std::size_t bytes_seeking = nsamples * _config.ndms() * _config.nbeams() * + std::size_t bytes_seeking = nsamples * _config.ndms() * + _config.nbeams() * sizeof(InputVectorType::value_type); - _bridge_readers[freq]->seekg(nsamples * _config.ndms() * _config.nbeams() * sizeof(InputVectorType::value_type), + _bridge_readers[freq]->seekg(nsamples * _config.ndms() * + _config.nbeams() * + sizeof(InputVectorType::value_type), std::ios_base::beg); - std::size_t data_size = _bridge_readers[freq]->get_total_size() - bytes_seeking; + std::size_t data_size = + _bridge_readers[freq]->get_total_size() - bytes_seeking; BOOST_LOG_TRIVIAL(debug) << "Data size for frequency: " << freq << " is " << data_size; if(data_size < smallest_data_size) { smallest_data_size = data_size; } - } BOOST_LOG_TRIVIAL(debug) << "Smallest data size: " << smallest_data_size; @@ -238,10 +246,9 @@ void SkyCleaver::init_readers() BOOST_LOG_TRIVIAL(info) << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; - - _header = _bridge_readers[_available_freqs[0]]->get_header(); + _header = _bridge_readers[_available_freqs[0]]->get_header(); BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver"; - BOOST_LOG_TRIVIAL(info) << "Header: " << _header.to_string(); + BOOST_LOG_TRIVIAL(info) << "Header: " << _header.to_string(); _header.nchans = _header.nchans * _config.nbridges(); _header.nbeams = _config.nbeams(); } @@ -385,23 +392,21 @@ void SkyCleaver::cleave() std::size_t nbeams = _config.nbeams(); #pragma omp parallel for schedule(static) collapse(2) - for(std::size_t idm = 0; idm < ndms; idm++) { - for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { + for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { + for(std::size_t idm = 0; idm < ndms; idm++) { #pragma omp simd - for(std::size_t isample = 0; isample < nsamples_per_block; - isample++) { - const std::size_t base_index = isample * ndms * nbeams - + idm * nbeams + ibeam; + for(std::size_t isample = 0; isample < nsamples_per_block; isample++) { + const std::size_t base_index = + isample * ndms * nbeams + idm * nbeams + ibeam; std::size_t ifreq = 0; - std::size_t out_offset = isample * nbridges; - for(const auto& [freq, ifreq_data]: _bridge_data) { // for each frequency + const std::size_t out_offset = isample * nbridges; + for(const auto& [freq, ifreq_data]: + _bridge_data) { // for each frequency _beam_data[idm][ibeam]->at(out_offset + nbridges - 1 - - ifreq++) = - ifreq_data->at(base_index); + ifreq) = clamp(127 + ifreq_data->at(base_index)); + ++ifreq; } - - } } } @@ -413,7 +418,8 @@ void SkyCleaver::cleave() #pragma omp parallel for schedule(static) collapse(2) for(int idm = 0; idm < _config.ndms(); idm++) { for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], _config.stream_id()); + _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], + _config.stream_id()); } } From a82358747f4d98d13907d4ceb014442394043bcd Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 11 Sep 2024 17:26:46 +0200 Subject: [PATCH 07/65] add has key, get_or_default methods to header --- cpp/skyweaver/Header.hpp | 21 +++++++++++++++++++++ cpp/skyweaver/src/Header.cpp | 9 ++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/Header.hpp b/cpp/skyweaver/Header.hpp index 6b5e18a..7945eea 100644 --- a/cpp/skyweaver/Header.hpp +++ b/cpp/skyweaver/Header.hpp @@ -37,6 +37,26 @@ class Header template T get(char const* key) const; + + /** + * @brief Get a value from the header or return a default value + * + * @param key An ASCII key (the name of the value to get) + * @param[in] default_value The default value to return if the key is not found + * + * @tparam T The data type of the parameter to be read + * + * @return The value corresponding to the given key or the default value + */ + template + T get_or_default(char const* key, T default_value) const{ + if(has_key(key)){ + return get(key); + } + BOOST_LOG_TRIVIAL(warning) << "Header did not have key: " << key << ". Returning default value: " << default_value; + return default_value; + } + /** * @brief Set a value in the header * @@ -72,6 +92,7 @@ class Header private: void fetch_header_string(char const* key) const; + bool has_key(char const* key) const; private: psrdada_cpp::RawBytes& _header; diff --git a/cpp/skyweaver/src/Header.cpp b/cpp/skyweaver/src/Header.cpp index 15aadf8..8709439 100644 --- a/cpp/skyweaver/src/Header.cpp +++ b/cpp/skyweaver/src/Header.cpp @@ -24,7 +24,7 @@ void Header::purge() void Header::fetch_header_string(char const* key) const { - if(ascii_header_get(_header.ptr(), key, "%s", _buffer) == -1) { + if(!has_key(key)) { throw std::runtime_error(std::string("Could not find ") + key + " key in DADA header."); } @@ -34,6 +34,13 @@ void Header::fetch_header_string(char const* key) const } } +bool Header::has_key(char const* key) const +{ + return ascii_header_get(_header.ptr(), key, "%s", _buffer) != -1; +} + + + template <> long double Header::get(char const* key) const { From e14fc6425d2490c99e4e85a4a7a90ef7f7c1d053 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 11 Sep 2024 17:27:14 +0200 Subject: [PATCH 08/65] get-> get_or_default for few keys --- cpp/skyweaver/src/ObservationHeader.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index 1694deb..8014f58 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -11,7 +11,7 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, Header parser(raw_header); header.order = parser.get("ORDER"); - if(header.order.find("A") != std::string::npos) { + if(header.order.find("A") != std::string::npos) { header.nantennas = parser.get("NANT"); header.sample_clock = parser.get("SAMPLE_CLOCK"); @@ -21,20 +21,15 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.sync_time = parser.get("SYNC_TIME"); } - else { + + header.refdm = parser.get_or_default("COHERENT_DM", 0.0); - header.refdm = parser.get("COHERENT_DM"); - } - header.obs_frequency = - parser.get("OBS_FREQUENCY"); - header.obs_nchans = parser.get("OBS_NCHAN"); header.npol = parser.get("NPOL"); header.nbits = parser.get("NBIT"); header.nchans = parser.get("NCHAN"); header.bandwidth = parser.get("BW"); - header.obs_bandwidth = parser.get("OBS_BW"); header.frequency = parser.get("FREQ"); header.tsamp = parser.get("TSAMP"); @@ -48,6 +43,12 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.instrument = parser.get("INSTRUMENT"); header.chan0_idx = parser.get("CHAN0_IDX"); header.obs_offset = parser.get("OBS_OFFSET"); + + header.obs_bandwidth = parser.get_or_default("OBS_BW", header.bandwidth); + header.obs_nchans = parser.get_or_default("OBS_NCHAN", header.nchans); + header.obs_frequency = parser.get_or_default("OBS_FREQUENCY", header.frequency); + + } void validate_header(ObservationHeader const& header, PipelineConfig const& config) From 4f72d083c58b95c50c54619bc54db4dba4c33301 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 11 Sep 2024 18:43:44 +0200 Subject: [PATCH 09/65] add basefilename to multifilewriterconfig and change it as needed in callbacks --- cpp/skyweaver/MultiFileWriter.cuh | 9 ++-- cpp/skyweaver/detail/MultiFileWriter.cu | 23 +++++++--- .../detail/file_writer_callbacks.cpp | 42 ++++++++----------- 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index ceeb2d8..c0a220f 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -24,11 +24,13 @@ struct MultiFileWriterConfig{ std::string output_dir; std::string prefix; std::string extension; + std::string output_basename; + - MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("test"), prefix("test"), extension("default"){}; - MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, std::string extension) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension){}; - MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), stokes_mode(other.stokes_mode), output_dir(other.output_dir), prefix(other.prefix), extension(other.extension){}; + MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("default/"), prefix(""), extension(""){}; + MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, std::string extension) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), output_basename(""){ }; + MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), stokes_mode(other.stokes_mode), output_dir(other.output_dir), prefix(other.prefix), extension(other.extension), output_basename(other.output_basename){}; MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other){ header_size = other.header_size; max_file_size = other.max_file_size; @@ -36,6 +38,7 @@ struct MultiFileWriterConfig{ output_dir = other.output_dir; prefix = other.prefix; extension = other.extension; + output_basename = other.output_basename; return *this; } diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 0fdead7..790f52f 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -49,6 +49,8 @@ MultiFileWriter::MultiFileWriter(PipelineConfig const& config, writer_config.header_size = config.dada_header_size(); writer_config.max_file_size = config.max_output_filesize(); writer_config.stokes_mode = config.stokes_mode(); + writer_config.output_dir = config.output_dir(); + _config = writer_config; } @@ -84,15 +86,21 @@ MultiFileWriter::create_stream(VectorType const& stream_data, std::size_t stream_idx) { - // config.output_dir = get_output_dir(stream_data, stream_idx); - // config.prefix = get_basefilename(stream_data, stream_idx); - // config.extension = get_extension(stream_data); BOOST_LOG_TRIVIAL(info) << "Creating stream " << stream_idx << " in " << _config.output_dir; BOOST_LOG_TRIVIAL(info) << "Prefix: " << _config.prefix; BOOST_LOG_TRIVIAL(info) << "Extension: " << _config.extension; BOOST_LOG_TRIVIAL(info) << "Output directory: " << _config.output_dir; + if(_config.output_dir.empty()) { + _config.output_dir = get_output_dir(stream_data, stream_idx); + } + + if(_config.extension.empty()) { + _config.extension = get_extension(stream_data); + } + + _config.output_basename = get_basefilename(stream_data, stream_idx); _file_streams[stream_idx] = _create_stream_callback(_config, _header, stream_data, stream_idx); return *_file_streams[stream_idx]; @@ -128,8 +136,7 @@ MultiFileWriter::get_basefilename(VectorType const& stream_data, base_filename << get_formatted_time(_header.utc_start) << "_" << stream_idx << "_" << std::fixed << std::setprecision(3) << std::setfill('0') << std::setw(9) - << stream_data.reference_dm() << "_" << std::setprecision(0) - << std::setfill('0') << std::setw(9) << _header.frequency; + << stream_data.reference_dm(); if(!_tag.empty()) { base_filename << "_" << _tag; } @@ -142,6 +149,12 @@ MultiFileWriter::get_extension(VectorType const& stream_data) { std::string dims = stream_data.dims_as_string(); for(auto& c: dims) { c = std::tolower(static_cast(c)); } + if(dims =="t") { + return ".dat"; + } + else if(dims == "tf") { + return ".fil"; + } return "." + dims; } diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index da380ff..96387f7 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -14,6 +14,7 @@ #include #include #include +#include @@ -86,9 +87,19 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig BOOST_LOG_TRIVIAL(debug) << "Maximum allowed file size = " << filesize << " bytes (+header)"; + std::stringstream output_dir; + output_dir << config.output_dir << "/" + << std::fixed << std::setfill('0') << std::setw(9) + << static_cast(header.frequency); + + std::stringstream output_basename; + output_basename << config.output_basename << "_" + << std::fixed << std::setfill('0') << std::setw(9) + << static_cast(header.frequency); + std::unique_ptr file_stream = std::make_unique( - config.output_dir, - config.prefix, + output_dir.str(), + output_basename.str(), config.extension, filesize, [&, header, stream_data, stream_idx, filesize]( @@ -238,6 +249,7 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon BOOST_LOG_TRIVIAL(info) << "outdir: " << config.output_dir; BOOST_LOG_TRIVIAL(info) << "prefix: " << config.prefix; BOOST_LOG_TRIVIAL(info) << "extension: " << config.extension; + BOOST_LOG_TRIVIAL(info) << "output_basename: " << config.output_basename; // // Here we should update the tstart of the default header to be the // // start of the stream // // reset the total bytes counter to keep the time tracked correctly @@ -252,8 +264,7 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon // // causing compiler bugs prior to g++ 5.x // char formatted_time[80]; // strftime(formatted_time, 80, "%Y-%m-%d-%H:%M:%S", ptm); - // base_filename << formatted_time; - + // base_filename << formatted_time; std::unique_ptr file_stream = std::make_unique( @@ -269,27 +280,8 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon // created below std::ostringstream header_stream; // get ostream from temp_header - BOOST_LOG_TRIVIAL(info) << "Creating Sigproc header"; - BOOST_LOG_TRIVIAL(info) << "rawfile: " << header.rawfile; - BOOST_LOG_TRIVIAL(info) << "source: " << header.source_name; - BOOST_LOG_TRIVIAL(info) << "ra: " << header.ra; - BOOST_LOG_TRIVIAL(info) << "dec: " << header.dec; - BOOST_LOG_TRIVIAL(info) << "fch1: " << header.fch1; - BOOST_LOG_TRIVIAL(info) << "foff: " << header.foff; - BOOST_LOG_TRIVIAL(info) << "rdm: " << header.refdm; - BOOST_LOG_TRIVIAL(info) << "tsamp: " << header.tsamp; - BOOST_LOG_TRIVIAL(info) << "tstart: " << header.mjd_start; - BOOST_LOG_TRIVIAL(info) << "az: " << header.az; - BOOST_LOG_TRIVIAL(info) << "za: " << header.za; - BOOST_LOG_TRIVIAL(info) << "datatype: " << header.datatype; - BOOST_LOG_TRIVIAL(info) << "barycentric: " << header.barycentric; - BOOST_LOG_TRIVIAL(info) << "ibeam: " << header.ibeam; - BOOST_LOG_TRIVIAL(info) << "machineid: " << header.machineid; - BOOST_LOG_TRIVIAL(info) << "nbeams: " << header.nbeams; - BOOST_LOG_TRIVIAL(info) << "nbits: " << header.nbits; - BOOST_LOG_TRIVIAL(info) << "nchans: " << header.nchans; - BOOST_LOG_TRIVIAL(info) << "nifs: " << header.nifs; - BOOST_LOG_TRIVIAL(info) << "telescopeid: " << header.telescopeid; + + SigprocHeader sigproc_header(header); double mjd_offset = (((bytes_written / (header.nbits / 8.0)) / (header.nchans)) * From 36b1271d80a2a516f509a1b6288c3ec0d59d38a0 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 11 Sep 2024 18:43:59 +0200 Subject: [PATCH 10/65] reuse bytes_seeking --- cpp/skyweaver/src/SkyCleaver.cu | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index 6ec7574..92b236f 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -203,13 +203,11 @@ void SkyCleaver::init_readers() << "Seeking " << nsamples * _config.ndms() * _config.nbeams() << " bytes in bridge reader for frequency: " << freq; - std::size_t bytes_seeking = nsamples * _config.ndms() * + std::size_t bytes_seeking = (nsamples * _config.ndms() * _config.nbeams() * - sizeof(InputVectorType::value_type); + sizeof(InputVectorType::value_type)); - _bridge_readers[freq]->seekg(nsamples * _config.ndms() * - _config.nbeams() * - sizeof(InputVectorType::value_type), + _bridge_readers[freq]->seekg(bytes_seeking, std::ios_base::beg); std::size_t data_size = From b36cb9ff0df5f00dfebaaaa1b064b09d65255903 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 11 Sep 2024 18:51:28 +0200 Subject: [PATCH 11/65] reflect file writer callback changes in tester --- .../test/src/BeamformerPipelineTester.cu | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/cpp/skyweaver/test/src/BeamformerPipelineTester.cu b/cpp/skyweaver/test/src/BeamformerPipelineTester.cu index 44720b9..b7d3a64 100644 --- a/cpp/skyweaver/test/src/BeamformerPipelineTester.cu +++ b/cpp/skyweaver/test/src/BeamformerPipelineTester.cu @@ -70,7 +70,7 @@ class NullHandler { public: template - void init(Args... args) {}; + void init(Args... args){}; template bool operator()(Args... args) @@ -80,7 +80,8 @@ class NullHandler }; template -BeamformerPipelineTester::BeamformerPipelineTester(): ::testing::Test() +BeamformerPipelineTester::BeamformerPipelineTester() + : ::testing::Test() { } @@ -127,7 +128,10 @@ TYPED_TEST(BeamformerPipelineTester, instantiate) BeamformerPipeline(this->_config, cb_handler, ib_handler, stats_handler); + BfTraits>(this->_config, + cb_handler, + ib_handler, + stats_handler); } TYPED_TEST(BeamformerPipelineTester, full_pipeline_test) @@ -146,9 +150,12 @@ TYPED_TEST(BeamformerPipelineTester, full_pipeline_test) read_dada_header(raw_header, header); validate_header(header, this->_config); update_config(this->_config, header); - using WriterType = MultiFileWriter>; - typename WriterType::CreateStreamCallBackType create_stream_callback = detail::create_dada_file_stream>; - WriterType cb_handler(_config, "cb", create_stream_callback); + using WriterType = + MultiFileWriter>; + typename WriterType::CreateStreamCallBackType create_stream_callback = + detail::create_dada_file_stream< + TDBPowersH>; + WriterType cb_handler(this->_config, "cb", create_stream_callback); NullHandler ib_handler; NullHandler stats_handler; using IDPipelineType = @@ -156,9 +163,9 @@ TYPED_TEST(BeamformerPipelineTester, full_pipeline_test) typename BfTraits::QuantisedPowerType, decltype(cb_handler)>; using BPipelineType = BeamformerPipeline; + decltype(ib_handler), + decltype(stats_handler), + BfTraits>; using InputVectorTypeH = typename BPipelineType::VoltageVectorTypeH; IDPipelineType dedispersion_pipeline(this->_config, cb_handler); @@ -168,11 +175,12 @@ TYPED_TEST(BeamformerPipelineTester, full_pipeline_test) stats_handler); InputVectorTypeH input({ - this->_config.gulp_length_samps() / this->_config.nsamples_per_heap(), // T - header.nantennas, // A - this->_config.nchans(), // F - this->_config.nsamples_per_heap(), // T - this->_config.npol() // P + this->_config.gulp_length_samps() / + this->_config.nsamples_per_heap(), // T + header.nantennas, // A + this->_config.nchans(), // F + this->_config.nsamples_per_heap(), // T + this->_config.npol() // P }); input.frequencies(this->_config.channel_frequencies()); input.dms({0.0f}); From 4826368ac2a4982aa722376567dd206cd0b83ecb Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 19 Sep 2024 18:05:25 +0200 Subject: [PATCH 12/65] fix paths and filenames --- cpp/skyweaver/MultiFileWriter.cuh | 5 +- .../detail/IncoherentDedispersionPipeline.cu | 6 -- cpp/skyweaver/detail/MultiFileWriter.cu | 13 ++-- .../detail/file_writer_callbacks.cpp | 61 ++++++++++--------- cpp/skyweaver/src/MultiFileReader.cu | 2 +- 5 files changed, 41 insertions(+), 46 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index c0a220f..305f356 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -22,6 +22,7 @@ struct MultiFileWriterConfig{ std::size_t max_file_size; std::string stokes_mode; std::string output_dir; + std::string base_output_dir; std::string prefix; std::string extension; std::string output_basename; @@ -30,7 +31,8 @@ struct MultiFileWriterConfig{ MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("default/"), prefix(""), extension(""){}; MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, std::string extension) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), output_basename(""){ }; - MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), stokes_mode(other.stokes_mode), output_dir(other.output_dir), prefix(other.prefix), extension(other.extension), output_basename(other.output_basename){}; + MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), + stokes_mode(other.stokes_mode), output_dir(other.output_dir), base_output_dir(other.base_output_dir), prefix(other.prefix), extension(other.extension), output_basename(other.output_basename){}; MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other){ header_size = other.header_size; max_file_size = other.max_file_size; @@ -39,6 +41,7 @@ struct MultiFileWriterConfig{ prefix = other.prefix; extension = other.extension; output_basename = other.output_basename; + base_output_dir = other.base_output_dir; return *this; } diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index 5102daf..62f722b 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -111,12 +111,6 @@ void IncoherentDedispersionPipeline::operator()( _output_buffers[ref_dm_idx].utc_offset( data.utc_offset() + _dedispersers[ref_dm_idx]->max_sample_delay() * data.tsamp()); - BOOST_LOG_TRIVIAL(warning) << "Old UTC offset was " << data.utc_offset(); - BOOST_LOG_TRIVIAL(warning) << "Incoherent Max delay is " - << _dedispersers[ref_dm_idx]->max_sample_delay(); - BOOST_LOG_TRIVIAL(warning) << "tsamp is " << data.tsamp(); - BOOST_LOG_TRIVIAL(warning) << "Setting UTC offset to " - << _output_buffers[ref_dm_idx].utc_offset(); _agg_buffers[ref_dm_idx]->push_back(data.vector()); } diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 790f52f..3f72fba 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -49,7 +49,7 @@ MultiFileWriter::MultiFileWriter(PipelineConfig const& config, writer_config.header_size = config.dada_header_size(); writer_config.max_file_size = config.max_output_filesize(); writer_config.stokes_mode = config.stokes_mode(); - writer_config.output_dir = config.output_dir(); + writer_config.base_output_dir = config.output_dir(); _config = writer_config; } @@ -92,9 +92,7 @@ MultiFileWriter::create_stream(VectorType const& stream_data, BOOST_LOG_TRIVIAL(info) << "Extension: " << _config.extension; BOOST_LOG_TRIVIAL(info) << "Output directory: " << _config.output_dir; - if(_config.output_dir.empty()) { - _config.output_dir = get_output_dir(stream_data, stream_idx); - } + _config.output_dir = get_output_dir(stream_data, stream_idx); if(_config.extension.empty()) { _config.extension = get_extension(stream_data); @@ -114,11 +112,10 @@ MultiFileWriter::get_output_dir(VectorType const& stream_data, // Output directory format // // std::stringstream output_dir; - output_dir << _config.output_dir << "/" + output_dir << _config.base_output_dir << "/" << get_formatted_time(_header.utc_start) << "/" - << stream_idx << "/" - << std::fixed << std::setfill('0') << std::setw(9) - << static_cast(_header.frequency); + << stream_idx; + return output_dir.str(); } diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 96387f7..0a65481 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -33,36 +33,36 @@ namespace { * bridge */ std::string default_dada_header = R"( - HEADER DADA - HDR_VERSION 1.0 - HDR_SIZE 4096 - DADA_VERSION 1.0 - - FILE_SIZE 100000000000 - FILE_NUMBER 0 - - UTC_START 1708082229.000020336 - MJD_START 60356.47024305579093 - - SOURCE J1644-4559 - RA 16:44:49.27 - DEC -45:59:09.7 - TELESCOPE MeerKAT - INSTRUMENT CBF-Feng - RECEIVER L-band - FREQ 1284000000.000000 - BW 856000000.000000 - TSAMP 4.7850467290 - STOKES I - - NBIT 8 - NDIM 1 - NPOL 1 - NCHAN 64 - NBEAM 800 - ORDER TFB - - CHAN0_IDX 2688 +HEADER DADA +HDR_VERSION 1.0 +HDR_SIZE 4096 +DADA_VERSION 1.0 + +FILE_SIZE 100000000000 +FILE_NUMBER 0 + +UTC_START 1708082229.000020336 +MJD_START 60356.47024305579093 + +SOURCE J1644-4559 +RA 16:44:49.27 +DEC -45:59:09.7 +TELESCOPE MeerKAT +INSTRUMENT CBF-Feng +RECEIVER L-band +FREQ 1284000000.000000 +BW 856000000.000000 +TSAMP 4.7850467290 +STOKES I + +NBIT 8 +NDIM 1 +NPOL 1 +NCHAN 64 +NBEAM 800 +ORDER TFB + +CHAN0_IDX 2688 )"; } namespace skyweaver @@ -166,6 +166,7 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig std::shared_ptr header_ptr( temp_header, std::default_delete()); + return header_ptr; }); return file_stream; diff --git a/cpp/skyweaver/src/MultiFileReader.cu b/cpp/skyweaver/src/MultiFileReader.cu index 3bd3dae..821b78a 100644 --- a/cpp/skyweaver/src/MultiFileReader.cu +++ b/cpp/skyweaver/src/MultiFileReader.cu @@ -28,7 +28,7 @@ std::size_t MultiFileReader::safe_read(std::ifstream& input_stream, << std::strerror(errno); throw std::runtime_error(error_msg.str().c_str()); } - BOOST_LOG_TRIVIAL(debug) << "Read complete"; + BOOST_LOG_TRIVIAL(debug) << "Safe Read complete"; return nbytes_read; } From 4aec59b65fecc2f4b757ad1b0c291718a07c15c5 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 19 Sep 2024 18:10:55 +0200 Subject: [PATCH 13/65] check for both OBS_FREQ & OBS_FREQUENCY --- cpp/skyweaver/src/ObservationHeader.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index 8014f58..9d0e1d1 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -46,7 +46,9 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.obs_bandwidth = parser.get_or_default("OBS_BW", header.bandwidth); header.obs_nchans = parser.get_or_default("OBS_NCHAN", header.nchans); - header.obs_frequency = parser.get_or_default("OBS_FREQUENCY", header.frequency); + header.obs_frequency = parser.get_or_default("OBS_FREQUENCY", + parser.get_or_default("OBS_FREQ", + header.frequency)); } From 2462dad77a44dfd49dc4d8a401e438e2a77eac9b Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Fri, 20 Sep 2024 16:04:27 +0200 Subject: [PATCH 14/65] proper naming, more header parameters to read from dada files --- cpp/skyweaver/MultiFileWriter.cuh | 4 +++- cpp/skyweaver/ObservationHeader.hpp | 3 +++ cpp/skyweaver/detail/MultiFileWriter.cu | 9 ++++---- .../detail/file_writer_callbacks.cpp | 8 ++++--- cpp/skyweaver/src/FileOutputStream.cpp | 14 +++++++++++-- cpp/skyweaver/src/ObservationHeader.cpp | 19 +++++++++++++++++ cpp/skyweaver/src/SkyCleaver.cu | 21 +++++++++++-------- 7 files changed, 59 insertions(+), 19 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 305f356..15825c6 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -46,7 +46,9 @@ struct MultiFileWriterConfig{ } std::string to_string(){ - return "header_size: " + std::to_string(header_size) + ", max_file_size: " + std::to_string(max_file_size) + ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + ", prefix: " + prefix + ", extension: " + extension; + return "header_size: " + std::to_string(header_size) + ", max_file_size: " + std::to_string(max_file_size) + + ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + ", prefix: " + prefix + ", extension: " + extension + + ", output_basename: " + output_basename + ", base_output_dir: " + base_output_dir; } }; /** diff --git a/cpp/skyweaver/ObservationHeader.hpp b/cpp/skyweaver/ObservationHeader.hpp index 753ac2a..356d71e 100644 --- a/cpp/skyweaver/ObservationHeader.hpp +++ b/cpp/skyweaver/ObservationHeader.hpp @@ -40,6 +40,9 @@ struct ObservationHeader { std::string telescope; // Telescope name std::string instrument; // Name of the recording instrument std::string order; // Order of the dimensions in the data + std::string ndms; // Number of DMs + std::vector dms; // DMs + std::string to_string() const; // Convert the header to a string long double az; // Azimuth diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 3f72fba..7529621 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -87,10 +87,7 @@ MultiFileWriter::create_stream(VectorType const& stream_data, { - BOOST_LOG_TRIVIAL(info) << "Creating stream " << stream_idx << " in " << _config.output_dir; - BOOST_LOG_TRIVIAL(info) << "Prefix: " << _config.prefix; - BOOST_LOG_TRIVIAL(info) << "Extension: " << _config.extension; - BOOST_LOG_TRIVIAL(info) << "Output directory: " << _config.output_dir; + _config.output_dir = get_output_dir(stream_data, stream_idx); @@ -99,8 +96,12 @@ MultiFileWriter::create_stream(VectorType const& stream_data, } _config.output_basename = get_basefilename(stream_data, stream_idx); + + _file_streams[stream_idx] = _create_stream_callback(_config, _header, stream_data, stream_idx); + + return *_file_streams[stream_idx]; } diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 0a65481..46e112c 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -15,7 +15,7 @@ #include #include #include - +#include namespace { @@ -265,12 +265,14 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon // // causing compiler bugs prior to g++ 5.x // char formatted_time[80]; // strftime(formatted_time, 80, "%Y-%m-%d-%H:%M:%S", ptm); - // base_filename << formatted_time; + // base_filename << formatted_time; + + //make config.output_dir if it does not exist std::unique_ptr file_stream = std::make_unique( config.output_dir, - config.prefix, + config.output_basename, config.extension, filesize, [header](std::size_t& header_size, diff --git a/cpp/skyweaver/src/FileOutputStream.cpp b/cpp/skyweaver/src/FileOutputStream.cpp index 1044f1c..f4d2ffa 100644 --- a/cpp/skyweaver/src/FileOutputStream.cpp +++ b/cpp/skyweaver/src/FileOutputStream.cpp @@ -13,13 +13,23 @@ namespace fs = std::filesystem; void create_directories(const fs::path& path) { + // Check if the directory already exists if(!fs::exists(path)) { // Directory does not exist, attempt to create it - if(!fs::create_directories(path)) { + // doing it via try catch for multi-threaded use + try{ + fs::create_directories(path); + } catch (const std::filesystem::filesystem_error& e) { + if(!fs::exists(path)) { + throw std::runtime_error("Failed to create directory: " + + path.string()); + } + } catch (const std::exception& e) { throw std::runtime_error("Failed to create directory: " + - path.string()); + path.string()); } + } else if(!fs::is_directory(path)) { // Path exists but is not a directory throw std::runtime_error("Path exists but is not a directory: " + diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index 9d0e1d1..2b49307 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -5,6 +5,20 @@ #include namespace skyweaver { + +std::vector parse_float_list(std::string const& str) +{ + std::vector values; + std::size_t start = 0; + std::size_t end = 0; + while(end != std::string::npos) { + end = str.find(',', start); + values.push_back(std::stof(str.substr(start, end - start))); + start = end + 1; + } + return values; +} + void read_dada_header(psrdada_cpp::RawBytes& raw_header, ObservationHeader& header) { @@ -50,6 +64,11 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, parser.get_or_default("OBS_FREQ", header.frequency)); + header.ndms = parser.get_or_default("NDMS", "0"); + if(header.ndms != "0") { + header.dms = parse_float_list(parser.get("DMS")); + } + } void validate_header(ObservationHeader const& header, diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index 92b236f..d3107f2 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -259,25 +259,28 @@ void SkyCleaver::init_writers() if(!fs::exists(_config.output_dir())) { fs::create_directories(_config.output_dir()); } + std::string out_prefix = _config.out_prefix(); + std::string output_dir = _config.output_dir(); + if(!out_prefix.empty()) { + out_prefix = out_prefix + "_"; + } for(int idm = 0; idm < _config.ndms(); idm++) { + if(_config.ndms() > 1) { + out_prefix = out_prefix + "idm_" + + to_string_with_padding(_header.dms[idm], 9, 3) + "_"; + } for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - std::string out_prefix = _config.out_prefix(); - std::string output_dir = _config.output_dir(); - std::string prefix = out_prefix + "_dm" + - to_string_with_padding(idm, 3) + "_cfbf" + + + std::string prefix = out_prefix + "cb_" + to_string_with_padding(ibeam, 5); - std::string beam_filename = prefix + ".fil"; - _beam_filenames.push_back(beam_filename); - std::string beam_filepath = output_dir + "/" + beam_filename; - BOOST_LOG_TRIVIAL(info) << "Beam file path: " << beam_filepath; MultiFileWriterConfig writer_config; writer_config.header_size = _config.dada_header_size(); writer_config.max_file_size = _config.max_output_filesize(); writer_config.stokes_mode = _config.stokes_mode(); - writer_config.output_dir = output_dir; + writer_config.base_output_dir = output_dir; writer_config.prefix = prefix; writer_config.extension = ".fil"; From c55bf3c7117b26592f828f7cb66938df5d833e43 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Fri, 20 Sep 2024 18:31:12 +0200 Subject: [PATCH 15/65] get mjd_start from utc_offset --- .../detail/file_writer_callbacks.cpp | 51 ++++++------------- cpp/skyweaver/src/ObservationHeader.cpp | 8 +++ cpp/skyweaver/src/SkyCleaver.cu | 5 +- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 46e112c..e560aac 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -64,6 +64,8 @@ ORDER TFB CHAN0_IDX 2688 )"; + +#define MJD_UNIX_EPOCH 40587.0 } namespace skyweaver { @@ -160,9 +162,11 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig header_writer.set("FILE_NUMBER", file_idx); header_writer.set("OBS_OFFSET", bytes_written); header_writer.set("OBS_OVERLAP", 0); - header_writer.set("UTC_START", - header.utc_start + - stream_data.utc_offset()); + + double tstart = header.utc_start + stream_data.utc_offset(); + + header_writer.set("UTC_START", tstart); + header_writer.set("MJD_START", MJD_UNIX_EPOCH + tstart / 86400.0); std::shared_ptr header_ptr( temp_header, std::default_delete()); @@ -186,7 +190,7 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon ObservationHeader header = obs_header; - BOOST_LOG_TRIVIAL(debug) << "Header: " << header.to_string(); + BOOST_LOG_TRIVIAL(info) << "Header: " << header.to_string(); // Here we round the file size to a multiple of the stream prototype std::size_t filesize = @@ -199,16 +203,18 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon double foff = -1* static_cast(header.obs_bandwidth / header.nchans)/1e6;// MHz - double fch1 = static_cast(header.obs_frequency + header.obs_bandwidth / 2.0)/1e6 + 0.5 * foff; // MHz - double tstart = static_cast(header.mjd_start); + // 1* foff instead of 0.5* foff below because the dedispersion causes all the frequencies to change by half the bandwidth to refer to the bottom of the channel + double fch1 = static_cast(header.obs_frequency + header.obs_bandwidth / 2.0)/1e6 + foff; // MHz + + double utc_start = static_cast(header.utc_start); + header.mjd_start = (utc_start / 86400.0) + MJD_UNIX_EPOCH; + uint32_t datatype = 0; uint32_t barycentric = 0; // uint32_t ibeam = 0; double az = 0.0; double za = 0.0; - uint32_t machineid = 0; uint32_t nifs = header.npol; - uint32_t telescopeid = 64; header.sigproc_params = true; header.rawfile = std::string("unset"); @@ -220,37 +226,12 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon header.datatype = datatype; header.barycentric = barycentric; header.nifs = nifs; - header.telescopeid = telescopeid; + header.telescopeid = 64; BOOST_LOG_TRIVIAL(info) << "Creating Sigproc file stream"; - BOOST_LOG_TRIVIAL(info) << "fch1: " << header.obs_frequency + header.obs_bandwidth / 2.0; - - BOOST_LOG_TRIVIAL(info) << "rawfile: " << header.rawfile; - BOOST_LOG_TRIVIAL(info) << "source: " << header.source_name; - BOOST_LOG_TRIVIAL(info) << "ra: " << header.ra; - BOOST_LOG_TRIVIAL(info) << "dec: " << header.dec; - BOOST_LOG_TRIVIAL(info) << "fch1: " << fch1; - BOOST_LOG_TRIVIAL(info) << "foff: " << foff; - BOOST_LOG_TRIVIAL(info) << "rdm: " << header.refdm; - BOOST_LOG_TRIVIAL(info) << "tsamp: " << header.tsamp; - BOOST_LOG_TRIVIAL(info) << "tstart: " << tstart; - BOOST_LOG_TRIVIAL(info) << "az: " << az; - BOOST_LOG_TRIVIAL(info) << "za: " << za; - BOOST_LOG_TRIVIAL(info) << "datatype: " << header.datatype; - BOOST_LOG_TRIVIAL(info) << "barycentric: " << header.barycentric; - BOOST_LOG_TRIVIAL(info) << "ibeam: " << header.ibeam; - BOOST_LOG_TRIVIAL(info) << "machineid: " << machineid; - BOOST_LOG_TRIVIAL(info) << "nbeams: " << header.nbeams; - BOOST_LOG_TRIVIAL(info) << "nbits: " << header.nbits; - BOOST_LOG_TRIVIAL(info) << "nchans: " << header.nchans; - BOOST_LOG_TRIVIAL(info) << "nifs: " << nifs; - BOOST_LOG_TRIVIAL(info) << "telescopeid: " << telescopeid; - BOOST_LOG_TRIVIAL(info) << "outdir: " << config.output_dir; - BOOST_LOG_TRIVIAL(info) << "prefix: " << config.prefix; - BOOST_LOG_TRIVIAL(info) << "extension: " << config.extension; - BOOST_LOG_TRIVIAL(info) << "output_basename: " << config.output_basename; + // // Here we should update the tstart of the default header to be the // // start of the stream // // reset the total bytes counter to keep the time tracked correctly diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index 2b49307..f554a59 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -150,6 +150,14 @@ std::string ObservationHeader::to_string() const << " instrument: " << instrument << "\n" << " chan0_idx: " << chan0_idx << "\n" << " obs_offset: " << obs_offset << "\n"; + if(ndms != "0") { + oss << " ndms: " << ndms << "\n"; + oss << " dms: "; + for(auto dm : dms) { + oss << dm << " "; + } + oss << "\n"; + } if(sigproc_params) { diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index d3107f2..a0728ff 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -150,9 +150,6 @@ void SkyCleaver::init_readers() << "Read header from first file: " << header.to_string(); float obs_centre_freq = header.obs_frequency; - if(obs_centre_freq == 0.0) { - obs_centre_freq = 1284000000; - } float obs_bandwidth = header.obs_bandwidth; for(int i = 0; i < nbridges; i++) { int ifreq = std::lround(obs_centre_freq - obs_bandwidth / 2 + @@ -304,6 +301,8 @@ void SkyCleaver::init_writers() _config.nbridges()}, 0); + _beam_data[idm][ibeam]->reference_dm(_header.refdm); + _total_beam_writers++; } } From 819ee96bb190dd3ca05356239b7aec4a47b16857 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Mon, 23 Sep 2024 12:39:21 +0200 Subject: [PATCH 16/65] fix snake naming bug --- cpp/skyweaver/src/SkyCleaver.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index a0728ff..1b197f2 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -261,15 +261,15 @@ void SkyCleaver::init_writers() if(!out_prefix.empty()) { out_prefix = out_prefix + "_"; } - + std::string prefix = ""; for(int idm = 0; idm < _config.ndms(); idm++) { if(_config.ndms() > 1) { - out_prefix = out_prefix + "idm_" + + prefix = out_prefix + "idm_" + to_string_with_padding(_header.dms[idm], 9, 3) + "_"; } for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - std::string prefix = out_prefix + "cb_" + + std::string prefix = prefix + "cb_" + to_string_with_padding(ibeam, 5); MultiFileWriterConfig writer_config; From 63b85f8ce36e4ab4973ecc4a6a87e6027dedd06d Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Mon, 23 Sep 2024 16:50:29 +0200 Subject: [PATCH 17/65] actually fix snake naming bug --- cpp/skyweaver/src/SkyCleaver.cu | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index 1b197f2..9df09e3 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -256,29 +256,24 @@ void SkyCleaver::init_writers() if(!fs::exists(_config.output_dir())) { fs::create_directories(_config.output_dir()); } - std::string out_prefix = _config.out_prefix(); + std::string out_prefix = _config.out_prefix().empty() + ? "" + : _config.out_prefix() + "_"; std::string output_dir = _config.output_dir(); - if(!out_prefix.empty()) { - out_prefix = out_prefix + "_"; - } - std::string prefix = ""; + for(int idm = 0; idm < _config.ndms(); idm++) { - if(_config.ndms() > 1) { - prefix = out_prefix + "idm_" + - to_string_with_padding(_header.dms[idm], 9, 3) + "_"; - } + + std::string prefix = _config.ndms() > 1 ? out_prefix + "idm_" + + to_string_with_padding(idm, 9, 3) + "_": out_prefix; + for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - - std::string prefix = prefix + "cb_" + - to_string_with_padding(ibeam, 5); MultiFileWriterConfig writer_config; - writer_config.header_size = _config.dada_header_size(); writer_config.max_file_size = _config.max_output_filesize(); writer_config.stokes_mode = _config.stokes_mode(); writer_config.base_output_dir = output_dir; - writer_config.prefix = prefix; + writer_config.prefix = prefix + "cb_" + to_string_with_padding(ibeam, 5);; writer_config.extension = ".fil"; BOOST_LOG_TRIVIAL(info) From 510fc01a7c36a71d2e4a6961354d94e4e9f5d6eb Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 25 Sep 2024 12:18:00 +0200 Subject: [PATCH 18/65] use long double for times, incorporate downstream changes for half channel cfreq change --- .../detail/file_writer_callbacks.cpp | 2 +- cpp/skyweaver/src/ObservationHeader.cpp | 4 +-- cpp/skyweaver/src/SkyCleaver.cu | 28 ++++++++++++------- cpp/skyweaver/src/skyweaver_cli.cu | 2 +- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index e560aac..71f0b42 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -163,7 +163,7 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig header_writer.set("OBS_OFFSET", bytes_written); header_writer.set("OBS_OVERLAP", 0); - double tstart = header.utc_start + stream_data.utc_offset(); + long double tstart = header.utc_start + stream_data.utc_offset(); header_writer.set("UTC_START", tstart); header_writer.set("MJD_START", MJD_UNIX_EPOCH + tstart / 86400.0); diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index f554a59..13255aa 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -58,8 +58,8 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.chan0_idx = parser.get("CHAN0_IDX"); header.obs_offset = parser.get("OBS_OFFSET"); - header.obs_bandwidth = parser.get_or_default("OBS_BW", header.bandwidth); - header.obs_nchans = parser.get_or_default("OBS_NCHAN", header.nchans); + header.obs_bandwidth = parser.get_or_default("OBS_BW", 856e6); + header.obs_nchans = parser.get_or_default("OBS_NCHAN", 4096); header.obs_frequency = parser.get_or_default("OBS_FREQUENCY", parser.get_or_default("OBS_FREQ", header.frequency)); diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index 9df09e3..bf195cd 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -149,11 +149,17 @@ void SkyCleaver::init_readers() BOOST_LOG_TRIVIAL(info) << "Read header from first file: " << header.to_string(); - float obs_centre_freq = header.obs_frequency; - float obs_bandwidth = header.obs_bandwidth; + long double obs_centre_freq = header.obs_frequency; + long double obs_bandwidth = header.obs_bandwidth; + std::size_t obs_nchans = header.obs_nchans; + + long double start_freq = obs_centre_freq - obs_bandwidth / 2; + + + for(int i = 0; i < nbridges; i++) { - int ifreq = std::lround(obs_centre_freq - obs_bandwidth / 2 + - (i + 0.5) * obs_bandwidth / nbridges); + + int ifreq = std::lround( std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) << "Expected frequency [" << i << "]: " << ifreq; @@ -170,6 +176,8 @@ void SkyCleaver::init_readers() 0); } + // print _expected_freqs + std::size_t smallest_data_size = std::numeric_limits::max(); for(const auto& [freq, reader]: _bridge_readers) { @@ -190,13 +198,13 @@ void SkyCleaver::init_readers() 1e-6; // Header has it in microseconds, converting to seconds std::size_t nsamples = std::floor(time_diff / tsamp); - BOOST_LOG_TRIVIAL(info) + BOOST_LOG_TRIVIAL(debug) << "Frequency: " << freq << " Timestamp: " << timestamp << "tsamp: " << tsamp << " Latest timestamp: " << latest_timestamp << " Time difference: " << time_diff << " Number of samples to skip: " << nsamples; - BOOST_LOG_TRIVIAL(info) + BOOST_LOG_TRIVIAL(debug) << "Seeking " << nsamples * _config.ndms() * _config.nbeams() << " bytes in bridge reader for frequency: " << freq; @@ -242,8 +250,7 @@ void SkyCleaver::init_readers() << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; _header = _bridge_readers[_available_freqs[0]]->get_header(); - BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver"; - BOOST_LOG_TRIVIAL(info) << "Header: " << _header.to_string(); + BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver: " << _header.to_string(); _header.nchans = _header.nchans * _config.nbridges(); _header.nbeams = _config.nbeams(); } @@ -260,11 +267,12 @@ void SkyCleaver::init_writers() ? "" : _config.out_prefix() + "_"; std::string output_dir = _config.output_dir(); + for(int idm = 0; idm < _config.ndms(); idm++) { std::string prefix = _config.ndms() > 1 ? out_prefix + "idm_" + - to_string_with_padding(idm, 9, 3) + "_": out_prefix; + to_string_with_padding(_header.dms[idm], 9, 3) + "_": out_prefix; for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { @@ -359,7 +367,7 @@ void SkyCleaver::cleave() reader->read(reinterpret_cast(thrust::raw_pointer_cast( _bridge_data[freq]->data())), gulp_size); // read a big chunk of data - BOOST_LOG_TRIVIAL(info) + BOOST_LOG_TRIVIAL(debug) << "Read " << read_size << " bytes from bridge" << freq; if(read_size < gulp_size * sizeof(InputVectorType::value_type)) { BOOST_LOG_TRIVIAL(warning) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index e168f63..3092164 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -268,7 +268,7 @@ void setup_pipeline(skyweaver::PipelineConfig& config) create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; - StatsWriterType stats_handler(config, "stats", create_stream_callback_stats); + StatsWriterType stats_handler(config, "stats", create_stream_callback_stats); if constexpr(enable_incoherent_dedispersion) { From 4ae82abc1aa6ee824cfee5877f3f3a3c3b99b4ea Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 26 Sep 2024 11:37:29 +0200 Subject: [PATCH 19/65] fixed formatting on MultiFileWriter.cuh --- cpp/skyweaver/MultiFileWriter.cuh | 112 +++++++++++++++++------------- 1 file changed, 64 insertions(+), 48 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 15825c6..c0e8c9d 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -13,43 +13,55 @@ namespace skyweaver { - - - -struct MultiFileWriterConfig{ - - std::size_t header_size; - std::size_t max_file_size; - std::string stokes_mode; - std::string output_dir; - std::string base_output_dir; - std::string prefix; - std::string extension; - std::string output_basename; - - - - MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("default/"), prefix(""), extension(""){}; - MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, std::string extension) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), output_basename(""){ }; - MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), - stokes_mode(other.stokes_mode), output_dir(other.output_dir), base_output_dir(other.base_output_dir), prefix(other.prefix), extension(other.extension), output_basename(other.output_basename){}; - MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other){ - header_size = other.header_size; - max_file_size = other.max_file_size; - stokes_mode = other.stokes_mode; - output_dir = other.output_dir; - prefix = other.prefix; - extension = other.extension; - output_basename = other.output_basename; - base_output_dir = other.base_output_dir; - return *this; - } - - std::string to_string(){ - return "header_size: " + std::to_string(header_size) + ", max_file_size: " + std::to_string(max_file_size) - + ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + ", prefix: " + prefix + ", extension: " + extension - + ", output_basename: " + output_basename + ", base_output_dir: " + base_output_dir; - } +struct MultiFileWriterConfig { + std::size_t header_size; + std::size_t max_file_size; + std::string stokes_mode; + std::string output_dir; + std::string base_output_dir; + std::string prefix; + std::string extension; + std::string output_basename; + + MultiFileWriterConfig() + : header_size(4096), max_file_size(2147483647), stokes_mode("I"), + output_dir("default/"), prefix(""), extension("") {}; + MultiFileWriterConfig(std::size_t header_size, + std::size_t max_file_size, + std::string stokes_mode, + std::string output_dir, + std::string prefix, + std::string extension) + : header_size(header_size), max_file_size(max_file_size), + stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), + extension(extension), output_basename("") {}; + MultiFileWriterConfig(MultiFileWriterConfig const& other) + : header_size(other.header_size), max_file_size(other.max_file_size), + stokes_mode(other.stokes_mode), output_dir(other.output_dir), + base_output_dir(other.base_output_dir), prefix(other.prefix), + extension(other.extension), output_basename(other.output_basename) {}; + MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other) + { + header_size = other.header_size; + max_file_size = other.max_file_size; + stokes_mode = other.stokes_mode; + output_dir = other.output_dir; + prefix = other.prefix; + extension = other.extension; + output_basename = other.output_basename; + base_output_dir = other.base_output_dir; + return *this; + } + + std::string to_string() + { + return "header_size: " + std::to_string(header_size) + + ", max_file_size: " + std::to_string(max_file_size) + + ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + + ", prefix: " + prefix + ", extension: " + extension + + ", output_basename: " + output_basename + + ", base_output_dir: " + base_output_dir; + } }; /** * @brief A class for handling writing of DescribedVectors @@ -58,12 +70,13 @@ struct MultiFileWriterConfig{ template class MultiFileWriter { -public: - - using CreateStreamCallBackType = std::function(MultiFileWriterConfig const&, - ObservationHeader const&, - VectorType const&, - std::size_t)>; + public: + using CreateStreamCallBackType = + std::function( + MultiFileWriterConfig const&, + ObservationHeader const&, + VectorType const&, + std::size_t)>; public: /** @@ -74,8 +87,12 @@ public: * (used to avoid clashing file names). */ // MultiFileWriter(PipelineConfig const& config, std::string tag = ""); - MultiFileWriter(PipelineConfig const& config, std::string tag, CreateStreamCallBackType create_stream_callback); - MultiFileWriter(MultiFileWriterConfig config, std::string tag, CreateStreamCallBackType create_stream_callback); + MultiFileWriter(PipelineConfig const& config, + std::string tag, + CreateStreamCallBackType create_stream_callback); + MultiFileWriter(MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback); MultiFileWriter(MultiFileWriter const&) = delete; /** @@ -110,13 +127,12 @@ public: */ bool operator()(VectorType const& stream_data, std::size_t stream_idx = 0); - bool write(VectorType const& stream_data, - std::size_t stream_idx = 0); + bool write(VectorType const& stream_data, std::size_t stream_idx = 0); private: bool has_stream(std::size_t stream_idx); FileOutputStream& create_stream(VectorType const& stream_data, - std::size_t stream_idx); + std::size_t stream_idx); std::string get_output_dir(VectorType const& stream_data, std::size_t stream_idx); std::string get_basefilename(VectorType const& stream_data, From 2787d007eeef1f0a5a54586c75cd8c7f9d577a55 Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 26 Sep 2024 12:04:05 +0200 Subject: [PATCH 20/65] formatting --- cpp/skyweaver/MultiFileWriter.cuh | 3 + cpp/skyweaver/ObservationHeader.hpp | 70 ++-- cpp/skyweaver/SkyCleaver.cuh | 53 +-- cpp/skyweaver/SkyCleaverConfig.hpp | 95 ++--- cpp/skyweaver/detail/MultiFileWriter.cu | 56 ++- .../detail/file_writer_callbacks.cpp | 344 +++++++++--------- cpp/skyweaver/src/CoherentDedisperser.cu | 2 +- cpp/skyweaver/src/ObservationHeader.cpp | 44 ++- cpp/skyweaver/src/skycleaver_cli.cu | 315 ++++++++-------- cpp/skyweaver/src/skyweaver_cli.cu | 55 +-- 10 files changed, 512 insertions(+), 525 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index c0e8c9d..352b2f5 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -26,6 +26,7 @@ struct MultiFileWriterConfig { MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("default/"), prefix(""), extension("") {}; + MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, @@ -35,11 +36,13 @@ struct MultiFileWriterConfig { : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), output_basename("") {}; + MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), stokes_mode(other.stokes_mode), output_dir(other.output_dir), base_output_dir(other.base_output_dir), prefix(other.prefix), extension(other.extension), output_basename(other.output_basename) {}; + MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other) { header_size = other.header_size; diff --git a/cpp/skyweaver/ObservationHeader.hpp b/cpp/skyweaver/ObservationHeader.hpp index 356d71e..d09564d 100644 --- a/cpp/skyweaver/ObservationHeader.hpp +++ b/cpp/skyweaver/ObservationHeader.hpp @@ -4,12 +4,12 @@ #include "psrdada_cpp/raw_bytes.hpp" #include "skyweaver/Header.hpp" #include "skyweaver/PipelineConfig.hpp" + #include namespace skyweaver { - struct ObservationHeader { std::size_t nchans = 0; // Number of frequency channels in the subband std::size_t npol = 0; // Number of polarisations @@ -30,37 +30,37 @@ struct ObservationHeader { long double sync_time = 0.0; // The UNIX epoch of the sampler zero long double utc_start = 0.0; // The UTC start time of the data long double mjd_start = 0.0; // The MJD start time of the data - std::size_t obs_offset = 0; // The offset of the current file from UTC_START in bytesß - long double refdm = 0.0; // Reference DM - std::size_t ibeam = 0.0; // Beam number - std::size_t nbeams = 0; // Number of beams - std::string source_name; // Name of observation target - std::string ra; // Right ascension - std::string dec; // Declination - std::string telescope; // Telescope name - std::string instrument; // Name of the recording instrument - std::string order; // Order of the dimensions in the data - std::string ndms; // Number of DMs - std::vector dms; // DMs - - std::string to_string() const; // Convert the header to a string - - long double az; // Azimuth - long double za; // Zenith angle - std::size_t machineid = 0; // Machine ID - std::size_t nifs = 0; // Number of IFs - std::size_t telescopeid = 0; // Telescope ID - std::size_t datatype = 0; // Data type - std::size_t barycentric = 0; // Barycentric correction - std::string rawfile; // Raw file name - double fch1 = 0.0; // Centre frequency of the first channel - double foff = 0.0; // Frequency offset between channels - - bool sigproc_params = false; // Whether to include sigproc parameters + std::size_t obs_offset = + 0; // The offset of the current file from UTC_START in bytesß + long double refdm = 0.0; // Reference DM + std::size_t ibeam = 0.0; // Beam number + std::size_t nbeams = 0; // Number of beams + std::string source_name; // Name of observation target + std::string ra; // Right ascension + std::string dec; // Declination + std::string telescope; // Telescope name + std::string instrument; // Name of the recording instrument + std::string order; // Order of the dimensions in the data + std::string ndms; // Number of DMs + std::vector dms; // DMs + + std::string to_string() const; // Convert the header to a string + + long double az; // Azimuth + long double za; // Zenith angle + std::size_t machineid = 0; // Machine ID + std::size_t nifs = 0; // Number of IFs + std::size_t telescopeid = 0; // Telescope ID + std::size_t datatype = 0; // Data type + std::size_t barycentric = 0; // Barycentric correction + std::string rawfile; // Raw file name + double fch1 = 0.0; // Centre frequency of the first channel + double foff = 0.0; // Frequency offset between channels + + bool sigproc_params = false; // Whether to include sigproc parameters ObservationHeader() = default; - ObservationHeader(ObservationHeader const&) = default; + ObservationHeader(ObservationHeader const&) = default; ObservationHeader& operator=(ObservationHeader const&) = default; - }; // template for comparing two floating point objects @@ -72,7 +72,6 @@ is_close(T a, T b, T tolerance = 1e-12) return std::fabs(a - b) < tolerance; } - /** * @brief Parse header information for a DADA header block * @@ -90,15 +89,6 @@ void update_config(PipelineConfig& config, ObservationHeader const& header); bool are_headers_similar(ObservationHeader const& header1, ObservationHeader const& header2); - - - - - - } // namespace skyweaver - - - #endif // SKYWEAVER_OBSERVATIONHEADER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaver.cuh b/cpp/skyweaver/SkyCleaver.cuh index 8c91290..4c7be58 100644 --- a/cpp/skyweaver/SkyCleaver.cuh +++ b/cpp/skyweaver/SkyCleaver.cuh @@ -1,78 +1,81 @@ #ifndef SKYWEAVER_SKYCLEAVER_HPP #define SKYWEAVER_SKYCLEAVER_HPP -#include "skyweaver/DescribedVector.hpp" #include "boost/log/trivial.hpp" #include "psrdada_cpp/psrdadaheader.hpp" #include "psrdada_cpp/raw_bytes.hpp" +#include "skyweaver/DescribedVector.hpp" +#include "skyweaver/MultiFileReader.cuh" +#include "skyweaver/MultiFileWriter.cuh" #include "skyweaver/ObservationHeader.hpp" +#include "skyweaver/SkyCleaverConfig.hpp" +#include "skyweaver/Timer.hpp" #include #include #include #include +#include #include #include #include -#include -#include "skyweaver/MultiFileReader.cuh" -#include "skyweaver/SkyCleaverConfig.hpp" -#include "skyweaver/MultiFileWriter.cuh" -#include "skyweaver/ObservationHeader.hpp" -#include "skyweaver/Timer.hpp" namespace skyweaver { -struct BridgeReader -{ +struct BridgeReader { public: std::vector _tdb_filenames; std::unique_ptr _tdb_reader; std::string freq; }; // BridgeReader + + class SkyCleaver { public: - using InputType = int8_t; - using OutputType = uint8_t; - using InputVectorType = TDBPowersH; + using InputType = int8_t; + using OutputType = uint8_t; + using InputVectorType = TDBPowersH; using OutputVectorType = TFPowersH; - using FreqType = std::size_t; // up to the nearest Hz - using BeamNumberType = std::size_t; - using DMNumberType = std::size_t; + using FreqType = std::size_t; // up to the nearest Hz + using BeamNumberType = std::size_t; + using DMNumberType = std::size_t; - - private: + private: SkyCleaverConfig const& _config; std::map> _bridge_readers; - std::map> _bridge_data; - + std::map> _bridge_data; + std::vector _expected_freqs; std::vector _available_freqs; std::size_t _nsamples_to_read; ObservationHeader _header; std::vector _beam_filenames; - std::map>>> _beam_writers; - std::map>> _beam_data; + std::map>>> + _beam_writers; + std::map>> + _beam_data; std::size_t _total_beam_writers; - void init_readers(); void init_writers(); Timer _timer; - public: SkyCleaver(SkyCleaverConfig const& config); - SkyCleaver(SkyCleaver const&) = delete; + SkyCleaver(SkyCleaver const&) = delete; void operator=(SkyCleaver const&) = delete; void cleave(); - + }; // class SkyCleaver + } // namespace skyweaver #endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index a1ae8c7..feadd9b 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -1,6 +1,7 @@ #ifndef SKYCLEAVERCONFIG_HPP #define SKYCLEAVERCONFIG_HPP -namespace skyweaver { +namespace skyweaver +{ class SkyCleaverConfig { @@ -20,52 +21,56 @@ class SkyCleaverConfig std::string _stokes_mode; std::size_t _dada_header_size; - - - - - public: - - SkyCleaverConfig() : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096) {} - SkyCleaverConfig(SkyCleaverConfig const&) = delete; - - void output_dir(std::string output_dir) { _output_dir = output_dir; } - void root_dir(std::string root_dir) { _root_dir = root_dir; } - void root_prefix(std::string root_prefix) { _root_prefix = root_prefix; } - void out_prefix(std::string out_prefix) { _out_prefix = out_prefix; } - void nthreads(std::size_t nthreads) { _nthreads = nthreads; } - void nsamples_per_block(std::size_t nsamples_per_block) { _nsamples_per_block = nsamples_per_block; } - void nchans(std::size_t nchans) { _nchans = nchans; } - void nbeams(std::size_t nbeams) { _nbeams = nbeams; } - void max_ram_gb(std::size_t max_ram_gb) { _max_ram_gb = max_ram_gb; } - void max_output_filesize(std::size_t max_output_filesize) { _max_output_filesize = max_output_filesize; } - void stream_id(std::size_t stream_id) { _stream_id = stream_id; } - void nbridges(std::size_t nbridges) { _nbridges = nbridges; } - void ndms(std::size_t ndms) { _ndms = ndms; } - void stokes_mode(std::string stokes_mode) { _stokes_mode = stokes_mode; } - void dada_header_size(std::size_t dada_header_size) { _dada_header_size = dada_header_size; } - - - - std::string output_dir() const { return _output_dir; } - std::string root_dir() const { return _root_dir; } - std::string root_prefix() const { return _root_prefix; } - std::string out_prefix() const { return _out_prefix; } - std::size_t nthreads() const { return _nthreads; } - std::size_t nsamples_per_block() const { return _nsamples_per_block; } - std::size_t nchans() const { return _nchans; } - std::size_t nbeams() const { return _nbeams; } - std::size_t max_ram_gb() const { return _max_ram_gb; } - std::size_t max_output_filesize() const { return _max_output_filesize; } - std::size_t stream_id() const { return _stream_id; } - std::size_t nbridges() const { return _nbridges; } - std::size_t ndms() const { return _ndms; } - std::string stokes_mode() const { return _stokes_mode; } - std::size_t dada_header_size() const { return _dada_header_size; } - - + public: + SkyCleaverConfig() + : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), + _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), + _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), + _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096) + { + } + SkyCleaverConfig(SkyCleaverConfig const&) = delete; + void output_dir(std::string output_dir) { _output_dir = output_dir; } + void root_dir(std::string root_dir) { _root_dir = root_dir; } + void root_prefix(std::string root_prefix) { _root_prefix = root_prefix; } + void out_prefix(std::string out_prefix) { _out_prefix = out_prefix; } + void nthreads(std::size_t nthreads) { _nthreads = nthreads; } + void nsamples_per_block(std::size_t nsamples_per_block) + { + _nsamples_per_block = nsamples_per_block; + } + void nchans(std::size_t nchans) { _nchans = nchans; } + void nbeams(std::size_t nbeams) { _nbeams = nbeams; } + void max_ram_gb(std::size_t max_ram_gb) { _max_ram_gb = max_ram_gb; } + void max_output_filesize(std::size_t max_output_filesize) + { + _max_output_filesize = max_output_filesize; + } + void stream_id(std::size_t stream_id) { _stream_id = stream_id; } + void nbridges(std::size_t nbridges) { _nbridges = nbridges; } + void ndms(std::size_t ndms) { _ndms = ndms; } + void stokes_mode(std::string stokes_mode) { _stokes_mode = stokes_mode; } + void dada_header_size(std::size_t dada_header_size) + { + _dada_header_size = dada_header_size; + } + std::string output_dir() const { return _output_dir; } + std::string root_dir() const { return _root_dir; } + std::string root_prefix() const { return _root_prefix; } + std::string out_prefix() const { return _out_prefix; } + std::size_t nthreads() const { return _nthreads; } + std::size_t nsamples_per_block() const { return _nsamples_per_block; } + std::size_t nchans() const { return _nchans; } + std::size_t nbeams() const { return _nbeams; } + std::size_t max_ram_gb() const { return _max_ram_gb; } + std::size_t max_output_filesize() const { return _max_output_filesize; } + std::size_t stream_id() const { return _stream_id; } + std::size_t nbridges() const { return _nbridges; } + std::size_t ndms() const { return _ndms; } + std::string stokes_mode() const { return _stokes_mode; } + std::size_t dada_header_size() const { return _dada_header_size; } }; } // namespace skyweaver #endif // SKYCLEAVERCONFIG_HPP \ No newline at end of file diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 7529621..cfe8a9e 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -7,8 +7,6 @@ #include #include - - /** * Now write a DADA file per DM * with optional time splitting @@ -20,7 +18,6 @@ namespace skyweaver namespace { - std::string get_formatted_time(long double unix_timestamp) { char formatted_time[80]; @@ -40,30 +37,31 @@ std::string get_formatted_time(long double unix_timestamp) // } template -MultiFileWriter::MultiFileWriter(PipelineConfig const& config, - std::string tag, - CreateStreamCallBackType create_stream_callback) +MultiFileWriter::MultiFileWriter( + PipelineConfig const& config, + std::string tag, + CreateStreamCallBackType create_stream_callback) : _tag(tag), _create_stream_callback(create_stream_callback) { MultiFileWriterConfig writer_config; - writer_config.header_size = config.dada_header_size(); - writer_config.max_file_size = config.max_output_filesize(); - writer_config.stokes_mode = config.stokes_mode(); + writer_config.header_size = config.dada_header_size(); + writer_config.max_file_size = config.max_output_filesize(); + writer_config.stokes_mode = config.stokes_mode(); writer_config.base_output_dir = config.output_dir(); _config = writer_config; } template -MultiFileWriter::MultiFileWriter(MultiFileWriterConfig config, - std::string tag, - CreateStreamCallBackType create_stream_callback) - : _config(config), _tag(tag), _create_stream_callback(create_stream_callback) +MultiFileWriter::MultiFileWriter( + MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback) + : _config(config), _tag(tag), + _create_stream_callback(create_stream_callback) { } - - template MultiFileWriter::~MultiFileWriter(){}; @@ -81,14 +79,10 @@ bool MultiFileWriter::has_stream(std::size_t stream_idx) } template -FileOutputStream& +FileOutputStream& MultiFileWriter::create_stream(VectorType const& stream_data, std::size_t stream_idx) { - - - - _config.output_dir = get_output_dir(stream_data, stream_idx); if(_config.extension.empty()) { @@ -97,10 +91,8 @@ MultiFileWriter::create_stream(VectorType const& stream_data, _config.output_basename = get_basefilename(stream_data, stream_idx); - - _file_streams[stream_idx] = _create_stream_callback(_config, _header, stream_data, stream_idx); - - + _file_streams[stream_idx] = + _create_stream_callback(_config, _header, stream_data, stream_idx); return *_file_streams[stream_idx]; } @@ -114,9 +106,8 @@ MultiFileWriter::get_output_dir(VectorType const& stream_data, // // std::stringstream output_dir; output_dir << _config.base_output_dir << "/" - << get_formatted_time(_header.utc_start) << "/" - << stream_idx; - + << get_formatted_time(_header.utc_start) << "/" << stream_idx; + return output_dir.str(); } @@ -147,10 +138,9 @@ MultiFileWriter::get_extension(VectorType const& stream_data) { std::string dims = stream_data.dims_as_string(); for(auto& c: dims) { c = std::tolower(static_cast(c)); } - if(dims =="t") { + if(dims == "t") { return ".dat"; - } - else if(dims == "tf") { + } else if(dims == "tf") { return ".fil"; } return "." + dims; @@ -182,11 +172,9 @@ bool MultiFileWriter::operator()(VectorType const& stream_data, template bool MultiFileWriter::write(VectorType const& stream_data, - std::size_t stream_idx) + std::size_t stream_idx) { - - return this->operator()(stream_data, stream_idx); - + return this->operator()(stream_data, stream_idx); } } // namespace skyweaver \ No newline at end of file diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 71f0b42..10d9929 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -4,22 +4,23 @@ #include "skyweaver/MultiFileWriter.cuh" #include "skyweaver/ObservationHeader.hpp" #include "skyweaver/SigprocHeader.hpp" + #include -#include -#include -#include -#include #include +#include +#include +#include +#include #include +#include #include -#include -#include #include -#include - +#include +#include -namespace { - /** +namespace +{ +/** * The expectation is that each file contains data in * TBTF order. Need to explicitly update: * INNER_T --> the number of timesamples per block that was processed @@ -66,16 +67,17 @@ CHAN0_IDX 2688 )"; #define MJD_UNIX_EPOCH 40587.0 -} +} // namespace namespace skyweaver { -namespace detail +namespace detail { -template inline -std::unique_ptr create_dada_file_stream(MultiFileWriterConfig const& config, - ObservationHeader const& header, - VectorType const& stream_data, - std::size_t stream_idx) +template +inline std::unique_ptr +create_dada_file_stream(MultiFileWriterConfig const& config, + ObservationHeader const& header, + VectorType const& stream_data, + std::size_t stream_idx) { BOOST_LOG_TRIVIAL(debug) << "Creating stream based on stream prototype: " << stream_data.describe(); @@ -85,113 +87,120 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig config.max_file_size / stream_data.size() / sizeof(typename VectorType::value_type)) * stream_data.size(); - + BOOST_LOG_TRIVIAL(debug) << "Maximum allowed file size = " << filesize << " bytes (+header)"; std::stringstream output_dir; - output_dir << config.output_dir << "/" - << std::fixed << std::setfill('0') << std::setw(9) - << static_cast(header.frequency); - - std::stringstream output_basename; - output_basename << config.output_basename << "_" - << std::fixed << std::setfill('0') << std::setw(9) - << static_cast(header.frequency); - - std::unique_ptr file_stream = std::make_unique( - output_dir.str(), - output_basename.str(), - config.extension, - filesize, - [&, header, stream_data, stream_idx, filesize]( - std::size_t& header_size, - std::size_t bytes_written, - std::size_t file_idx) -> std::shared_ptr { - header_size = config.header_size; - char* temp_header = new char[header_size]; - std::fill(temp_header, temp_header + header_size, 0); - std::memcpy(temp_header, - default_dada_header.c_str(), - default_dada_header.size()); - psrdada_cpp::RawBytes bytes(temp_header, - header_size, - header_size, - false); - Header header_writer(bytes); - header_writer.set("SOURCE", header.source_name); - header_writer.set("RA", header.ra); - header_writer.set("DEC", header.dec); - header_writer.set("NBEAM", stream_data.nbeams()); - header_writer.set("NCHAN", stream_data.nchannels()); - header_writer.set("OBS_NCHAN", header.obs_nchans); - header_writer.set("OBS_FREQUENCY", - header.obs_frequency); - header_writer.set("OBS_BW", header.obs_bandwidth); - header_writer.set("NSAMP", stream_data.nsamples()); - if(stream_data.ndms()) { - header_writer.set("NDMS", stream_data.ndms()); - header_writer.set("DMS", stream_data.dms(), 7); - } - header_writer.set( - "COHERENT_DM", - static_cast(stream_data.reference_dm())); - try{ - header_writer.set("FREQ", std::accumulate( - stream_data.frequencies().begin(), - stream_data.frequencies().end(), - 0.0)/stream_data.frequencies().size()); - } catch(std::runtime_error& ){ - BOOST_LOG_TRIVIAL(warning) << "Warning: Frequencies array was stale, using the centre frequency from the header"; - header_writer.set("FREQ", header.frequency); - } - - header_writer.set("BW", header.bandwidth); - header_writer.set("TSAMP", stream_data.tsamp() * 1e6); - if(config.stokes_mode == "IQUV") { - header_writer.set("NPOL", 4); - } else { - header_writer.set("NPOL", 1); - } - header_writer.set("STOKES_MODE", - config.stokes_mode); - header_writer.set("ORDER", - stream_data.dims_as_string()); - header_writer.set("CHAN0_IDX", header.chan0_idx); - header_writer.set("FILE_SIZE", filesize); - header_writer.set("FILE_NUMBER", file_idx); - header_writer.set("OBS_OFFSET", bytes_written); - header_writer.set("OBS_OVERLAP", 0); - - long double tstart = header.utc_start + stream_data.utc_offset(); - - header_writer.set("UTC_START", tstart); - header_writer.set("MJD_START", MJD_UNIX_EPOCH + tstart / 86400.0); - std::shared_ptr header_ptr( - temp_header, - std::default_delete()); - - return header_ptr; - }); + output_dir << config.output_dir << "/" << std::fixed << std::setfill('0') + << std::setw(9) << static_cast(header.frequency); + + std::stringstream output_basename; + output_basename << config.output_basename << "_" << std::fixed + << std::setfill('0') << std::setw(9) + << static_cast(header.frequency); + + std::unique_ptr file_stream = + std::make_unique( + output_dir.str(), + output_basename.str(), + config.extension, + filesize, + [&, header, stream_data, stream_idx, filesize]( + std::size_t& header_size, + std::size_t bytes_written, + std::size_t file_idx) -> std::shared_ptr { + header_size = config.header_size; + char* temp_header = new char[header_size]; + std::fill(temp_header, temp_header + header_size, 0); + std::memcpy(temp_header, + default_dada_header.c_str(), + default_dada_header.size()); + psrdada_cpp::RawBytes bytes(temp_header, + header_size, + header_size, + false); + Header header_writer(bytes); + header_writer.set("SOURCE", header.source_name); + header_writer.set("RA", header.ra); + header_writer.set("DEC", header.dec); + header_writer.set("NBEAM", stream_data.nbeams()); + header_writer.set("NCHAN", + stream_data.nchannels()); + header_writer.set("OBS_NCHAN", header.obs_nchans); + header_writer.set("OBS_FREQUENCY", + header.obs_frequency); + header_writer.set("OBS_BW", header.obs_bandwidth); + header_writer.set("NSAMP", stream_data.nsamples()); + if(stream_data.ndms()) { + header_writer.set("NDMS", stream_data.ndms()); + header_writer.set("DMS", stream_data.dms(), 7); + } + header_writer.set( + "COHERENT_DM", + static_cast(stream_data.reference_dm())); + try { + header_writer.set( + "FREQ", + std::accumulate(stream_data.frequencies().begin(), + stream_data.frequencies().end(), + 0.0) / + stream_data.frequencies().size()); + } catch(std::runtime_error&) { + BOOST_LOG_TRIVIAL(warning) + << "Warning: Frequencies array was stale, using the " + "centre frequency from the header"; + header_writer.set("FREQ", header.frequency); + } + + header_writer.set("BW", header.bandwidth); + header_writer.set("TSAMP", + stream_data.tsamp() * 1e6); + if(config.stokes_mode == "IQUV") { + header_writer.set("NPOL", 4); + } else { + header_writer.set("NPOL", 1); + } + header_writer.set("STOKES_MODE", + config.stokes_mode); + header_writer.set("ORDER", + stream_data.dims_as_string()); + header_writer.set("CHAN0_IDX", header.chan0_idx); + header_writer.set("FILE_SIZE", filesize); + header_writer.set("FILE_NUMBER", file_idx); + header_writer.set("OBS_OFFSET", bytes_written); + header_writer.set("OBS_OVERLAP", 0); + + long double tstart = + header.utc_start + stream_data.utc_offset(); + + header_writer.set("UTC_START", tstart); + header_writer.set("MJD_START", + MJD_UNIX_EPOCH + + tstart / 86400.0); + std::shared_ptr header_ptr( + temp_header, + std::default_delete()); + + return header_ptr; + }); return file_stream; } -template inline -std::unique_ptr create_sigproc_file_stream(MultiFileWriterConfig const& config, - ObservationHeader const& obs_header, - VectorType const& stream_data, - std::size_t stream_idx) +template +inline std::unique_ptr +create_sigproc_file_stream(MultiFileWriterConfig const& config, + ObservationHeader const& obs_header, + VectorType const& stream_data, + std::size_t stream_idx) { - - - BOOST_LOG_TRIVIAL(debug) << "Creating stream based on stream prototype: " << stream_data.describe(); ObservationHeader header = obs_header; BOOST_LOG_TRIVIAL(info) << "Header: " << header.to_string(); - + // Here we round the file size to a multiple of the stream prototype std::size_t filesize = std::max(1ul, @@ -199,36 +208,40 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon sizeof(typename VectorType::value_type)) * stream_data.size(); BOOST_LOG_TRIVIAL(debug) - << "Maximum allowed file size = " << filesize << " bytes (+header)"; + << "Maximum allowed file size = " << filesize << " bytes (+header)"; - - double foff = -1* static_cast(header.obs_bandwidth / header.nchans)/1e6;// MHz - // 1* foff instead of 0.5* foff below because the dedispersion causes all the frequencies to change by half the bandwidth to refer to the bottom of the channel - double fch1 = static_cast(header.obs_frequency + header.obs_bandwidth / 2.0)/1e6 + foff; // MHz + double foff = -1 * + static_cast(header.obs_bandwidth / header.nchans) / + 1e6; // MHz + // 1* foff instead of 0.5* foff below because the dedispersion causes all + // the frequencies to change by half the bandwidth to refer to the bottom of + // the channel + double fch1 = + static_cast(header.obs_frequency + header.obs_bandwidth / 2.0) / + 1e6 + + foff; // MHz double utc_start = static_cast(header.utc_start); header.mjd_start = (utc_start / 86400.0) + MJD_UNIX_EPOCH; - uint32_t datatype = 0; + uint32_t datatype = 0; uint32_t barycentric = 0; // uint32_t ibeam = 0; - double az = 0.0; - double za = 0.0; + double az = 0.0; + double za = 0.0; uint32_t nifs = header.npol; header.sigproc_params = true; - header.rawfile = std::string("unset"); - header.fch1 = fch1; - header.foff = foff; - header.tsamp = header.tsamp/1e6; - header.az = az; - header.za = za; - header.datatype = datatype; - header.barycentric = barycentric; - header.nifs = nifs; - header.telescopeid = 64; - - + header.rawfile = std::string("unset"); + header.fch1 = fch1; + header.foff = foff; + header.tsamp = header.tsamp / 1e6; + header.az = az; + header.za = za; + header.datatype = datatype; + header.barycentric = barycentric; + header.nifs = nifs; + header.telescopeid = 64; BOOST_LOG_TRIVIAL(info) << "Creating Sigproc file stream"; @@ -246,42 +259,41 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon // // causing compiler bugs prior to g++ 5.x // char formatted_time[80]; // strftime(formatted_time, 80, "%Y-%m-%d-%H:%M:%S", ptm); - // base_filename << formatted_time; - - //make config.output_dir if it does not exist - - - std::unique_ptr file_stream = std::make_unique( - config.output_dir, - config.output_basename, - config.extension, - filesize, - [header](std::size_t& header_size, - std::size_t bytes_written, - std::size_t file_idx) -> std::shared_ptr { - // We do not explicitly delete[] this array - // Cleanup is handled by the shared pointer - // created below - std::ostringstream header_stream; - // get ostream from temp_header - - - - SigprocHeader sigproc_header(header); - double mjd_offset = (((bytes_written / (header.nbits / 8.0)) / (header.nchans)) * - header.tsamp) / - (86400.0); - sigproc_header.add_time_offset(mjd_offset); - sigproc_header.write_header(header_stream); - std::string header_str = header_stream.str(); - header_size = header_str.size(); - char* header_cstr = new char[header_size]; - std::copy(header_str.begin(), header_str.end(), header_cstr); - std::shared_ptr header_ptr( - header_cstr, - std::default_delete()); - return header_ptr; - }); + // base_filename << formatted_time; + + // make config.output_dir if it does not exist + + std::unique_ptr file_stream = + std::make_unique( + config.output_dir, + config.output_basename, + config.extension, + filesize, + [header](std::size_t& header_size, + std::size_t bytes_written, + std::size_t file_idx) -> std::shared_ptr { + // We do not explicitly delete[] this array + // Cleanup is handled by the shared pointer + // created below + std::ostringstream header_stream; + // get ostream from temp_header + + SigprocHeader sigproc_header(header); + double mjd_offset = (((bytes_written / (header.nbits / 8.0)) / + (header.nchans)) * + header.tsamp) / + (86400.0); + sigproc_header.add_time_offset(mjd_offset); + sigproc_header.write_header(header_stream); + std::string header_str = header_stream.str(); + header_size = header_str.size(); + char* header_cstr = new char[header_size]; + std::copy(header_str.begin(), header_str.end(), header_cstr); + std::shared_ptr header_ptr( + header_cstr, + std::default_delete()); + return header_ptr; + }); return file_stream; } diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 52e4f41..12d9976 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -61,7 +61,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, pipeline_config.coherent_dms()); } /* - * @brief Create a new CoherentDedis§rser object, mostly used only for + * @brief Create a new CoherentDedisperser object, mostly used only for * testing * * @param config The config reference diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index 13255aa..7040a4e 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -10,7 +10,7 @@ std::vector parse_float_list(std::string const& str) { std::vector values; std::size_t start = 0; - std::size_t end = 0; + std::size_t end = 0; while(end != std::string::npos) { end = str.find(',', start); values.push_back(std::stof(str.substr(start, end - start))); @@ -25,7 +25,7 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, Header parser(raw_header); header.order = parser.get("ORDER"); - if(header.order.find("A") != std::string::npos) { + if(header.order.find("A") != std::string::npos) { header.nantennas = parser.get("NANT"); header.sample_clock = parser.get("SAMPLE_CLOCK"); @@ -35,16 +35,16 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.sync_time = parser.get("SYNC_TIME"); } - - header.refdm = parser.get_or_default("COHERENT_DM", 0.0); + header.refdm = + parser.get_or_default("COHERENT_DM", 0.0); - header.npol = parser.get("NPOL"); - header.nbits = parser.get("NBIT"); - header.nchans = parser.get("NCHAN"); + header.npol = parser.get("NPOL"); + header.nbits = parser.get("NBIT"); + header.nchans = parser.get("NCHAN"); - header.bandwidth = parser.get("BW"); - header.frequency = parser.get("FREQ"); + header.bandwidth = parser.get("BW"); + header.frequency = parser.get("FREQ"); header.tsamp = parser.get("TSAMP"); @@ -58,18 +58,21 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.chan0_idx = parser.get("CHAN0_IDX"); header.obs_offset = parser.get("OBS_OFFSET"); - header.obs_bandwidth = parser.get_or_default("OBS_BW", 856e6); - header.obs_nchans = parser.get_or_default("OBS_NCHAN", 4096); - header.obs_frequency = parser.get_or_default("OBS_FREQUENCY", - parser.get_or_default("OBS_FREQ", - header.frequency)); + header.obs_bandwidth = + parser.get_or_default("OBS_BW", 856e6); + header.obs_nchans = + parser.get_or_default("OBS_NCHAN", 4096); + header.obs_frequency = + parser.get_or_default( + "OBS_FREQUENCY", + parser.get_or_default( + "OBS_FREQ", + header.frequency)); header.ndms = parser.get_or_default("NDMS", "0"); if(header.ndms != "0") { header.dms = parse_float_list(parser.get("DMS")); } - - } void validate_header(ObservationHeader const& header, PipelineConfig const& config) @@ -104,7 +107,6 @@ void update_config(PipelineConfig& config, ObservationHeader const& header) // TO DO: might need to add other variables in the future. } - bool are_headers_similar(ObservationHeader const& header1, ObservationHeader const& header2) { @@ -153,14 +155,10 @@ std::string ObservationHeader::to_string() const if(ndms != "0") { oss << " ndms: " << ndms << "\n"; oss << " dms: "; - for(auto dm : dms) { - oss << dm << " "; - } + for(auto dm: dms) { oss << dm << " "; } oss << "\n"; } - if(sigproc_params) - { - + if(sigproc_params) { oss << " Sigproc parameters:\n" << " az: " << az << "\n" << " za: " << za << "\n" diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cu index ff7ad8f..fdb9278 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cu +++ b/cpp/skyweaver/src/skycleaver_cli.cu @@ -1,4 +1,11 @@ +#include "skyweaver/SkyCleaver.cuh" +#include "skyweaver/SkyCleaverConfig.hpp" +#include "skyweaver/logging.hpp" + #include +#include +#include +#include #include #include #include @@ -12,20 +19,12 @@ #include #include #include -#include -#include -#include -#include "skyweaver/logging.hpp" -#include "skyweaver/SkyCleaverConfig.hpp" -#include "skyweaver/SkyCleaver.cuh" #define BOOST_LOG_DYN_LINK 1 - -namespace +namespace { - -std::string skycleaver_splash=R"( +std::string skycleaver_splash = R"( __ __ .-----.| |--.--.--.----.| |.-----.---.-.--.--.-----.----. |__ --|| <| | | __|| || -__| _ | | | -__| _| @@ -39,8 +38,7 @@ const size_t ERROR_UNHANDLED_EXCEPTION = 2; const char* build_time = __DATE__ " " __TIME__; - -} +} // namespace namespace std { @@ -51,9 +49,6 @@ std::ostream& operator<<(std::ostream& os, const std::vector& vec) } } // namespace std - - - int main(int argc, char** argv) { std::cout << skycleaver_splash; @@ -62,7 +57,6 @@ int main(int argc, char** argv) skyweaver::SkyCleaverConfig config; - namespace po = boost::program_options; po::options_description generic("Generic options"); @@ -71,158 +65,143 @@ int main(int argc, char** argv) "Skycleaver configuration file"); po::options_description main_options("Main options"); - main_options.add_options() - ("help,h", "Produce help message") - ("root-dir,r", - po::value() - -> required() - ->notifier( - [&config](std::string key) { config.root_dir(key); }), - "The output directory for all results") - ("output-dir", - po::value() - ->default_value(config.output_dir()) - ->notifier( - [&config](std::string key) { config.output_dir(key); }), - "The output directory for all results") - ("root-prefix", - po::value() - ->default_value(config.root_prefix()) - ->notifier( - [&config](std::string key) { config.root_prefix(key); }), - "The prefix for all output files") - ("out-prefix", - po::value() - ->default_value(config.out_prefix()) - ->notifier( - [&config](std::string key) { config.out_prefix(key); }), - "The prefix for all output files") - ("nthreads", - po::value() - ->default_value(config.nthreads()) - ->notifier( - [&config](unsigned int key) { config.nthreads(key); }), - "The number of threads to use for processing") - ("nsamples-per-block", - po::value() - ->default_value(config.nsamples_per_block()) - ->notifier( - [&config](std::size_t key) { config.nsamples_per_block(key); }), - "The number of samples per block") - ("nchans", - po::value() - ->default_value(config.nchans()) - ->notifier( - [&config](std::size_t key) { config.nchans(key); }), - "The number of channels") - ("nbridges", - po::value() - ->default_value(config.nbridges()) - ->notifier( - [&config](std::size_t key) { config.nbridges(key); }), - "The number of bridges") - ("nbeams", - po::value() - ->default_value(config.nbeams()) - ->notifier( - [&config](std::size_t key) { config.nbeams(key); }), - "The number of beams") - ("ndms", - po::value() - ->default_value(config.ndms()) - ->notifier( - [&config](std::size_t key) { config.ndms(key); }), - "The number of DMs") - ("stokes-mode", - po::value() - ->default_value(config.stokes_mode()) - ->notifier( - [&config](std::string key) { config.stokes_mode(key); }), - "The stokes mode") - ("stream-id", - po::value() - ->default_value(config.stream_id()) - ->notifier( - [&config](std::size_t key) { config.stream_id(key); }), - "The stream id") - ("max-ram-gb", - po::value() - ->default_value(config.max_ram_gb()) - ->notifier( - [&config](std::size_t key) { config.max_ram_gb(key); }), - "The maximum amount of RAM to use in GB") - ("max-output-filesize", - po::value() - ->default_value(config.max_output_filesize()) - ->notifier( - [&config](std::size_t key) { config.max_output_filesize(key); }), - "The maximum output file size in bytes") - ("dada-header-size", - po::value() - ->default_value(config.dada_header_size()) - ->notifier( - [&config](std::size_t key) { config.dada_header_size(key); }), - "The size of the DADA header") - ("log-level", - po::value()->default_value("info")->notifier( - [](std::string level) { skyweaver::init_logging(level); }), - "The logging level to use (debug, info, warning, error)"); - - po::options_description cmdline_options; - cmdline_options.add(generic).add(main_options); - - // set options allowed in config file - po::options_description config_file_options; - config_file_options.add(main_options); - po::variables_map variable_map; - try { - po::store(po::command_line_parser(argc, argv) - .options(cmdline_options) - .run(), - variable_map); - if(variable_map.count("help")) { - std::cout << "skycleaver -- A pipeline that cleaves input TDB files, " - "and cleaves them to form output Sigproc Filterbank files." - << std::endl - << cmdline_options << std::endl; - return SUCCESS; - } - } catch(po::error& e) { - std::cerr << "ERROR: " << e.what() << std::endl << std::endl; - return ERROR_IN_COMMAND_LINE; + main_options.add_options()("help,h", "Produce help message")( + "root-dir,r", + po::value()->required()->notifier( + [&config](std::string key) { config.root_dir(key); }), + "The output directory for all results")( + "output-dir", + po::value() + ->default_value(config.output_dir()) + ->notifier([&config](std::string key) { config.output_dir(key); }), + "The output directory for all results")( + "root-prefix", + po::value() + ->default_value(config.root_prefix()) + ->notifier([&config](std::string key) { config.root_prefix(key); }), + "The prefix for all output files")( + "out-prefix", + po::value() + ->default_value(config.out_prefix()) + ->notifier([&config](std::string key) { config.out_prefix(key); }), + "The prefix for all output files")( + "nthreads", + po::value() + ->default_value(config.nthreads()) + ->notifier([&config](unsigned int key) { config.nthreads(key); }), + "The number of threads to use for processing")( + "nsamples-per-block", + po::value() + ->default_value(config.nsamples_per_block()) + ->notifier( + [&config](std::size_t key) { config.nsamples_per_block(key); }), + "The number of samples per block")( + "nchans", + po::value() + ->default_value(config.nchans()) + ->notifier([&config](std::size_t key) { config.nchans(key); }), + "The number of channels")( + "nbridges", + po::value() + ->default_value(config.nbridges()) + ->notifier([&config](std::size_t key) { config.nbridges(key); }), + "The number of bridges")( + "nbeams", + po::value() + ->default_value(config.nbeams()) + ->notifier([&config](std::size_t key) { config.nbeams(key); }), + "The number of beams")( + "ndms", + po::value() + ->default_value(config.ndms()) + ->notifier([&config](std::size_t key) { config.ndms(key); }), + "The number of DMs")( + "stokes-mode", + po::value() + ->default_value(config.stokes_mode()) + ->notifier([&config](std::string key) { config.stokes_mode(key); }), + "The stokes mode")( + "stream-id", + po::value() + ->default_value(config.stream_id()) + ->notifier([&config](std::size_t key) { config.stream_id(key); }), + "The stream id")( + "max-ram-gb", + po::value() + ->default_value(config.max_ram_gb()) + ->notifier([&config](std::size_t key) { config.max_ram_gb(key); }), + "The maximum amount of RAM to use in GB")( + "max-output-filesize", + po::value() + ->default_value(config.max_output_filesize()) + ->notifier([&config](std::size_t key) { + config.max_output_filesize(key); + }), + "The maximum output file size in bytes")( + "dada-header-size", + po::value() + ->default_value(config.dada_header_size()) + ->notifier( + [&config](std::size_t key) { config.dada_header_size(key); }), + "The size of the DADA header")( + "log-level", + po::value()->default_value("info")->notifier( + [](std::string level) { skyweaver::init_logging(level); }), + "The logging level to use (debug, info, warning, error)"); + + po::options_description cmdline_options; + cmdline_options.add(generic).add(main_options); + + // set options allowed in config file + po::options_description config_file_options; + config_file_options.add(main_options); + po::variables_map variable_map; + try { + po::store( + po::command_line_parser(argc, argv).options(cmdline_options).run(), + variable_map); + if(variable_map.count("help")) { + std::cout + << "skycleaver -- A pipeline that cleaves input TDB files, " + "and cleaves them to form output Sigproc Filterbank files." + << std::endl + << cmdline_options << std::endl; + return SUCCESS; } - - auto config_file = variable_map.at("cfg").as(); - - if(config_file != "") { - std::ifstream config_fs(config_file.c_str()); - if(!config_fs.is_open()) { - std::cerr << "Unable to open configuration file: " - << config_file << " (" << std::strerror(errno) - << ")\n"; - return ERROR_UNHANDLED_EXCEPTION; - } else { - po::store(po::parse_config_file(config_fs, config_file_options), - variable_map); - } + } catch(po::error& e) { + std::cerr << "ERROR: " << e.what() << std::endl << std::endl; + return ERROR_IN_COMMAND_LINE; + } + + auto config_file = variable_map.at("cfg").as(); + + if(config_file != "") { + std::ifstream config_fs(config_file.c_str()); + if(!config_fs.is_open()) { + std::cerr << "Unable to open configuration file: " << config_file + << " (" << std::strerror(errno) << ")\n"; + return ERROR_UNHANDLED_EXCEPTION; + } else { + po::store(po::parse_config_file(config_fs, config_file_options), + variable_map); } - po::notify(variable_map); - BOOST_LOG_NAMED_SCOPE("skycleaver_cli"); - BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; - BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); - BOOST_LOG_TRIVIAL(info) << "output_dir: " << config.output_dir(); - BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); - BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); - BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); - BOOST_LOG_TRIVIAL(info) << "nsamples_per_block: " << config.nsamples_per_block(); - BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); - BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); - BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); - BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); - - - skyweaver::SkyCleaver skycleaver(config); - skycleaver.cleave(); - - + } + po::notify(variable_map); + BOOST_LOG_NAMED_SCOPE("skycleaver_cli"); + BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; + BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); + BOOST_LOG_TRIVIAL(info) << "output_dir: " << config.output_dir(); + BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); + BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); + BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); + BOOST_LOG_TRIVIAL(info) + << "nsamples_per_block: " << config.nsamples_per_block(); + BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); + BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); + BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); + BOOST_LOG_TRIVIAL(info) + << "max_output_filesize: " << config.max_output_filesize(); + + skyweaver::SkyCleaver skycleaver(config); + skycleaver.cleave(); } \ No newline at end of file diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 3092164..8204ca3 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -52,7 +52,7 @@ class NullHandler { public: template - void init(Args... args){}; + void init(Args... args) {}; template bool operator()(Args... args) @@ -263,19 +263,22 @@ void setup_pipeline(skyweaver::PipelineConfig& config) IBWriterType ib_handler(config, "ib", create_stream_callback_ib); using StatsWriterType = - skyweaver::MultiFileWriter>; + skyweaver::MultiFileWriter>; typename StatsWriterType::CreateStreamCallBackType create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; - StatsWriterType stats_handler(config, "stats", create_stream_callback_stats); - + StatsWriterType stats_handler(config, + "stats", + create_stream_callback_stats); if constexpr(enable_incoherent_dedispersion) { - using CBWriterType = skyweaver::MultiFileWriter>; + using CBWriterType = + skyweaver::MultiFileWriter>; typename CBWriterType::CreateStreamCallBackType - create_stream_callback_cb = - skyweaver::detail::create_dada_file_stream>; + create_stream_callback_cb = + skyweaver::detail::create_dada_file_stream< + skyweaver::TDBPowersH>; skyweaver::MultiFileWriter> cb_file_writer(config, "cb", create_stream_callback_cb); skyweaver::IncoherentDedispersionPipeline>; + using CBWriterType = + skyweaver::MultiFileWriter>; typename CBWriterType::CreateStreamCallBackType - create_stream_callback_cb = - skyweaver::detail::create_dada_file_stream>; - CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb); + create_stream_callback_cb = + skyweaver::detail::create_dada_file_stream< + skyweaver::TFBPowersD>; + CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb); skyweaver::BeamformerPipeline, true>(config); } else if(config.stokes_mode() == "QU") { - setup_pipeline, - true>(config); + setup_pipeline< + skyweaver::StokesTraits, + true>(config); } else if(config.stokes_mode() == "IV") { - setup_pipeline, - true>(config); + setup_pipeline< + skyweaver::StokesTraits, + true>(config); } else if(config.stokes_mode() == "IQUV") { setup_pipeline( config); @@ -586,13 +593,15 @@ int main(int argc, char** argv) skyweaver::StokesParameter::V>, false>(config); } else if(config.stokes_mode() == "QU") { - setup_pipeline, - false>(config); + setup_pipeline< + skyweaver::StokesTraits, + false>(config); } else if(config.stokes_mode() == "IV") { - setup_pipeline, - false>(config); + setup_pipeline< + skyweaver::StokesTraits, + false>(config); } else if(config.stokes_mode() == "IQUV") { setup_pipeline( config); From 6a331081fdbb4744fe8e6d2f566da8f64b151e96 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 26 Sep 2024 16:02:57 +0000 Subject: [PATCH 21/65] Add .tmp suffix to output filenames, and remove it when they are closed. --- cpp/skyweaver/FileOutputStream.hpp | 2 ++ cpp/skyweaver/src/FileOutputStream.cpp | 29 +++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cpp/skyweaver/FileOutputStream.hpp b/cpp/skyweaver/FileOutputStream.hpp index 53dc988..968cc92 100644 --- a/cpp/skyweaver/FileOutputStream.hpp +++ b/cpp/skyweaver/FileOutputStream.hpp @@ -48,6 +48,8 @@ class FileOutputStream std::string _full_path; std::size_t _bytes_requested; std::size_t _bytes_written; + std::string _temporary_suffix; + std::string _temporary_path; std::ofstream _stream; }; diff --git a/cpp/skyweaver/src/FileOutputStream.cpp b/cpp/skyweaver/src/FileOutputStream.cpp index f4d2ffa..40ca0ad 100644 --- a/cpp/skyweaver/src/FileOutputStream.cpp +++ b/cpp/skyweaver/src/FileOutputStream.cpp @@ -38,15 +38,16 @@ void create_directories(const fs::path& path) } FileOutputStream::File::File(std::string const& fname, std::size_t bytes) - : _full_path(fname), _bytes_requested(bytes), _bytes_written(0) + : _full_path(fname), _bytes_requested(bytes), _bytes_written(0), _temporary_suffix(".tmp") { + _temporary_path = _full_path + _temporary_suffix; _stream.exceptions(std::ofstream::failbit | std::ofstream::badbit); - _stream.open(_full_path, std::ofstream::out | std::ofstream::binary); + _stream.open(_temporary_path, std::ofstream::out | std::ofstream::binary); if(_stream.is_open()) { - BOOST_LOG_TRIVIAL(info) << "Opened output file " << _full_path; + BOOST_LOG_TRIVIAL(info) << "Opened output file " << _temporary_path; } else { std::stringstream error_message; - error_message << "Could not open file " << _full_path; + error_message << "Could not open file " << _temporary_path; BOOST_LOG_TRIVIAL(error) << error_message.str(); throw std::runtime_error(error_message.str()); } @@ -55,15 +56,29 @@ FileOutputStream::File::File(std::string const& fname, std::size_t bytes) FileOutputStream::File::~File() { if(_stream.is_open()) { - BOOST_LOG_TRIVIAL(info) << "Closing file " << _full_path; + BOOST_LOG_TRIVIAL(info) << "Closing file " << _temporary_path; _stream.close(); } + BOOST_LOG_TRIVIAL(info) << "Renaming file " << _temporary_path + << " to " << _full_path; + int res = std::rename(_temporary_path.c_str(), _full_path.c_str()); + + // Can't throw an exception from a destructor, + // but we'll at least put a warning in the log! + if (res) + { + std::stringstream error_message; + error_message << "Error renaming file " << _temporary_path + << " to " << _full_path + << " (" << res << ")."; + BOOST_LOG_TRIVIAL(error) << error_message.str(); + } } std::size_t FileOutputStream::File::write(char const* ptr, std::size_t bytes) { BOOST_LOG_TRIVIAL(debug) - << "Writing " << bytes << " bytes to " << _full_path; + << "Writing " << bytes << " bytes to " << _temporary_path; std::size_t bytes_remaining = _bytes_requested - _bytes_written; try { if(bytes > bytes_remaining) { @@ -89,7 +104,7 @@ std::size_t FileOutputStream::File::write(char const* ptr, std::size_t bytes) reason = "eofbit set."; } - BOOST_LOG_TRIVIAL(error) << "Error while writing to " << _full_path + BOOST_LOG_TRIVIAL(error) << "Error while writing to " << _temporary_path << " (" << e.what() << ") because of reason: " << reason; throw; From 456719ad1891e5f1d13240428512ac5f489979ca Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Mon, 30 Sep 2024 16:40:12 +0200 Subject: [PATCH 22/65] cleave only required samples --- cpp/skyweaver/SkyCleaverConfig.hpp | 13 ++++++++++++- cpp/skyweaver/src/SkyCleaver.cu | 27 +++++++++++++++++++++------ cpp/skyweaver/src/skycleaver_cli.cu | 21 ++++++++++++++++----- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index feadd9b..1972812 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -20,13 +20,16 @@ class SkyCleaverConfig std::size_t _ndms; std::string _stokes_mode; std::size_t _dada_header_size; + std::size_t _start_sample; + std::size_t _nsamples_to_read; public: SkyCleaverConfig() : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), - _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096) + _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096), + _start_sample(0), _nsamples_to_read(0) { } SkyCleaverConfig(SkyCleaverConfig const&) = delete; @@ -55,6 +58,11 @@ class SkyCleaverConfig { _dada_header_size = dada_header_size; } + void start_sample(std::size_t start_sample) { _start_sample = start_sample; } + void nsamples_to_read(std::size_t nsamples_to_read) + { + _nsamples_to_read = nsamples_to_read; + } std::string output_dir() const { return _output_dir; } std::string root_dir() const { return _root_dir; } @@ -71,6 +79,9 @@ class SkyCleaverConfig std::size_t ndms() const { return _ndms; } std::string stokes_mode() const { return _stokes_mode; } std::size_t dada_header_size() const { return _dada_header_size; } + std::size_t start_sample() const { return _start_sample; } + std::size_t nsamples_to_read() const { return _nsamples_to_read; } + }; } // namespace skyweaver #endif // SKYCLEAVERCONFIG_HPP \ No newline at end of file diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index bf195cd..b645414 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -151,7 +151,6 @@ void SkyCleaver::init_readers() long double obs_centre_freq = header.obs_frequency; long double obs_bandwidth = header.obs_bandwidth; - std::size_t obs_nchans = header.obs_nchans; long double start_freq = obs_centre_freq - obs_bandwidth / 2; @@ -159,7 +158,7 @@ void SkyCleaver::init_readers() for(int i = 0; i < nbridges; i++) { - int ifreq = std::lround( std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); + int ifreq = std::lround(std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) << "Expected frequency [" << i << "]: " << ifreq; @@ -224,9 +223,6 @@ void SkyCleaver::init_readers() } } - BOOST_LOG_TRIVIAL(debug) << "Smallest data size: " << smallest_data_size; - BOOST_LOG_TRIVIAL(debug) << "ndm: " << _config.ndms(); - BOOST_LOG_TRIVIAL(debug) << "nbeams: " << _config.nbeams(); if(smallest_data_size % (_config.ndms() * _config.nbeams()) != 0) { std::runtime_error("Data size is not a multiple of ndms * nbeams"); @@ -235,6 +231,12 @@ void SkyCleaver::init_readers() std::size_t smallest_nsamples = std::floor(smallest_data_size / _config.ndms() / _config.nbeams()); + if (smallest_nsamples < _config.start_sample()) { + std::runtime_error("start_sample is greater than the smallest_nsamples in the data."); + } + + smallest_nsamples = smallest_nsamples - _config.start_sample(); + BOOST_LOG_TRIVIAL(info) << "Smallest data size: " << smallest_data_size << " Smallest number of samples: " << smallest_nsamples; @@ -244,7 +246,20 @@ void SkyCleaver::init_readers() "Smallest data size is less than nsamples_per_block"); } - _nsamples_to_read = smallest_nsamples; + if(smallest_nsamples < _config.nsamples_to_read()) { + std::runtime_error( + "Smallest data size is less than nsamples_to_read"); + } + + + _nsamples_to_read = _config.nsamples_to_read(); + + std::size_t bytes_seeking = (_config.start_sample() * _config.ndms() * _config.nbeams() * sizeof(InputVectorType::value_type)); + for(const auto& [freq, reader]: _bridge_readers) { + _bridge_readers[freq]->seekg(bytes_seeking, + std::ios_base::beg); + } + BOOST_LOG_TRIVIAL(info) << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cu index fdb9278..d8fc2bd 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cu +++ b/cpp/skyweaver/src/skycleaver_cli.cu @@ -147,7 +147,16 @@ int main(int argc, char** argv) "log-level", po::value()->default_value("info")->notifier( [](std::string level) { skyweaver::init_logging(level); }), - "The logging level to use (debug, info, warning, error)"); + "The logging level to use (debug, info, warning, error)")( + "start_sample", + po::value() + ->default_value(config.start_sample()) + ->notifier([&config](std::size_t key) { config.start_sample(key); }), + "Start from this sample")("nsamples_to_read", + po::value() + ->default_value(config.start_sample()) + ->notifier([&config](std::size_t key) { config.nsamples_to_read(key); }), + "total number of samples to read from start_sample"); po::options_description cmdline_options; cmdline_options.add(generic).add(main_options); @@ -187,6 +196,7 @@ int main(int argc, char** argv) } } po::notify(variable_map); + BOOST_LOG_NAMED_SCOPE("skycleaver_cli"); BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); @@ -194,13 +204,14 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); - BOOST_LOG_TRIVIAL(info) - << "nsamples_per_block: " << config.nsamples_per_block(); + BOOST_LOG_TRIVIAL(info) << "nsamples_per_block: " << config.nsamples_per_block(); BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); + BOOST_LOG_TRIVIAL(info) << "nbridges: " << config.nbridges(); + BOOST_LOG_TRIVIAL(info) << "ndms: " << config.ndms(); BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); - BOOST_LOG_TRIVIAL(info) - << "max_output_filesize: " << config.max_output_filesize(); + BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); + skyweaver::SkyCleaver skycleaver(config); skycleaver.cleave(); From 48adc14b91afa565369e9419abef356f202c0630 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 2 Oct 2024 15:24:25 +0200 Subject: [PATCH 23/65] cleave only required dms and beams, code review changes --- cmake/compiler_settings.cmake | 4 +- cpp/skyweaver/MultiFileWriter.cuh | 18 ----- cpp/skyweaver/SkyCleaverConfig.hpp | 14 +++- cpp/skyweaver/src/SkyCleaver.cu | 55 +++++++++++++-- cpp/skyweaver/src/skycleaver_cli.cu | 106 +++++++++++++++++++++++++--- 5 files changed, 160 insertions(+), 37 deletions(-) diff --git a/cmake/compiler_settings.cmake b/cmake/compiler_settings.cmake index 879cf70..500e4f0 100644 --- a/cmake/compiler_settings.cmake +++ b/cmake/compiler_settings.cmake @@ -6,9 +6,9 @@ if (NOT CMAKE_BUILD_TYPE) endif () #set(CMAKE_VERBOSE_MAKEFILE 1) - +set (CMAKE_CXX_STANDARD 20) # Set compiler flags -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fopenmp -std=gnu++20") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fopenmp") set(ARCH "broadwell" CACHE STRING "target architecture (-march=native, x86-64), defautls to broadwell") diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 352b2f5..6533d6a 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -37,24 +37,6 @@ struct MultiFileWriterConfig { stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), output_basename("") {}; - MultiFileWriterConfig(MultiFileWriterConfig const& other) - : header_size(other.header_size), max_file_size(other.max_file_size), - stokes_mode(other.stokes_mode), output_dir(other.output_dir), - base_output_dir(other.base_output_dir), prefix(other.prefix), - extension(other.extension), output_basename(other.output_basename) {}; - - MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other) - { - header_size = other.header_size; - max_file_size = other.max_file_size; - stokes_mode = other.stokes_mode; - output_dir = other.output_dir; - prefix = other.prefix; - extension = other.extension; - output_basename = other.output_basename; - base_output_dir = other.base_output_dir; - return *this; - } std::string to_string() { diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index 1972812..8b77dcc 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -22,6 +22,9 @@ class SkyCleaverConfig std::size_t _dada_header_size; std::size_t _start_sample; std::size_t _nsamples_to_read; + std::vector _required_beams; + std::vector _required_dms; + public: SkyCleaverConfig() @@ -29,7 +32,7 @@ class SkyCleaverConfig _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096), - _start_sample(0), _nsamples_to_read(0) + _start_sample(0), _nsamples_to_read(0), _required_beams({}), _required_dms({}) { } SkyCleaverConfig(SkyCleaverConfig const&) = delete; @@ -63,6 +66,11 @@ class SkyCleaverConfig { _nsamples_to_read = nsamples_to_read; } + void required_beams(std::vector required_beams) { _required_beams = required_beams; } + void required_dms(std::vector required_dms) { _required_dms = required_dms; } + + + std::string output_dir() const { return _output_dir; } std::string root_dir() const { return _root_dir; } @@ -81,6 +89,10 @@ class SkyCleaverConfig std::size_t dada_header_size() const { return _dada_header_size; } std::size_t start_sample() const { return _start_sample; } std::size_t nsamples_to_read() const { return _nsamples_to_read; } + std::vector required_beams() const { return _required_beams; } + std::vector required_dms() const { return _required_dms; } + + }; } // namespace skyweaver diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index b645414..6655ff1 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "skyweaver/types.cuh" #include "skyweaver/SkyCleaver.cuh" @@ -246,13 +247,19 @@ void SkyCleaver::init_readers() "Smallest data size is less than nsamples_per_block"); } - if(smallest_nsamples < _config.nsamples_to_read()) { - std::runtime_error( - "Smallest data size is less than nsamples_to_read"); - } - + if(_config.nsamples_to_read() > 0) { + + if(smallest_nsamples < _config.nsamples_to_read()) { + std::runtime_error( + "Smallest data size is less than nsamples_to_read"); + } + - _nsamples_to_read = _config.nsamples_to_read(); + _nsamples_to_read = _config.nsamples_to_read(); + } + else{ + _nsamples_to_read = smallest_nsamples; + } std::size_t bytes_seeking = (_config.start_sample() * _config.ndms() * _config.nbeams() * sizeof(InputVectorType::value_type)); for(const auto& [freq, reader]: _bridge_readers) { @@ -275,6 +282,8 @@ void SkyCleaver::init_writers() BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_writers") BOOST_LOG_TRIVIAL(debug) << "_config.output_dir(); " << _config.output_dir(); + + if(!fs::exists(_config.output_dir())) { fs::create_directories(_config.output_dir()); } @@ -282,6 +291,7 @@ void SkyCleaver::init_writers() ? "" : _config.out_prefix() + "_"; std::string output_dir = _config.output_dir(); + for(int idm = 0; idm < _config.ndms(); idm++) { @@ -289,8 +299,26 @@ void SkyCleaver::init_writers() std::string prefix = _config.ndms() > 1 ? out_prefix + "idm_" + to_string_with_padding(_header.dms[idm], 9, 3) + "_": out_prefix; + if(! _config.required_dms().empty()) { + const auto& required_dms = _config.required_dms(); + if(std::ranges::find(required_dms, _header.dms[idm]) == required_dms.end()) { + BOOST_LOG_TRIVIAL(info) + << "DM " << _header.dms[idm] << " is not required, skipping from writing"; + continue; + } + } for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + // skip if beam is not used + if (! _config.required_beams().empty()) { + const auto& required_beams = _config.required_beams(); + std::cerr << "required_beams: " << required_beams.size() << "ibeam: " << ibeam << std::endl; + if(std::ranges::find(required_beams, ibeam) == required_beams.end()) { + BOOST_LOG_TRIVIAL(info) + << "Beam " << ibeam << " is not required, skipping from writing"; + continue; + } + } MultiFileWriterConfig writer_config; writer_config.header_size = _config.dada_header_size(); writer_config.max_file_size = _config.max_output_filesize(); @@ -412,6 +440,14 @@ void SkyCleaver::cleave() #pragma omp parallel for schedule(static) collapse(2) for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { for(std::size_t idm = 0; idm < ndms; idm++) { + // _beam_data[idm][ibeam] is not found, skip + if(_beam_data.find(idm) == _beam_data.end()) { + continue; + } + if(_beam_data[idm].find(ibeam) == _beam_data[idm].end()) { + continue; + } + #pragma omp simd for(std::size_t isample = 0; isample < nsamples_per_block; isample++) { const std::size_t base_index = @@ -436,6 +472,13 @@ void SkyCleaver::cleave() #pragma omp parallel for schedule(static) collapse(2) for(int idm = 0; idm < _config.ndms(); idm++) { for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + + if(_beam_data.find(idm) == _beam_data.end()) { + continue; + } + if(_beam_data[idm].find(ibeam) == _beam_data[idm].end()) { + continue; + } _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], _config.stream_id()); } diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cu index d8fc2bd..0b56bac 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cu +++ b/cpp/skyweaver/src/skycleaver_cli.cu @@ -19,6 +19,10 @@ #include #include #include +#include +#include +#include + #define BOOST_LOG_DYN_LINK 1 namespace @@ -38,6 +42,53 @@ const size_t ERROR_UNHANDLED_EXCEPTION = 2; const char* build_time = __DATE__ " " __TIME__; + +template +std::vector +get_list_from_string(const std::string& value, + T epsilon = std::numeric_limits::epsilon()) +{ + std::vector output; + std::vector comma_chunks; + + // Split the input string by commas + std::stringstream ss(value); + std::string token; + while(std::getline(ss, token, ',')) { comma_chunks.push_back(token); } + + for(const auto& comma_chunk: comma_chunks) { + // Check if the chunk contains a colon (indicating a range) + if(comma_chunk.find(':') == std::string::npos) { + output.push_back(static_cast(std::atof(comma_chunk.c_str()))); + continue; + } + + // Split the range chunk by colons + std::stringstream ss_chunk(comma_chunk); + std::vector colon_chunks; + std::string colon_token; + while(std::getline(ss_chunk, colon_token, ':')) { + colon_chunks.push_back( + static_cast(std::atof(colon_token.c_str()))); + } + + // Determine the step size + T step = colon_chunks.size() == 3 ? colon_chunks[2] : static_cast(1); + T start = colon_chunks[0]; + T stop = colon_chunks[1]; + + // Loop and add values to the output vector + if constexpr(std::is_floating_point::value) { + for(T k = start; k <= stop + epsilon; k += step) { + output.push_back(k); + } + } else { + for(T k = start; k <= stop; k += step) { output.push_back(k); } + } + } + return output; +} + } // namespace namespace std @@ -149,14 +200,33 @@ int main(int argc, char** argv) [](std::string level) { skyweaver::init_logging(level); }), "The logging level to use (debug, info, warning, error)")( "start_sample", - po::value() - ->default_value(config.start_sample()) - ->notifier([&config](std::size_t key) { config.start_sample(key); }), - "Start from this sample")("nsamples_to_read", - po::value() - ->default_value(config.start_sample()) - ->notifier([&config](std::size_t key) { config.nsamples_to_read(key); }), - "total number of samples to read from start_sample"); + po::value() + ->default_value(config.start_sample()) + ->notifier( + [&config](std::size_t key) { config.start_sample(key); }), + "Start from this sample")( + "nsamples_to_read", + po::value() + ->default_value(config.start_sample()) + ->notifier( + [&config](std::size_t key) { config.nsamples_to_read(key); }), + "total number of samples to read from start_sample")( + "required_beams", + po::value() + ->default_value("") + ->notifier([&config](std::string key) { + config.required_beams(get_list_from_string(key)); + }), + "Comma separated list of beams to process. Syntax - beam1, beam2, " + "beam3:beam4:step, beam5 etc..")( + "required_dms", + po::value() + ->default_value("") + ->notifier([&config](std::string key) { + config.required_dms(get_list_from_string(key)); + }), + "Comma separated list of DMs to process. Syntax - dm1, dm2, " + "dm1:dm2:step, etc.."); po::options_description cmdline_options; cmdline_options.add(generic).add(main_options); @@ -204,13 +274,29 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); - BOOST_LOG_TRIVIAL(info) << "nsamples_per_block: " << config.nsamples_per_block(); + BOOST_LOG_TRIVIAL(info) + << "nsamples_per_block: " << config.nsamples_per_block(); BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); BOOST_LOG_TRIVIAL(info) << "nbridges: " << config.nbridges(); BOOST_LOG_TRIVIAL(info) << "ndms: " << config.ndms(); BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); - BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); + BOOST_LOG_TRIVIAL(info) + << "max_output_filesize: " << config.max_output_filesize(); + BOOST_LOG_TRIVIAL(info) << "dada_header_size: " << config.dada_header_size(); + BOOST_LOG_TRIVIAL(info) << "start_sample: " << config.start_sample(); + BOOST_LOG_TRIVIAL(info) << "nsamples_to_read: " << config.nsamples_to_read(); + if(config.required_beams().size() > 0) { + for (auto beam : config.required_beams()) { + BOOST_LOG_TRIVIAL(info) << "required_beam: " << beam; + } + } + if(config.required_dms().size() > 0) { + for (auto dm : config.required_dms()) { + BOOST_LOG_TRIVIAL(info) << "required_dm: " << dm; + } + } + skyweaver::SkyCleaver skycleaver(config); From 0cfe1f91b9348bbd073c21022a51dcf9b14f3cc5 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 3 Oct 2024 16:00:24 +0200 Subject: [PATCH 24/65] seek to start sample after dedisp delay correction --- cpp/skyweaver/src/SkyCleaver.cu | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index 6655ff1..4f73cce 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -261,13 +261,19 @@ void SkyCleaver::init_readers() _nsamples_to_read = smallest_nsamples; } - std::size_t bytes_seeking = (_config.start_sample() * _config.ndms() * _config.nbeams() * sizeof(InputVectorType::value_type)); - for(const auto& [freq, reader]: _bridge_readers) { - _bridge_readers[freq]->seekg(bytes_seeking, - std::ios_base::beg); + std::size_t bytes_seeking = + (_config.start_sample() * _config.ndms() * _config.nbeams() * + sizeof(InputVectorType::value_type)); + + if(bytes_seeking > 0) { + BOOST_LOG_TRIVIAL(info) << "Seeking " << bytes_seeking + << " bytes in bridge readers to start sample: " + << _config.start_sample(); + for(const auto& [freq, reader]: _bridge_readers) { + _bridge_readers[freq]->seekg(bytes_seeking, std::ios_base::cur); + } } - BOOST_LOG_TRIVIAL(info) << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; @@ -371,6 +377,8 @@ void SkyCleaver::cleave() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::cleave") + /** read data **/ + for(std::size_t nsamples_read = 0; nsamples_read < _nsamples_to_read; nsamples_read += _config.nsamples_per_block()) { std::size_t gulp_samples = @@ -430,6 +438,8 @@ void SkyCleaver::cleave() _timer.stop("skyweaver::read_data"); _timer.start("skyweaver::process_data"); + /** Process data **/ + omp_set_num_threads(_config.nthreads()); std::size_t nbridges = _config.nbridges(); @@ -467,12 +477,15 @@ void SkyCleaver::cleave() BOOST_LOG_TRIVIAL(info) << "Processed data"; _timer.stop("skyweaver::process_data"); + /** Write data **/ + + _timer.start("skyweaver::write_data"); #pragma omp parallel for schedule(static) collapse(2) for(int idm = 0; idm < _config.ndms(); idm++) { for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - + if(_beam_data.find(idm) == _beam_data.end()) { continue; } From ee35ae22cf5b982108cc11982b0cec28773e4252 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 3 Oct 2024 16:54:25 +0200 Subject: [PATCH 25/65] CDD all data including overlap --- cpp/skyweaver/src/CoherentDedisperser.cu | 118 ++++++++++++++++++----- 1 file changed, 92 insertions(+), 26 deletions(-) diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 12d9976..73c9f27 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -13,6 +13,49 @@ #include #include #include + + +namespace{ + +#define NCHANS_PER_BLOCK 128 + +// Function to check if a number's prime factors are only 2, 3, 5, or +bool has_small_prime_factors(size_t n) { + if (n == 0) return false; + while (n % 2 == 0) n /= 2; + while (n % 3 == 0) n /= 3; + while (n % 5 == 0) n /= 5; + while (n % 7 == 0) n /= 7; + return n == 1; +} + +// Function to find the next optimal FFT size +size_t next_optimal_fft_size(size_t N) { + size_t n = N; + while (!has_small_prime_factors(n)) { + n++; + } + return n; +} + +// Function to compute padding amount +size_t compute_padding(size_t N) { + size_t optimal_size = 0; + size_t n = N-1; + + do{ + n = n+1; + optimal_size = next_optimal_fft_size(n); + BOOST_LOG_TRIVIAL(info) << "Trying optimal size: " << optimal_size; + } while (optimal_size % NCHANS_PER_BLOCK != 0); + return optimal_size - N; +} + + + +} + + namespace skyweaver { @@ -45,7 +88,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, "Gulp length must be at least 2 times the maximum DM delay"); } - if(max_dm_delay_samps %2 !=0) { + if((pipeline_config.gulp_length_samps() + max_dm_delay_samps) %2 !=0) { max_dm_delay_samps++; } @@ -78,7 +121,6 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, std::vector dms) { config.gulp_samps = gulp_samps; - config.overlap_samps = overlap_samps; config.num_coarse_chans = num_coarse_chans; config.npols = npols; config.nantennas = nantennas; @@ -88,9 +130,13 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, config.high_freq = low_freq + bw; config.coarse_chan_bw = bw / num_coarse_chans; config.filter_delay = tsamp * overlap_samps / 2.0; - BOOST_LOG_TRIVIAL(warning) << "tsamp in create_coherent_dedisperser_config: " << config.tsamp; - BOOST_LOG_TRIVIAL(warning) << "overlap_samps in create_coherent_dedisperser_config: " << config.overlap_samps; - BOOST_LOG_TRIVIAL(warning) << "Filter delay: " << config.filter_delay; + BOOST_LOG_TRIVIAL(debug) << "tsamp in create_coherent_dedisperser_config: " << config.tsamp; + BOOST_LOG_TRIVIAL(info) << "overlap_samps just due to dedispersion: " << overlap_samps; + BOOST_LOG_TRIVIAL(info) << "Filter delay: " << config.filter_delay; + + config.overlap_samps = compute_padding(gulp_samps + overlap_samps) + overlap_samps; + + BOOST_LOG_TRIVIAL(info) << "overlap_samps after padding for optimal FFT size " << config.overlap_samps; /* Precompute DM constants */ @@ -99,7 +145,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, config._d_dm_prefactor.resize(dms.size()); config._d_ism_responses.resize(dms.size()); for(int i = 0; i < dms.size(); i++) { - config._d_ism_responses[i].resize(num_coarse_chans * gulp_samps); + config._d_ism_responses[i].resize(num_coarse_chans * (config.gulp_samps + config.overlap_samps)); } thrust::transform(config._d_dms.begin(), @@ -147,10 +193,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, } -namespace -{ -#define NCHANS_PER_BLOCK 128 -} // namespace + void CoherentDedisperser::dedisperse( TPAVoltagesD const& d_tpa_voltages_in, @@ -159,7 +202,8 @@ void CoherentDedisperser::dedisperse( unsigned int dm_idx) { BOOST_LOG_NAMED_SCOPE("CoherentDedisperser::dedisperse"); - _d_fpa_spectra.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); + //d_tpa_voltages_in.size() is with overlap + _d_fpa_spectra.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); _d_tpa_voltages_in_cufft.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); _d_tpa_voltages_dedispersed.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); @@ -216,6 +260,10 @@ void CoherentDedisperser::dedisperse( BOOST_LOG_TRIVIAL(debug) << "copying from input from " << discard_size << " to " << _d_tpa_voltages_dedispersed.size() - discard_size; + BOOST_LOG_TRIVIAL(debug) << "Total elements copied: " + << _d_tpa_voltages_dedispersed.size() - 2 * discard_size; + BOOST_LOG_TRIVIAL(debug) << "Remaining space in output: " + << d_ftpa_voltages_out.size() - out_offset; BOOST_LOG_TRIVIAL(debug) << "copying to output from " << out_offset << " to " << out_offset + _d_tpa_voltages_dedispersed.size() - 2 * discard_size; @@ -223,8 +271,8 @@ void CoherentDedisperser::dedisperse( std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; + - // transform: divide by d_tpa_voltages_in.size() thrust::transform(_d_tpa_voltages_dedispersed.begin() + discard_size, _d_tpa_voltages_dedispersed.end() - discard_size, d_ftpa_voltages_out.begin() + out_offset, @@ -236,6 +284,8 @@ void CoherentDedisperser::dedisperse( static_cast(__float2int_rn(val.y / fft_size)); return char2_val; }); + + BOOST_LOG_TRIVIAL(debug) << "Transformed dedispersed voltages to char2"; d_ftpa_voltages_out.reference_dm(_config._h_dms[dm_idx]); } @@ -246,15 +296,22 @@ void CoherentDedisperser::multiply_by_chirp( unsigned int freq_idx, unsigned int dm_idx) { - std::size_t total_chans = _config._d_ism_responses[dm_idx].size(); - std::size_t response_offset = freq_idx * _config.gulp_samps; + std::size_t total_chans = _config._d_ism_responses[dm_idx].size(); // all coarse + fine chans + std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; // ONLY FINE CHANS + std::size_t response_offset = freq_idx * fft_size; BOOST_LOG_TRIVIAL(debug) << "Freq idx: " << freq_idx; BOOST_LOG_TRIVIAL(debug) << "_config.gulp_samps: " << _config.gulp_samps; + BOOST_LOG_TRIVIAL(debug) << "_config.overlap_samps: " << _config.overlap_samps; BOOST_LOG_TRIVIAL(debug) << "response_offset: " << response_offset; - - dim3 blockSize(_config.nantennas * _config.npols); - dim3 gridSize(_config.gulp_samps / NCHANS_PER_BLOCK); + BOOST_LOG_TRIVIAL(debug) << "total_chans: " << total_chans; + BOOST_LOG_TRIVIAL(debug) << "chirp multiply input size: " + << _d_fpa_spectra_in.size(); + BOOST_LOG_TRIVIAL(debug) << "chirp multiply output size: " + << _d_fpa_spectra_out.size(); + + dim3 blockSize(_config.nantennas * _config.npols); + dim3 gridSize(fft_size / NCHANS_PER_BLOCK); kernels::dedisperse<<>>( thrust::raw_pointer_cast(_config._d_ism_responses[dm_idx].data() + response_offset), @@ -273,14 +330,14 @@ __global__ void dedisperse(cufftComplex const* __restrict__ _d_ism_response, cufftComplex* out, unsigned total_chans) { - const unsigned pa_size = blockDim.x; + const unsigned pa_size = blockDim.x; // NANT * NPOL volatile __shared__ cufftComplex response[NCHANS_PER_BLOCK]; - const unsigned block_start_chan_idx = blockIdx.x * NCHANS_PER_BLOCK; + const unsigned block_start_chan_idx = blockIdx.x * NCHANS_PER_BLOCK; // coarse chan idx const unsigned remainder = - min(total_chans - block_start_chan_idx, NCHANS_PER_BLOCK); + min(total_chans - block_start_chan_idx, NCHANS_PER_BLOCK); // how many channels to process for(int idx = threadIdx.x; idx < remainder; idx += pa_size) { cufftComplex const temp = _d_ism_response[block_start_chan_idx + idx]; @@ -331,7 +388,7 @@ struct DMResponse { int fine_chan = tid % num_fine_chans; // fine channel double nu_0 = low_freq + chan * coarse_chan_bw - - 0.5f * coarse_chan_bw; // + fine_chan * fine_chan_bw; + 0.5f * coarse_chan_bw; double nu = fine_chan * fine_chan_bw; // fine_chan_freq @@ -341,7 +398,7 @@ struct DMResponse { cufftDoubleComplex weight; sincos(phase, &weight.y, - &weight.x); // TO DO: test if it is not approximate + &weight.x); // TODO: test if it is not approximate cufftComplex float_weight; float_weight.x = static_cast(weight.x); float_weight.y = static_cast(weight.y); @@ -357,20 +414,29 @@ void get_dm_responses(CoherentDedisperserConfig& config, thrust::device_vector& response) { BOOST_LOG_TRIVIAL(debug) << "Generating DM responses"; - thrust::device_vector indices(config.num_coarse_chans * - config.gulp_samps); - thrust::sequence(indices.begin(), indices.end()); + std::size_t fft_size = config.gulp_samps + config.overlap_samps; + thrust::device_vector indices(config.num_coarse_chans * fft_size); + thrust::sequence(indices.begin(), indices.end()); + + // store raw responses in a temporary variable + thrust::device_vector temp_response(response.size()); + BOOST_LOG_TRIVIAL(warning) << "DOING FFTSHIFT" << std::endl; // Apply the DMResponse functor using thrust's transform thrust::transform(indices.begin(), indices.end(), - response.begin(), + temp_response.begin(), kernels::DMResponse(config.num_coarse_chans, config.gulp_samps, config.low_freq, config.coarse_chan_bw, config.fine_chan_bw, dm_prefactor)); + + // rotate the response to match cufft output + std::size_t shift = fft_size / 2; + thrust::copy(temp_response.begin() + shift, temp_response.end(), response.begin()); + thrust::copy(temp_response.begin(), temp_response.begin() + shift, response.begin() + (fft_size - shift)); } } // namespace skyweaver From 52b1ab31178d821807f105c185df904b5f555439 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Mon, 7 Oct 2024 17:05:19 +0200 Subject: [PATCH 26/65] fix FFT shift, change left over gulps -> fft_size where appropriate --- cpp/skyweaver/src/CoherentDedisperser.cu | 202 ++++++++++++----------- 1 file changed, 108 insertions(+), 94 deletions(-) diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 73c9f27..79fc4be 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -14,47 +14,46 @@ #include #include - -namespace{ +namespace +{ #define NCHANS_PER_BLOCK 128 -// Function to check if a number's prime factors are only 2, 3, 5, or -bool has_small_prime_factors(size_t n) { - if (n == 0) return false; - while (n % 2 == 0) n /= 2; - while (n % 3 == 0) n /= 3; - while (n % 5 == 0) n /= 5; - while (n % 7 == 0) n /= 7; +// Function to check if a number's prime factors are only 2, 3, 5, or +bool has_small_prime_factors(size_t n) +{ + if(n == 0) + return false; + while(n % 2 == 0) n /= 2; + while(n % 3 == 0) n /= 3; + while(n % 5 == 0) n /= 5; + while(n % 7 == 0) n /= 7; return n == 1; } // Function to find the next optimal FFT size -size_t next_optimal_fft_size(size_t N) { +size_t next_optimal_fft_size(size_t N) +{ size_t n = N; - while (!has_small_prime_factors(n)) { - n++; - } + while(!has_small_prime_factors(n)) { n++; } return n; } // Function to compute padding amount -size_t compute_padding(size_t N) { +size_t compute_padding(size_t N) +{ size_t optimal_size = 0; - size_t n = N-1; - - do{ - n = n+1; + size_t n = N - 1; + + do { + n = n + 1; optimal_size = next_optimal_fft_size(n); BOOST_LOG_TRIVIAL(info) << "Trying optimal size: " << optimal_size; - } while (optimal_size % NCHANS_PER_BLOCK != 0); + } while(optimal_size % NCHANS_PER_BLOCK != 0); return optimal_size - N; } - - -} - +} // namespace namespace skyweaver { @@ -62,16 +61,16 @@ namespace skyweaver void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, PipelineConfig const& pipeline_config) { - // the centre frequency and bandwidth are for the bridge. This is taken from Observation Header (not from the user) + // the centre frequency and bandwidth are for the bridge. This is taken from + // Observation Header (not from the user) float f_low = pipeline_config.centre_frequency() - pipeline_config.bandwidth() / 2.0f; - float f_high = f_low + pipeline_config.bandwidth()/pipeline_config.nchans(); - - // pipeline_config.centre_frequency() + pipeline_config.bandwidth() / 2.0f; - float tsamp = pipeline_config.nchans() / pipeline_config.bandwidth(); + float f_high = + f_low + pipeline_config.bandwidth() / pipeline_config.nchans(); - + // pipeline_config.centre_frequency() + pipeline_config.bandwidth() / 2.0f; + float tsamp = pipeline_config.nchans() / pipeline_config.bandwidth(); if(pipeline_config.coherent_dms().empty()) { throw std::runtime_error("No coherent DMs specified"); @@ -81,14 +80,15 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, pipeline_config.coherent_dms().end()); float max_dm = *it; BOOST_LOG_TRIVIAL(debug) << "Constructing coherent dedisperser plan"; - std::size_t max_dm_delay_samps = DMSampleDelay(max_dm, f_low, tsamp)(f_high); + std::size_t max_dm_delay_samps = + DMSampleDelay(max_dm, f_low, tsamp)(f_high); if(max_dm_delay_samps > 2 * pipeline_config.gulp_length_samps()) { throw std::runtime_error( "Gulp length must be at least 2 times the maximum DM delay"); } - if((pipeline_config.gulp_length_samps() + max_dm_delay_samps) %2 !=0) { + if((pipeline_config.gulp_length_samps() + max_dm_delay_samps) % 2 != 0) { max_dm_delay_samps++; } @@ -125,19 +125,23 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, config.npols = npols; config.nantennas = nantennas; config.tsamp = tsamp; - config.low_freq = low_freq; - config.bw = bw; - config.high_freq = low_freq + bw; - config.coarse_chan_bw = bw / num_coarse_chans; - config.filter_delay = tsamp * overlap_samps / 2.0; - BOOST_LOG_TRIVIAL(debug) << "tsamp in create_coherent_dedisperser_config: " << config.tsamp; - BOOST_LOG_TRIVIAL(info) << "overlap_samps just due to dedispersion: " << overlap_samps; + config.low_freq = low_freq; + config.bw = bw; + config.high_freq = low_freq + bw; + config.coarse_chan_bw = bw / num_coarse_chans; + config.filter_delay = tsamp * overlap_samps / 2.0; + BOOST_LOG_TRIVIAL(debug) + << "tsamp in create_coherent_dedisperser_config: " << config.tsamp; + BOOST_LOG_TRIVIAL(info) + << "overlap_samps just due to dedispersion: " << overlap_samps; BOOST_LOG_TRIVIAL(info) << "Filter delay: " << config.filter_delay; - config.overlap_samps = compute_padding(gulp_samps + overlap_samps) + overlap_samps; - - BOOST_LOG_TRIVIAL(info) << "overlap_samps after padding for optimal FFT size " << config.overlap_samps; + config.overlap_samps = + compute_padding(gulp_samps + overlap_samps) + overlap_samps; + BOOST_LOG_TRIVIAL(info) + << "overlap_samps after padding for optimal FFT size " + << config.overlap_samps; /* Precompute DM constants */ config._h_dms = dms; @@ -145,7 +149,8 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, config._d_dm_prefactor.resize(dms.size()); config._d_ism_responses.resize(dms.size()); for(int i = 0; i < dms.size(); i++) { - config._d_ism_responses[i].resize(num_coarse_chans * (config.gulp_samps + config.overlap_samps)); + config._d_ism_responses[i].resize( + num_coarse_chans * (config.gulp_samps + config.overlap_samps)); } thrust::transform(config._d_dms.begin(), @@ -164,16 +169,16 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, // data is FTPA order, we will loop over F, so we are left with TPA order. // Let's fuse PA to X, so TX order. // We stride and batch over X and transform T - std::size_t X = config.npols * config.nantennas; - std::size_t fft_size = config.gulp_samps + config.overlap_samps; - int n[1] = {static_cast(fft_size)}; // FFT size - int inembed[1] = {static_cast(fft_size)}; - int onembed[1] = {static_cast(fft_size)}; - int istride = X; - int ostride = X; - int idist = 1; - int odist = 1; - int batch = X; + std::size_t X = config.npols * config.nantennas; + std::size_t fft_size = config.gulp_samps + config.overlap_samps; + int n[1] = {static_cast(fft_size)}; // FFT size + int inembed[1] = {static_cast(fft_size)}; + int onembed[1] = {static_cast(fft_size)}; + int istride = X; + int ostride = X; + int idist = 1; + int odist = 1; + int batch = X; if(cufftPlanMany(&config._fft_plan, 1, @@ -192,9 +197,6 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, BOOST_LOG_TRIVIAL(debug) << "FFT plan created"; } - - - void CoherentDedisperser::dedisperse( TPAVoltagesD const& d_tpa_voltages_in, FTPAVoltagesD& d_ftpa_voltages_out, @@ -202,8 +204,8 @@ void CoherentDedisperser::dedisperse( unsigned int dm_idx) { BOOST_LOG_NAMED_SCOPE("CoherentDedisperser::dedisperse"); - //d_tpa_voltages_in.size() is with overlap - _d_fpa_spectra.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); + // d_tpa_voltages_in.size() is with overlap + _d_fpa_spectra.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); _d_tpa_voltages_in_cufft.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); _d_tpa_voltages_dedispersed.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); @@ -250,8 +252,8 @@ void CoherentDedisperser::dedisperse( BOOST_LOG_TRIVIAL(debug) << "Executed inverse FFT"; - std::size_t out_offset = freq_idx * _config.nantennas * _config.npols * - (_config.gulp_samps); + std::size_t out_offset = + freq_idx * _config.nantennas * _config.npols * (_config.gulp_samps); std::size_t discard_size = _config.nantennas * _config.npols * _config.overlap_samps / 2; @@ -260,30 +262,28 @@ void CoherentDedisperser::dedisperse( BOOST_LOG_TRIVIAL(debug) << "copying from input from " << discard_size << " to " << _d_tpa_voltages_dedispersed.size() - discard_size; - BOOST_LOG_TRIVIAL(debug) << "Total elements copied: " - << _d_tpa_voltages_dedispersed.size() - 2 * discard_size; + BOOST_LOG_TRIVIAL(debug) + << "Total elements copied: " + << _d_tpa_voltages_dedispersed.size() - 2 * discard_size; BOOST_LOG_TRIVIAL(debug) << "Remaining space in output: " << d_ftpa_voltages_out.size() - out_offset; BOOST_LOG_TRIVIAL(debug) << "copying to output from " << out_offset << " to " << out_offset + _d_tpa_voltages_dedispersed.size() - 2 * discard_size; + std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; - std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; - - - - thrust::transform(_d_tpa_voltages_dedispersed.begin() + discard_size, - _d_tpa_voltages_dedispersed.end() - discard_size, - d_ftpa_voltages_out.begin() + out_offset, - [=] __device__(cufftComplex const& val) { - char2 char2_val; - char2_val.x = static_cast( - __float2int_rn(val.x / fft_size)); // scale the data back - char2_val.y = - static_cast(__float2int_rn(val.y / fft_size)); - return char2_val; - }); + thrust::transform( + _d_tpa_voltages_dedispersed.begin() + discard_size, + _d_tpa_voltages_dedispersed.end() - discard_size, + d_ftpa_voltages_out.begin() + out_offset, + [=] __device__(cufftComplex const& val) { + char2 char2_val; + char2_val.x = static_cast( + __float2int_rn(val.x / fft_size)); // scale the data back + char2_val.y = static_cast(__float2int_rn(val.y / fft_size)); + return char2_val; + }); BOOST_LOG_TRIVIAL(debug) << "Transformed dedispersed voltages to char2"; d_ftpa_voltages_out.reference_dm(_config._h_dms[dm_idx]); @@ -296,21 +296,24 @@ void CoherentDedisperser::multiply_by_chirp( unsigned int freq_idx, unsigned int dm_idx) { - std::size_t total_chans = _config._d_ism_responses[dm_idx].size(); // all coarse + fine chans - std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; // ONLY FINE CHANS + std::size_t total_chans = + _config._d_ism_responses[dm_idx].size(); // all coarse + fine chans + std::size_t fft_size = + _config.gulp_samps + _config.overlap_samps; // ONLY FINE CHANS std::size_t response_offset = freq_idx * fft_size; BOOST_LOG_TRIVIAL(debug) << "Freq idx: " << freq_idx; BOOST_LOG_TRIVIAL(debug) << "_config.gulp_samps: " << _config.gulp_samps; - BOOST_LOG_TRIVIAL(debug) << "_config.overlap_samps: " << _config.overlap_samps; + BOOST_LOG_TRIVIAL(debug) + << "_config.overlap_samps: " << _config.overlap_samps; BOOST_LOG_TRIVIAL(debug) << "response_offset: " << response_offset; BOOST_LOG_TRIVIAL(debug) << "total_chans: " << total_chans; - BOOST_LOG_TRIVIAL(debug) << "chirp multiply input size: " - << _d_fpa_spectra_in.size(); - BOOST_LOG_TRIVIAL(debug) << "chirp multiply output size: " - << _d_fpa_spectra_out.size(); + BOOST_LOG_TRIVIAL(debug) + << "chirp multiply input size: " << _d_fpa_spectra_in.size(); + BOOST_LOG_TRIVIAL(debug) + << "chirp multiply output size: " << _d_fpa_spectra_out.size(); - dim3 blockSize(_config.nantennas * _config.npols); + dim3 blockSize(_config.nantennas * _config.npols); dim3 gridSize(fft_size / NCHANS_PER_BLOCK); kernels::dedisperse<<>>( thrust::raw_pointer_cast(_config._d_ism_responses[dm_idx].data() + @@ -334,10 +337,12 @@ __global__ void dedisperse(cufftComplex const* __restrict__ _d_ism_response, volatile __shared__ cufftComplex response[NCHANS_PER_BLOCK]; - const unsigned block_start_chan_idx = blockIdx.x * NCHANS_PER_BLOCK; // coarse chan idx + const unsigned block_start_chan_idx = + blockIdx.x * NCHANS_PER_BLOCK; // coarse chan idx const unsigned remainder = - min(total_chans - block_start_chan_idx, NCHANS_PER_BLOCK); // how many channels to process + min(total_chans - block_start_chan_idx, + NCHANS_PER_BLOCK); // how many channels to process for(int idx = threadIdx.x; idx < remainder; idx += pa_size) { cufftComplex const temp = _d_ism_response[block_start_chan_idx + idx]; @@ -387,8 +392,7 @@ struct DMResponse { int chan = tid / num_fine_chans; // Coarse channel int fine_chan = tid % num_fine_chans; // fine channel - double nu_0 = low_freq + chan * coarse_chan_bw - - 0.5f * coarse_chan_bw; + double nu_0 = low_freq + chan * coarse_chan_bw - 0.5f * coarse_chan_bw; double nu = fine_chan * fine_chan_bw; // fine_chan_freq @@ -414,11 +418,11 @@ void get_dm_responses(CoherentDedisperserConfig& config, thrust::device_vector& response) { BOOST_LOG_TRIVIAL(debug) << "Generating DM responses"; - std::size_t fft_size = config.gulp_samps + config.overlap_samps; + std::size_t fft_size = config.gulp_samps + config.overlap_samps; thrust::device_vector indices(config.num_coarse_chans * fft_size); thrust::sequence(indices.begin(), indices.end()); - + // store raw responses in a temporary variable thrust::device_vector temp_response(response.size()); BOOST_LOG_TRIVIAL(warning) << "DOING FFTSHIFT" << std::endl; @@ -427,16 +431,26 @@ void get_dm_responses(CoherentDedisperserConfig& config, indices.end(), temp_response.begin(), kernels::DMResponse(config.num_coarse_chans, - config.gulp_samps, + fft_size, config.low_freq, config.coarse_chan_bw, config.fine_chan_bw, dm_prefactor)); - // rotate the response to match cufft output + // rotate the response to match cufft output + + if(fft_size %2 != 0) { + throw std::runtime_error("FFT size must be even."); + } + std::size_t shift = fft_size / 2; - thrust::copy(temp_response.begin() + shift, temp_response.end(), response.begin()); - thrust::copy(temp_response.begin(), temp_response.begin() + shift, response.begin() + (fft_size - shift)); + + for(auto it = temp_response.begin(), it_out = response.begin(); + it < temp_response.end(); + it += fft_size, it_out += fft_size) { + thrust::copy(it, it + shift, it_out + shift); + thrust::copy(it + shift, it + fft_size, it_out); + } } } // namespace skyweaver From 4bf4df14edd3d25f975674a5f1afa343bf3cd3c7 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Fri, 11 Oct 2024 17:40:50 +0200 Subject: [PATCH 27/65] Make skycleaver a CPP application --- cpp/skyweaver/CMakeLists.txt | 2 +- cpp/skyweaver/SkyCleaver.cuh | 81 ------------------- cpp/skyweaver/SkyCleaver.hpp | 68 ++++++++++++---- .../src/{SkyCleaver.cu => SkyCleaver.cpp} | 4 +- .../{skycleaver_cli.cu => skycleaver_cli.cpp} | 0 5 files changed, 57 insertions(+), 98 deletions(-) delete mode 100644 cpp/skyweaver/SkyCleaver.cuh rename cpp/skyweaver/src/{SkyCleaver.cu => SkyCleaver.cpp} (99%) rename cpp/skyweaver/src/{skycleaver_cli.cu => skycleaver_cli.cpp} (100%) diff --git a/cpp/skyweaver/CMakeLists.txt b/cpp/skyweaver/CMakeLists.txt index 0f49287..76d6ae6 100644 --- a/cpp/skyweaver/CMakeLists.txt +++ b/cpp/skyweaver/CMakeLists.txt @@ -111,7 +111,7 @@ target_link_libraries(skyweavercpp OpenMP::OpenMP_CXX) install(TARGETS skyweavercpp DESTINATION bin) -cuda_add_executable(skycleaver src/skycleaver_cli.cu) +add_executable(skycleaver src/skycleaver_cli.cpp) target_link_libraries(skycleaver ${SKYWEAVER_LIBRARIES} ${DEPENDENCY_LIBRARIES} diff --git a/cpp/skyweaver/SkyCleaver.cuh b/cpp/skyweaver/SkyCleaver.cuh deleted file mode 100644 index 4c7be58..0000000 --- a/cpp/skyweaver/SkyCleaver.cuh +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef SKYWEAVER_SKYCLEAVER_HPP -#define SKYWEAVER_SKYCLEAVER_HPP -#include "boost/log/trivial.hpp" -#include "psrdada_cpp/psrdadaheader.hpp" -#include "psrdada_cpp/raw_bytes.hpp" -#include "skyweaver/DescribedVector.hpp" -#include "skyweaver/MultiFileReader.cuh" -#include "skyweaver/MultiFileWriter.cuh" -#include "skyweaver/ObservationHeader.hpp" -#include "skyweaver/SkyCleaverConfig.hpp" -#include "skyweaver/Timer.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace skyweaver -{ - -struct BridgeReader { - public: - std::vector _tdb_filenames; - std::unique_ptr _tdb_reader; - std::string freq; - -}; // BridgeReader - - -class SkyCleaver -{ - public: - using InputType = int8_t; - using OutputType = uint8_t; - using InputVectorType = TDBPowersH; - using OutputVectorType = TFPowersH; - using FreqType = std::size_t; // up to the nearest Hz - using BeamNumberType = std::size_t; - using DMNumberType = std::size_t; - - private: - SkyCleaverConfig const& _config; - std::map> _bridge_readers; - std::map> _bridge_data; - - std::vector _expected_freqs; - std::vector _available_freqs; - std::size_t _nsamples_to_read; - ObservationHeader _header; - - std::vector _beam_filenames; - std::map>>> - _beam_writers; - std::map>> - _beam_data; - std::size_t _total_beam_writers; - - void init_readers(); - void init_writers(); - - Timer _timer; - - public: - SkyCleaver(SkyCleaverConfig const& config); - SkyCleaver(SkyCleaver const&) = delete; - void operator=(SkyCleaver const&) = delete; - - void cleave(); - -}; // class SkyCleaver - -} // namespace skyweaver - -#endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaver.hpp b/cpp/skyweaver/SkyCleaver.hpp index c22ae3d..46449de 100644 --- a/cpp/skyweaver/SkyCleaver.hpp +++ b/cpp/skyweaver/SkyCleaver.hpp @@ -1,8 +1,14 @@ -#include "SigprocFileWriter.hpp" +#ifndef SKYWEAVER_SKYCLEAVER_HPP +#define SKYWEAVER_SKYCLEAVER_HPP #include "boost/log/trivial.hpp" #include "psrdada_cpp/psrdadaheader.hpp" #include "psrdada_cpp/raw_bytes.hpp" +#include "skyweaver/DescribedVector.hpp" +#include "skyweaver/MultiFileReader.cuh" +#include "skyweaver/MultiFileWriter.cuh" #include "skyweaver/ObservationHeader.hpp" +#include "skyweaver/SkyCleaverConfig.hpp" +#include "skyweaver/Timer.hpp" #include #include @@ -12,32 +18,64 @@ #include #include #include + namespace skyweaver { -struct Bridge -{ - private: - using PowerType = std::vector; +struct BridgeReader { public: std::vector _tdb_filenames; + std::unique_ptr _tdb_reader; std::string freq; -}; // class Bridge +}; // BridgeReader -class MultiBeamWriter + +class SkyCleaver { + public: + using InputType = int8_t; + using OutputType = uint8_t; + using InputVectorType = TDBPowersStdH; + using OutputVectorType = TFPowersStdH; + using FreqType = std::size_t; // up to the nearest Hz + using BeamNumberType = std::size_t; + using DMNumberType = std::size_t; + private: - using FreqType = unsigned int; // up to the nearest Hz - std::map> _bridges; + SkyCleaverConfig const& _config; + std::map> _bridge_readers; + std::map> _bridge_data; + + std::vector _expected_freqs; + std::vector _available_freqs; + std::size_t _nsamples_to_read; + ObservationHeader _header; + std::vector _beam_filenames; + std::map>>> + _beam_writers; + std::map>> + _beam_data; + std::size_t _total_beam_writers; + + void init_readers(); + void init_writers(); + + Timer _timer; + + public: + SkyCleaver(SkyCleaverConfig const& config); + SkyCleaver(SkyCleaver const&) = delete; + void operator=(SkyCleaver const&) = delete; + void cleave(); - public - : add_bridge(FreqType freq, std::vector tdb_filenames); - remove_bridge(FreqType freq); +}; // class SkyCleaver - init(); +} // namespace skyweaver -} // class MultiFileWriter -} // namespace skyweaver \ No newline at end of file +#endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cpp similarity index 99% rename from cpp/skyweaver/src/SkyCleaver.cu rename to cpp/skyweaver/src/SkyCleaver.cpp index 4f73cce..4420950 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cpp @@ -281,6 +281,7 @@ void SkyCleaver::init_readers() BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver: " << _header.to_string(); _header.nchans = _header.nchans * _config.nbridges(); _header.nbeams = _config.nbeams(); + } void SkyCleaver::init_writers() @@ -449,7 +450,8 @@ void SkyCleaver::cleave() #pragma omp parallel for schedule(static) collapse(2) for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { - for(std::size_t idm = 0; idm < ndms; idm++) { + for(std::size_t idm = 0; idm < ndms; idm++) { // cannot separate loops, so do checks later + // _beam_data[idm][ibeam] is not found, skip if(_beam_data.find(idm) == _beam_data.end()) { continue; diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cpp similarity index 100% rename from cpp/skyweaver/src/skycleaver_cli.cu rename to cpp/skyweaver/src/skycleaver_cli.cpp From 70647a981c30c94d9abb55ac406cbfc384dea1e1 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Fri, 11 Oct 2024 17:42:01 +0200 Subject: [PATCH 28/65] correctly set frequencies for described vectors, CDD, IDD --- cpp/skyweaver/DescribedVector.hpp | 12 +++++++++--- .../detail/IncoherentDedispersionPipeline.cu | 5 ++--- cpp/skyweaver/detail/file_writer_callbacks.cpp | 6 ++---- cpp/skyweaver/src/CoherentDedisperser.cu | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cpp/skyweaver/DescribedVector.hpp b/cpp/skyweaver/DescribedVector.hpp index 6fcdd0a..4950f30 100644 --- a/cpp/skyweaver/DescribedVector.hpp +++ b/cpp/skyweaver/DescribedVector.hpp @@ -366,12 +366,13 @@ struct DescribedVector { * @param freqs */ void frequencies(FrequenciesType const& freqs) - { + { if(freqs.size() != get_dim_extent()) { throw std::runtime_error("Invalid number of frequecies passed."); } _frequencies_stale = false; _frequencies = freqs; + } /** @@ -386,6 +387,7 @@ struct DescribedVector { } _frequencies_stale = false; _frequencies.resize(1, freq); + _frequencies[0] = freq; } /** @@ -691,9 +693,13 @@ using FPAStatsD = DescribedVector, FreqDim, PolnDim, AntennaDim>; //skycleaver output vectors - template -using TFPowersH = DescribedVector>, +using TDBPowersStdH = DescribedVector>, + TimeDim, + DispersionDim, + BeamDim>; +template +using TFPowersStdH = DescribedVector>, TimeDim, FreqDim>; diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index 62f722b..f295f5b 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -73,10 +73,9 @@ void IncoherentDedispersionPipeline:: // Set the correct DMs on the block _output_buffers[ref_dm_idx].dms(plan[ref_dm_idx].incoherent_dms); _output_buffers[ref_dm_idx].reference_dm(plan[ref_dm_idx].coherent_dm); - _output_buffers[ref_dm_idx].frequencies({_config.centre_frequency() - _config.bandwidth() / 2.0}); - - BOOST_LOG_TRIVIAL(debug) << "setting centre frequency to " << _output_buffers[ref_dm_idx].frequencies()[0]; + _output_buffers[ref_dm_idx].frequencies(_config.channel_frequencies().front()); + BOOST_LOG_TRIVIAL(debug) << "setting centre frequency to " << std::setprecision(15) << _output_buffers[ref_dm_idx].frequencies()[0]; BOOST_LOG_TRIVIAL(debug) << "Passing output buffer to handler: " << _output_buffers[ref_dm_idx].describe(); _timer.start("file writing"); diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 10d9929..f2b9601 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -54,7 +54,6 @@ RECEIVER L-band FREQ 1284000000.000000 BW 856000000.000000 TSAMP 4.7850467290 -STOKES I NBIT 8 NDIM 1 @@ -62,7 +61,6 @@ NPOL 1 NCHAN 64 NBEAM 800 ORDER TFB - CHAN0_IDX 2688 )"; @@ -125,9 +123,9 @@ create_dada_file_stream(MultiFileWriterConfig const& config, header_writer.set("RA", header.ra); header_writer.set("DEC", header.dec); header_writer.set("NBEAM", stream_data.nbeams()); + header_writer.set("OBS_NCHAN", header.obs_nchans); header_writer.set("NCHAN", stream_data.nchannels()); - header_writer.set("OBS_NCHAN", header.obs_nchans); header_writer.set("OBS_FREQUENCY", header.obs_frequency); header_writer.set("OBS_BW", header.obs_bandwidth); @@ -135,7 +133,7 @@ create_dada_file_stream(MultiFileWriterConfig const& config, if(stream_data.ndms()) { header_writer.set("NDMS", stream_data.ndms()); header_writer.set("DMS", stream_data.dms(), 7); - } + } header_writer.set( "COHERENT_DM", static_cast(stream_data.reference_dm())); diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 79fc4be..94b0d31 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -392,7 +392,7 @@ struct DMResponse { int chan = tid / num_fine_chans; // Coarse channel int fine_chan = tid % num_fine_chans; // fine channel - double nu_0 = low_freq + chan * coarse_chan_bw - 0.5f * coarse_chan_bw; + double nu_0 = low_freq + chan * coarse_chan_bw; double nu = fine_chan * fine_chan_bw; // fine_chan_freq From 0dde728270860a8ca853dd1016e311052e4e4153 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Mon, 21 Oct 2024 05:21:23 +0200 Subject: [PATCH 29/65] Added multiple tiling plot code, targets file, mosaic command file, psf, neighbouring and overlapping beam generator --- python/skyweaver/cli.py | 40 +-- python/skyweaver/skyweaver.py | 442 ++++++++++++++++++++++++++++++++-- 2 files changed, 452 insertions(+), 30 deletions(-) diff --git a/python/skyweaver/cli.py b/python/skyweaver/cli.py index c29fbfb..184b898 100644 --- a/python/skyweaver/cli.py +++ b/python/skyweaver/cli.py @@ -150,7 +150,8 @@ def delays_create( bfconfig: str, pointing_idx: int = None, step: float = 4.0, - outfile: str = None): + outfile: str = None, + hex: bool = True): """Create a delay file Args: @@ -160,6 +161,7 @@ def delays_create( step (float, optional): The time step between delay solutions. Defaults to 4.0 seconds. outfile (str, optional): The file to write delay models to. Defaults to a standard output filename. + hex (bool, optional): Use hex encoding for the output filename. Defaults to True. """ sm = skyweaver.SessionMetadata.from_file(metafile) bc = skyweaver.BeamformerConfig.from_file(bfconfig) @@ -173,23 +175,30 @@ def delays_create( raise ValueError("Pointing idx {} requested but only {} pointings in session") step = step * u.s pointing = pointings[pointing_idx] - delays, targets, _ = skyweaver.create_delays(sm, bc, pointing, step=step) + if outfile is None: - fname = "swdelays_{}_{}_to_{}_{}.bin".format( - pointing.phase_centre.name, - int(pointing.start_epoch.unix), - int(pointing.end_epoch.unix), - secrets.token_hex(3) - ) + if hex: + fname = "swdelays_{}_{}_to_{}_{}".format( + pointing.phase_centre.name, + int(pointing.start_epoch.unix), + int(pointing.end_epoch.unix), + secrets.token_hex(3) + ) + else: + fname = "swdelays_{}_{}_to_{}".format( + pointing.phase_centre.name, + int(pointing.start_epoch.unix), + int(pointing.end_epoch.unix) + ) else: fname = outfile - log.info("Writing delay model to file %s", fname) - with open(fname, "wb") as fo: + delays, targets, _ = skyweaver.create_delays(sm, bc, pointing, step=step, outfile=fname) + + log.info(f"Writing delay model to file {fname}.bin") + with open(fname + ".bin", "wb") as fo: for delay_model in delays: fo.write(delay_model.to_bytes()) - with open(fname + ".targets", "w") as fo: - for target in targets: - fo.write(target.format_katcp() +"\n") + def parse_default_args(args): @@ -255,13 +264,16 @@ def cli(): help="An HDF5 FBFUSE-BVR metadata file") delays_create_parser.add_argument("bfconfig", metavar="BFCONFIG", help="A YAML beamformer configuration file") + delays_create_parser.add_argument("--no-hex", action="store_false", dest="hex", + help="Do not use hex encoding for the output filename") delays_create_parser.set_defaults( func=lambda args: delays_create( args.metafile, args.bfconfig, args.pointing_idx, args.step, - args.outfile)) + args.outfile, + args.hex)) # parse and execute args = parser.parse_args() diff --git a/python/skyweaver/skyweaver.py b/python/skyweaver/skyweaver.py index 55ff79f..38e99d5 100644 --- a/python/skyweaver/skyweaver.py +++ b/python/skyweaver/skyweaver.py @@ -7,18 +7,28 @@ import logging import textwrap import ctypes -from typing import Any +from typing import Any, Tuple from dataclasses import dataclass from typing_extensions import Self +import sys +from collections import defaultdict +from matplotlib.patches import Ellipse +import matplotlib.patches as mpatches +from adjustText import adjust_text # 3rd party imports import h5py import yaml import numpy as np +import pandas as pd +import random from rich.progress import track from astropy.time import Time, TimeDelta from astropy import units as u +from astropy.coordinates import SkyCoord from astropy.units import Quantity +from astropy import wcs +import matplotlib.pyplot as plt from katpoint import Target, Antenna from mosaic.beamforming import ( DelayPolynomial, @@ -228,6 +238,8 @@ def add_tiling( for ii, (ra, dec) in enumerate(coordinates): self._targets.append( (Target(f"{prefix}_{ii:04d},radec,{ra},{dec}"), sub_array_idx)) + + def add_beam(self, target: Target, subarray: Subarray = None) -> None: """Add a single beam to the engine @@ -245,6 +257,7 @@ def add_beam(self, target: Target, subarray: Subarray = None) -> None: sub_array_idx = len(self._subarray_sets) - 1 self._targets.append((target, sub_array_idx)) + def _extract_weights(self) -> np.ndarray: # Here we extract the weights for each beam/antenna # as an optimisation we cache the antenna mask per @@ -649,6 +662,7 @@ class BeamSet: A beam set is a collection of beams which are formed from a common subarray. """ + name: str anntenna_names: list[str] beams: list[Target] tilings: list[dict] @@ -691,6 +705,7 @@ def from_file(cls, config_file: str) -> Self: for bs in data["beam_sets"]: beam_sets.append( BeamSet( + bs["name"], bs["antenna_set"], bs["beams"], bs["tilings"], @@ -707,10 +722,27 @@ def from_file(cls, config_file: str) -> Self: ) +class BeamSetOutput: + def __init__(self): + self.beam_names = [] + self.beam_set_id = None + self.beam_shape_x = None + self.beam_shape_y = None + self.beam_shape_z = None + self.overlap = None + self.ra = None # Right Ascension + self.dec = None # Declination + + def set_ra_dec(self, ra, dec): + """Set RA and Dec for the beam set.""" + self.ra = ra + self.dec = dec + + def make_tiling( pointing: PointingMetadata, subarray: Subarray, - tiling_desc: dict) -> Tiling: + tiling_desc: dict) -> Tuple[Tiling, BeamShape]: """Make a tiling using the complete mosaic tiling options Args: @@ -779,15 +811,37 @@ def make_tiling( antenna_strings: list[str] = [ ant.format_katcp() for ant in subarray.antenna_positions ] + psfsim: PsfSim = PsfSim(antenna_strings, ref_freq) - beam_shape: BeamShape = psfsim.get_beam_shape(target, epoch.unix) + psf_beam_shape: BeamShape = psfsim.get_beam_shape(target, epoch.unix) + #Build Mosaic command here. Remove T + mosaic_epoch = epoch.iso.replace("T", " ") + mosaic_epoch = mosaic_epoch.replace("-", ".") + mosaic_antenna_string = ','.join([item.replace('m', '') for item in subarray.names]) + mosaic_command=f"python maketiling.py --freq {ref_freq} --source {target.body._ra} {target.body._dec} --datetime {mosaic_epoch} --subarray {mosaic_antenna_string} --verbose --tiling_method {method} --tiling_shape {shape} --ants antenna.csv --beamnum {nbeams} --overlap {overlap}" + + tiling: Tiling = generate_nbeams_tiling( - beam_shape, nbeams, overlap, + psf_beam_shape, nbeams, overlap, method, shape, parameter=shape_params, coordinate_type=coordinate_type) - return tiling - + return tiling, psf_beam_shape, mosaic_command + +def pad_ra_dec(ra, dec): + # Split RA and Dec into components + ra_parts = ra.split(':') + dec_parts = dec.split(':') + + # Pad RA hours and Dec degrees to two digits + ra_parts[0] = ra_parts[0].zfill(2) # Ensure two digits for RA hours + dec_parts[0] = dec_parts[0].zfill(2) # Ensure two digits for Dec degrees + + # Join the parts back together + padded_ra = ':'.join(ra_parts) + padded_dec = ':'.join(dec_parts) + + return padded_ra, padded_dec def create_delays( session_metadata: SessionMetadata, @@ -795,7 +849,8 @@ def create_delays( pointing: PointingMetadata, start_epoch: Time = None, end_epoch: Time = None, - step: TimeDelta = 4 * u.s) -> list[DelayModel]: + step: TimeDelta = 4 * u.s, + outfile: str = None) -> list[DelayModel]: """Create a set of delay models Args: @@ -808,6 +863,7 @@ def create_delays( Defaults to the end of the pointing. step (TimeDelta, optional): The step size between consequtive solutions. Defaults to 4*u.s. + outfile (str, optional): The path to write the delay models to. Defaults is None. Returns: list[DelayModel]: A list of delay models @@ -824,27 +880,381 @@ def create_delays( log.info("Step size: %s", step.to(u.s)) full_subarray = om.get_subarray() de = DelayEngine(full_subarray, pointing.phase_centre) - bs_tilings = [] + # Initialize beam_set_lookup dictionary to track unique beam sets + beam_set_lookup = {} + beam_set_id = 0 + plot_beams = [] + neighbouring_beams = [] + #Initialise the known beams. PSF size is calculated for 50% overlap + nbeams_requested = 1 + overlap = 0.5 + #Iterate beams first through all beam sets for bs in bc.beam_sets: - tilings = [] - subarray_subset = om.get_subarray(bs.anntenna_names) - # add beams first as these are likely more important than tilings + sorted_antennas = sorted(bs.anntenna_names) + subarray_subset = om.get_subarray(sorted_antennas) + antenna_string = ','.join(sorted_antennas) if bs.beams is not None: for target_desc in bs.beams: - de.add_beam(Target(target_desc), subarray_subset) + target = Target(target_desc) + #Add the beam to the delay engine + de.add_beam(target, subarray_subset) + beam_key = (antenna_string, overlap, nbeams_requested) + if beam_key not in beam_set_lookup: + beam_set_lookup[beam_key] = beam_set_id + beam_set_id += 1 + ra = str(target.body._ra) + dec = str(target.body._dec) + # Pad RA and Dec to two digits + ra, dec = pad_ra_dec(ra, dec) + name = target.name + current_beam_set_id = beam_set_lookup[beam_key] + #Get the PSF Beam shape at 50 % overlap for beams defined in yaml. + tiling_desc = { + "nbeams": nbeams_requested, + "overlap": overlap, + "target": target_desc, + } + _, psf_beam_shape,_ = make_tiling(pointing, subarray_subset, tiling_desc) + #Add the beam to the plot beams list + plot_beams.append((name, ra, dec, round(psf_beam_shape.axisH, 5), round(psf_beam_shape.axisV, 5), round(psf_beam_shape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'known')) + neighbouring_beams.append((name, ra, dec, round(psf_beam_shape.axisH, 5), round(psf_beam_shape.axisV, 5), round(psf_beam_shape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'known')) + + #Iterate through all tilings + bs_tilings = [] + for bs in bc.beam_sets: + tilings = [] + sorted_antennas = sorted(bs.anntenna_names) + subarray_subset = om.get_subarray(sorted_antennas) + antenna_string = ','.join(sorted_antennas) if bs.tilings is not None: + output_prefix = f"{outfile}_{bs.name}" if outfile is not None else None for tiling_desc in bs.tilings: - tiling = make_tiling(pointing, subarray_subset, tiling_desc) + tiling, psf_beamshape, mosaic_command = make_tiling(pointing, subarray_subset, tiling_desc) + #Add the tiling to the delay engine de.add_tiling(tiling, subarray_subset) tilings.append(tiling) + cb_beamshape = tiling.meta["axis"][:3] #axisH, axisV, angle + overlap = tiling.meta["axis"][-1] + nbeams_requested = tiling_desc['nbeams'] + beam_key = (antenna_string, overlap, nbeams_requested) + if beam_key not in beam_set_lookup: + beam_set_lookup[beam_key] = beam_set_id + beam_set_id += 1 + current_beam_set_id = beam_set_lookup[beam_key] + coords = tiling.get_equatorial_coordinates() + coords = SkyCoord(coords, unit=u.deg) + mosaic_command+=f" --tiling_plot {output_prefix}_bid_{beam_set_id}.png --tiling_coordinate {output_prefix}_bid_{beam_set_id}.csv" + log.info("Mosaic command written to %s", f"{outfile}.mosaic") + log.info(f"Writing PSF of BeamSet {bs.name} for target {target.name} to {output_prefix}_bid_{beam_set_id}.fits") + psf_beamshape.psf.write_fits(f"{output_prefix}_bid_{beam_set_id}.fits") + log.info(f"PSF Plot of BeamSet {bs.name} for target {target.name} to {output_prefix}_bid_{beam_set_id}.png") + psf_beamshape.plot_psf(f"{output_prefix}_bid_{beam_set_id}.png") + with open(f"{outfile}.mosaic", "a") as f: + f.write(mosaic_command + "\n") + for index, coord in enumerate(coords): + ra_hms = coord.ra.to_string(unit=u.hour, sep=':', precision=2, pad=True) + dec_dms = coord.dec.to_string(unit=u.degree, sep=':', precision=1, alwayssign=True, pad=True) + plot_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(cb_beamshape[0], 5), round(cb_beamshape[1], 5), round(cb_beamshape[2], 5), current_beam_set_id, overlap, len(sorted_antennas), 'tiling')) + neighbouring_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(psf_beam_shape.axisH, 5), round(psf_beam_shape.axisV, 5), round(psf_beam_shape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'tiling')) bs_tilings.append(tilings) - log.info( - "Calculating solutions for %d antennas and %d beams", - full_subarray.nantennas, de.nbeams) + + target = tiling_desc.get("target", None) + if target is None: + print("No target specified") + target = pointing.phase_centre + else: + target = Target(target) + column_list = ['name', 'ra', 'dec', 'x', 'y', 'angle', 'beam_set_id', 'overlap', 'nantennas', 'type'] + plot_beams_df = pd.DataFrame(plot_beams, columns=column_list) + #Plot beams has beam shape for the overlap requested, wheras neighbouring beams has the PSF shape (50% overlap) + neighbouring_beams_df = pd.DataFrame(neighbouring_beams, columns=column_list) + log.info("Calculating solutions for %d antennas and %d beams", full_subarray.nantennas, de.nbeams) delays = de.calculate_delays(start_epoch, end_epoch, step) + log.info("Beams and tilings written to %s.targets", outfile) + plot_beams_df.to_csv(outfile + ".targets", index=False) + boresight_coords = psf_beamshape.bore_sight.equatorial + plot_multiple_tilings(pointing, neighbouring_beams_df, plot_beams_df, boresight_coords, outfile) return delays, de.targets, bs_tilings +def plot_multiple_tilings(pointing, neighbouring_beams_df, plot_beams_df, boresight_coords, outfile, HD=True, beam_size_scaling=1.0, annotate_beam_names=False): + # Initialize WCS projection + wcs_properties = wcs.WCS(naxis=2) + wcs_properties.wcs.crpix = [0, 0] + wcs_properties.wcs.ctype = ["RA---TAN", "DEC--TAN"] + wcs_properties.wcs.crval = boresight_coords + center = boresight_coords + + # Use neutral scaling for overlap detection + wcs_properties.wcs.cdelt = [1, 1] # Neutral pixel scale + + detector = BeamOverlapDetector(neighbouring_beams_df, wcs_properties) + overlaps_df = detector.find_overlapping_beams() + overlaps_df.to_csv(outfile + "_overlapping_beams.csv", index=False) + + # Update WCS projection for plotting + step = 1 / 10000000000.0 + wcs_properties.wcs.cdelt = [-step, step] + resolution = step + + thisDPI = 300 + if HD: + width = 3200. + extra_source_text_size = 8 + else: + width = 800. + extra_source_text_size = 3 + + fig = plt.figure(figsize=(width/thisDPI, width/thisDPI), dpi=thisDPI) + axis = fig.add_subplot(111, aspect='equal', projection=wcs_properties) + + # Define color palette + color_palette = ['#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33', '#a65628', '#f781bf'] + + # Extract relevant data from DataFrame in one step for efficiency and clarity + beam_ra, beam_dec, beam_name, beam_set_id, beam_x, beam_y, beam_angle, beam_type, nantennas, overlap = ( + plot_beams_df['ra'].astype(str), + plot_beams_df['dec'].astype(str), + plot_beams_df['name'].astype(str), + plot_beams_df['beam_set_id'].astype(int), + plot_beams_df['x'].astype(float), + plot_beams_df['y'].astype(float), + plot_beams_df['angle'].astype(float), + plot_beams_df['type'].astype(str), + plot_beams_df['nantennas'].astype(int), + plot_beams_df['overlap'].astype(float) + ) + + # Get equatorial coordinates + equatorialCoordinates = SkyCoord(beam_ra, beam_dec, frame='fk5', unit=(u.hourangle, u.deg)) + beam_coordinate = np.array(wcs_properties.wcs_world2pix(np.array([equatorialCoordinates.ra.deg, equatorialCoordinates.dec.deg]).T, 0)) + + # Plot the boresight for reference + axis.plot(0, 0, marker='+', markersize=15, color='black') + + # Store labels for legend creation based on beam_set_id + labels = {} + + # Loop through all beams to create ellipses and prepare legend text + for idx in range(len(beam_coordinate)): + coord = beam_coordinate[idx] + + # Create the ellipse for each beam + ellipse = Ellipse( + xy=coord, + width=2.0 * beam_x.iloc[idx] * beam_size_scaling / resolution, + height=2.0 * beam_y.iloc[idx] * beam_size_scaling / resolution, + angle=beam_angle.iloc[idx] + ) + ellipse.fill = False + color = color_palette[beam_set_id.iloc[idx] % len(color_palette)] + ellipse.set_edgecolor(color) + axis.add_artist(ellipse) + + # Prepare legend text based on the beam type criteria + if beam_type.iloc[idx] == 'known': + legend_text = f"Beam set{beam_set_id.iloc[idx]}: {beam_type.iloc[idx]}, {nantennas.iloc[idx]} antennas" + axis.annotate( + beam_name.iloc[idx], + xy=coord, + xytext=(coord[0] + 5, coord[1] + 5), # Text position, slightly offset + fontsize=8 + ) + else: + legend_text = f"Beam set{beam_set_id.iloc[idx]}: {beam_name.iloc[idx].split('_')[0]}, {nantennas.iloc[idx]} antennas, {overlap.iloc[idx]} overlap" + # Add an annotation with the beam name. Be careful with the font size, as it can be too large for the plot + if annotate_beam_names: + axis.annotate(beam_name.iloc[idx], xy=coord, xytext=(coord[0], coord[1]), fontsize=extra_source_text_size) + # If the legend entry for this beam_set_id does not exist yet, create it + if beam_set_id.iloc[idx] not in labels: + labels[beam_set_id.iloc[idx]] = (ellipse, legend_text) + + # Create a legend for all beam categories (all types) + handles = [labels[k][0] for k in labels] + legend_texts = [labels[k][1] for k in labels] + axis.legend(handles, legend_texts, loc='upper right') + margin = 1.1 * max(np.sqrt(np.sum(np.square(beam_coordinate), axis=1))) + axis.set_xlim(center[0] - margin, center[0] + margin) + axis.set_ylim(center[1] - margin, center[1] + margin) + axis.set_xlabel('RA', fontsize=30) + axis.set_ylabel('Dec', fontsize=30) + + # Save the original, unzoomed plot + # output_filename_unzoomed = f"{outfile}_original.png" + # log.info(f"Saving original unzoomed plot to {output_filename_unzoomed}") + # plt.tight_layout() + # plt.savefig(output_filename_unzoomed) + + # Now zoom into the tiling beam sets and save separate plots + tiling_df = plot_beams_df[plot_beams_df['type'] == 'tiling'] + + for beam_set, group_df in tiling_df.groupby('beam_set_id'): + equatorialCoordinates_tiling = SkyCoord(group_df['ra'].astype(str), group_df['dec'].astype(str), frame='fk5', unit=(u.hourangle, u.deg)) + beam_coordinate_tiling = np.array(wcs_properties.wcs_world2pix(np.array([equatorialCoordinates_tiling.ra.deg, equatorialCoordinates_tiling.dec.deg]).T, 0)) + # Recalculate margin for this specific beam set + margin = 1.3 * max(np.sqrt(np.sum(np.square(beam_coordinate_tiling), axis=1))) + + # Set the new axis limits for zooming into this region + axis.set_xlim(center[0] - margin, center[0] + margin) + axis.set_ylim(center[1] - margin, center[1] + margin) + + # Save the zoomed plot for the current beam_set_id + output_filename_zoomed = f"{outfile}_tiling_beamset_{beam_set}.png" + log.info(f"Saving zoomed plot for beam_set_id {beam_set} to {output_filename_zoomed}") + plt.title(f"Boresight: {pointing.phase_centre.name}, UTC Start {pointing.start_epoch.isot}, Zoomed into Beam set {beam_set}") + plt.tight_layout() + plt.savefig(output_filename_zoomed) + + +class BeamOverlapDetector: + def __init__(self, neighbour_df, wcs_properties): + self.neighbour_df = neighbour_df + self.wcs_properties = wcs_properties + self._convert_coordinates() + self._convert_to_pixel_coordinates() + + def _convert_coordinates(self): + # Convert RA and Dec from strings to degrees + coords = SkyCoord(ra=self.neighbour_df['ra'].values, dec=self.neighbour_df['dec'].values, unit=(u.hourangle, u.deg)) + self.neighbour_df['ra_deg'] = coords.ra.deg + self.neighbour_df['dec_deg'] = coords.dec.deg + + def _convert_to_pixel_coordinates(self): + # Convert RA and Dec to pixel coordinates using WCS + sky_coords = SkyCoord(ra=self.neighbour_df['ra_deg'].values, dec=self.neighbour_df['dec_deg'].values, unit='deg') + pixel_coords = np.array(self.wcs_properties.wcs_world2pix(np.array([sky_coords.ra.deg, sky_coords.dec.deg]).T, 0)) + self.neighbour_df['x_pix'], self.neighbour_df['y_pix'] = pixel_coords[:, 0], pixel_coords[:, 1] + + + def ellipse_parametric(self, t, a, b, x0, y0, theta): + cos_t = np.cos(t) + sin_t = np.sin(t) + x = x0 + a * cos_t * np.cos(theta) - b * sin_t * np.sin(theta) + y = y0 + a * cos_t * np.sin(theta) + b * sin_t * np.cos(theta) + return x, y + + def point_in_ellipse(self, x, y, ellipse): + x0, y0, a, b, theta = ellipse["x0"], ellipse["y0"], ellipse["a"], ellipse["b"], ellipse["theta"] + cos_theta = np.cos(-theta) + sin_theta = np.sin(-theta) + xr = cos_theta * (x - x0) - sin_theta * (y - y0) + yr = sin_theta * (x - x0) + cos_theta * (y - y0) + return (xr**2 / a**2) + (yr**2 / b**2) <= 1 + + def check_containment(self, ellipse1, ellipse2): + # Check if the center of ellipse1 is inside ellipse2 + x0, y0 = ellipse1["x0"], ellipse1["y0"] + return self.point_in_ellipse(x0, y0, ellipse2) + + def discrete_overlap(self, ellipse1, ellipse2, num_points=100): + # First, check if one ellipse contains the center of the other + if self.check_containment(ellipse1, ellipse2) or self.check_containment(ellipse2, ellipse1): + return True + + # Check points on the perimeters of both ellipses + t_values = np.linspace(0, 2 * np.pi, num_points) + #psf_x, psf_y -> semi-major axis (a), semi-minor axis (b) of the ellipse + x1, y1 = self.ellipse_parametric(t_values, ellipse1["a"], ellipse1["b"], ellipse1["x0"], ellipse1["y0"], ellipse1["theta"]) + + # Check if any points on the perimeter of ellipse1 lie inside ellipse2 + for x, y in zip(x1, y1): + if self.point_in_ellipse(x, y, ellipse2): + return True + return False + + def find_nearest_neighbours(self, beam, beams, n=6): + # Create a SkyCoord object for the target beam + target_coord = SkyCoord(ra=beam['ra_deg'] * u.deg, dec=beam['dec_deg'] * u.deg) + + # Create SkyCoord objects for all other beams + all_coords = SkyCoord(ra=[b['ra_deg'] for b in beams] * u.deg, + dec=[b['dec_deg'] for b in beams] * u.deg) + + # Calculate separations + separations = target_coord.separation(all_coords) + + # Sort by separation and get the nearest neighbours + nearest_indices = np.argsort(separations)[1:n+1] # Skip the first one (itself) + + # Return the list of nearest beam names in descending order of separation + nearest_neighbours = [beams[i]['name'] for i in nearest_indices] + return nearest_neighbours + + def find_overlapping_beams(self): + beams = [] + for _, row in self.neighbour_df.iterrows(): + beams.append({ + "name": row['name'], + "x0": row['x_pix'], + "y0": row['y_pix'], + "a": row['x'], + "b": row['y'], + "theta": np.radians(row['angle']), + "beam_set_id": row['beam_set_id'], + "ra_deg": row['ra_deg'], + "dec_deg": row['dec_deg'] + }) + beam_overlap_dict = {} + for i, beam1 in enumerate(beams): + # Initialize overlap list for beam1 if not already in the dict + if beam1['name'] not in beam_overlap_dict: + beam_overlap_dict[beam1['name']] = { + "name": beam1['name'], + "x_pix": beam1['x0'], + "y_pix": beam1['y0'], + "a": beam1['a'], + "b": beam1['b'], + "angle": np.degrees(beam1['theta']), + "beam_set_id": beam1['beam_set_id'], + "overlapping_beams": [], # Empty list initially + "neighbouring_beams": [] # Empty list for nearest neighbours + } + + # Inner loop to check overlaps + for j in range(i + 1, len(beams)): # Only check pairs once + beam2 = beams[j] + + # Avoid checking beams with the same beam_set_id. They have same PSF shape + if beam1['beam_set_id'] == beam2['beam_set_id']: + continue + + # Initialize overlap list for beam2 if not already in the dict + if beam2['name'] not in beam_overlap_dict: + beam_overlap_dict[beam2['name']] = { + "name": beam2['name'], + "x_pix": beam2['x0'], + "y_pix": beam2['y0'], + "a": beam2['a'], + "b": beam2['b'], + "angle": np.degrees(beam2['theta']), + "beam_set_id": beam2['beam_set_id'], + "overlapping_beams": [], # Empty list initially + "neighbouring_beams": [] # Empty list for nearest neighbours + } + + # Check if beam1 and beam2 overlap + if self.discrete_overlap(beam1, beam2): + # Update overlap lists for both beams + beam_overlap_dict[beam1['name']]['overlapping_beams'].append(beam2['name']) + beam_overlap_dict[beam2['name']]['overlapping_beams'].append(beam1['name']) + + # Find the 6 nearest neighbours for beam1 only if beam set IDs match + matching_beams = [b for b in beams if b['beam_set_id'] == beam1['beam_set_id']] + nearest_neighbours = self.find_nearest_neighbours(beam1, matching_beams) + beam_overlap_dict[beam1['name']]['neighbouring_beams'] = nearest_neighbours + + # Convert the dictionary to a DataFrame + overlap_results = pd.DataFrame(beam_overlap_dict.values()) + overlap_results = overlap_results.sort_values(by=['beam_set_id', 'name']) + + return overlap_results + + + + + + + def main(): """ What does this thing actually do? From 5ca8addac0aa97f0ac6d2dec41174c0b6bad7da5 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Mon, 21 Oct 2024 05:24:47 +0200 Subject: [PATCH 30/65] removed unused class --- python/skyweaver/skyweaver.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/python/skyweaver/skyweaver.py b/python/skyweaver/skyweaver.py index 38e99d5..61eb04d 100644 --- a/python/skyweaver/skyweaver.py +++ b/python/skyweaver/skyweaver.py @@ -721,24 +721,6 @@ def from_file(cls, config_file: str) -> Self: beam_sets ) - -class BeamSetOutput: - def __init__(self): - self.beam_names = [] - self.beam_set_id = None - self.beam_shape_x = None - self.beam_shape_y = None - self.beam_shape_z = None - self.overlap = None - self.ra = None # Right Ascension - self.dec = None # Declination - - def set_ra_dec(self, ra, dec): - """Set RA and Dec for the beam set.""" - self.ra = ra - self.dec = dec - - def make_tiling( pointing: PointingMetadata, subarray: Subarray, From 8fbb204d7e2cc3dda2e25b5d0b717624d34018a3 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Wed, 6 Nov 2024 19:04:03 +0100 Subject: [PATCH 31/65] changed python delay creation code to plot, even if no known beams are given by user --- .bdm.db | Bin 0 -> 12288 bytes .nextflow/.bdm.db | Bin 0 -> 12288 bytes .nextflow/plr/.bdm.db | Bin 0 -> 12288 bytes Dockerfile.skyweavercpp | 4 ++-- Dockerfile.skyweaverpy | 3 ++- cmake/.bdm.db | Bin 0 -> 12288 bytes cpp/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/CMakeLists.txt | 4 ++-- cpp/skyweaver/benchmark/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/detail/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/src/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/data/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/data/dada_files/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/data/dedispersion/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/src/.bdm.db | Bin 0 -> 12288 bytes doc/.bdm.db | Bin 0 -> 12288 bytes examples/.bdm.db | Bin 0 -> 12288 bytes examples/config_files/.bdm.db | Bin 0 -> 12288 bytes examples/metadata_files/.bdm.db | Bin 0 -> 12288 bytes prototyping/.bdm.db | Bin 0 -> 12288 bytes python/.bdm.db | Bin 0 -> 12288 bytes python/skyweaver/.bdm.db | Bin 0 -> 12288 bytes python/skyweaver/__pycache__/.bdm.db | Bin 0 -> 12288 bytes python/skyweaver/skyweaver.py | 9 ++++----- python/skyweaver_test_utils/.bdm.db | Bin 0 -> 12288 bytes python/tests/.bdm.db | Bin 0 -> 12288 bytes 28 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 .bdm.db create mode 100644 .nextflow/.bdm.db create mode 100644 .nextflow/plr/.bdm.db create mode 100644 cmake/.bdm.db create mode 100644 cpp/.bdm.db create mode 100644 cpp/skyweaver/.bdm.db create mode 100644 cpp/skyweaver/benchmark/.bdm.db create mode 100644 cpp/skyweaver/detail/.bdm.db create mode 100644 cpp/skyweaver/src/.bdm.db create mode 100644 cpp/skyweaver/test/.bdm.db create mode 100644 cpp/skyweaver/test/data/.bdm.db create mode 100644 cpp/skyweaver/test/data/dada_files/.bdm.db create mode 100644 cpp/skyweaver/test/data/dedispersion/.bdm.db create mode 100644 cpp/skyweaver/test/src/.bdm.db create mode 100644 doc/.bdm.db create mode 100644 examples/.bdm.db create mode 100644 examples/config_files/.bdm.db create mode 100644 examples/metadata_files/.bdm.db create mode 100644 prototyping/.bdm.db create mode 100644 python/.bdm.db create mode 100644 python/skyweaver/.bdm.db create mode 100644 python/skyweaver/__pycache__/.bdm.db create mode 100644 python/skyweaver_test_utils/.bdm.db create mode 100644 python/tests/.bdm.db diff --git a/.bdm.db b/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..ad9cfd5b69b9ad4dccece7d52f8f73c40ab5ce9e GIT binary patch literal 12288 zcmeHM&2QX96!(54yW4iNEhy@Ws=`%OwTaTj9?$rL3W=m$RZ)`=lC&i%2)S9?wX*gu zwzo+*AQlNuaDW>pB*XzB^-ti!5h3nSgv1|!gv1pI-mGU_JK9P;ml9)je&7_wl;tMe0_ZHq3;|FoiK@fC+5%V zTixZIu6$>;dspT<*^cGd={tLg%vN_6m*jX91a@@P@xwvSp-YD8$c+X8Uze?t4<0y? zBS(O7G<055yqVhk<;(T)85+Yw2IWPA(Xb%oqbW?}LRyq39vQ|lN2ph=&0o4yAGh+3 zMzIrdI69u@hU%Zyi)vSD^A|7H$IZN#o}W2^;WQthepVOCCm)n+XV0B0eYTz0_xlb@ zmpv*BeP?-Pt;=`jirFiJ70-g`2P{r9uw_xt-raj>@^b8|cK!8HR8^1@zTpdNEU9@R#=xQzoen>-PS=fg;!4GM~*-TFG!%4*1R89hK{bc`%uOKh< zlOR7+5WM_M@g@ZK<{dlljht}L2}eOXzwlANG&z7DzcM+1>`TZV<-!4E%IEU{ve$tf zV!qV{5phb>S*d%zZnz+5EftRdbW-MH37N z1_T3w0l|P^KrkQ}5DW+g1Ov|#0~?LDbfKkH8s&1i(x_Ie%~N&Sn$3>qC%zXBBFDvs zLM&ZFilQP@Hx*(i()gPfS6=!-`nfJi-j$y|b-(`c7uwhBZRzC}p?IYt9#s5ptam#> z&s7PwOjA`cA*P{eNJkc7c(*BB`?rRB``3u}ty){U%)=`c!VBRN=k^B&o&LacO~W*a zj!lbTtP=>`Dcenk6L^{GVc_3hBRn*ImdMHcK3!n$E11rba%!(^DXF95R& zu~s;_>y@^2ZfdM+tJk|5+g(>x3{_WD!$8EsnxPv2&)}*{6JB$B^-j4hU7HFwjn}>5 z&{eTwLE;D!2PP24s--gc`u6}o<1Laqyzi9S(!x~u>+ANuv*yQ~S7e!3C8mxIqyf8F z(R7B|{0mTP-r`ciIiUCaQtPAyf1(Kn1OtKr!GK^uFd!HZ3D?HqL``$_xb5MWa%ct ZP)NwF1C)Fts-`L!X@;h28qvUk{vT9hANv3R literal 0 HcmV?d00001 diff --git a/.nextflow/.bdm.db b/.nextflow/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..4ecbe7d1786bb1b6bd37c0995fe46d526466ef5b GIT binary patch literal 12288 zcmeI1-D(p-6vuay*rb)1iUh$@94Muxl(^Z=ry`h0D8>4f1~v3*n@rLnyPLSXX;Ux6 z_;u%n@8GR3;e+@FzJs&bndxp?p-4a|XUMYiKWEN4Gr!qcShDwGm$(>p2ezrZNRyT& zS(Y9mBuP@7UQv4Swm>^EF6dnj^#9h4OIhV!l6n^wThg^}SC^N5ENw6Tq5>Ka00KY& z2mk>f00b@@fk|W`o?cx|O!kimJ|1GrwF!3k{ZXS{ZPwAto%$=pZAf>JgALqv5gVP| z+(ORCG!{vq;_KQ^!j>Yk_r|YIoRguy#9G*Jo`nfUj^J-A@MN24O-bLa!S61Y&d#-+9V76R#i?Zy^YHhd9cjgZ1pq;19`gXm6o;P-$ zR2wbyxZc`AmTqFytiNts~DK%Sa)69LABjyH9F&g+a+}Ud;tU38_lIjEN_Zz75X?J zh~>>#cmjf0(-{j_K+$T3+dr*k28H38K~G5Zf*$u&1ai~4`l}X&Ly}lNAmI&3V)-nE zHzbL5U`)9ScSzG!O_}=B>foV!C^m&o%AW3p2n}ocm^cox+#FkzsgJ`3nmlZv>+}uk zYzQ}q5N;45+H43nh!AcNA^F)5ZV(~dz}`lD>FoXGuiL?IM%Zd&Yh?QV%m+iB9>C9c zrw5R|3E3?YI)Kc0IS(Lv9M~b|TOE=rN9udEu*?Ufi^t+gqJur@KLZ)>+nj#%LjwXp z00;m9AOHk_01yBIKmZ5;0U!VbE-?Y``(I9em*@`~5C8%|00;m9AOHk_01yBIKmZ8* zX9AyE@_p%cst}PQ5jnaTi^Y=3R7#t+l;3=zX{m(QqdxXaXo;*v|LyG>gX2z4DHn?Q zQc2U)qLR-QbH!r0`a61~cI#&(AxXW=`~6<#W7E47rJ|~uR#FwMST4|*LRl%DHI>hr m{(Y3&k}u>dikh!zrAk?;D1QJa;JHr# literal 0 HcmV?d00001 diff --git a/.nextflow/plr/.bdm.db b/.nextflow/plr/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..2211b5050431e4053ce512252953f14d51e6884f GIT binary patch literal 12288 zcmeH~?Q7FO9LLiK*K|@l8AWgm4#t=rVwF>B zr+2>a-w^yK{6qW?{2#m|cekWd7}6Iqen(60`}^JR;XaoHvioeCco-e_9n0{LCe2BS zgmfPvNs==3pQ3+0X6Pi%1-&LB^S@0q(z>3_#j>{=&262Z%q`METaR1%roMxo?rc43 z?Chb3`raC{4GW`|{<5`({L^65@bp5naNa+6g2Kt`AnoEn=ev51u#wa3) z?RP9Z5kc&rW8sP@TFvx&XSJ-TFg+{kiHUC1&O(I7HN8z-m)M@l_GB5OxPc~*8|XemLuoRE z8$<{^BffR^{tEZ);@3lLx3N96LVxkSnFt=hPq%{y zki7}nB^i4FnelQyfb4c)4>3RLm@ILmp;wE`azuJ~B<>`-*q7lwQ1rje=|?{dAOHk_ z01yBIKmZ5;0U!VbfB+Bx0zlvr6Y#(P6S;2^{lNeNKmZ5;0U!VbfB+Bx0zd!=0D=F4 zz{lOhU1>gFNhXrXY$}^hr*pY{UJFJ(9HG#(yzI}YkNxC|JTRS(TB<3UR<2j-RaM)l zmDP<(x$!GG-@N%lmZWa+-K%c#!@mDll#NnJ(`qF}tJW*!TCGx7YUfS$^QM1arG#Ue gDW{ohszffBY|6`2rCzDh!_>4=RVk}gwOXzJ21JCnKL7v# literal 0 HcmV?d00001 diff --git a/Dockerfile.skyweavercpp b/Dockerfile.skyweavercpp index 90df0fc..2b3f093 100644 --- a/Dockerfile.skyweavercpp +++ b/Dockerfile.skyweavercpp @@ -60,7 +60,7 @@ WORKDIR /usr/src/skyweaver COPY . . RUN cmake -S . -B build/ -DARCH=native -DPSRDADA_INCLUDE_DIR=/usr/local/include/psrdada \ -DPSRDADACPP_INCLUDE_DIR=/usr/local/include/psrdada_cpp -DSKYWEAVER_NANTENNAS=64 \ - -DSKYWEAVER_NBEAMS=128 -DSKYWEAVER_NCHANS=64 -DSKYWEAVER_IB_SUBTRACTION=1 -DBUILD_SUBMODULES=1 \ - -DENABLE_TESTING=1 -DENABLE_BENCHMARK=1 &&\ + -DSKYWEAVER_NBEAMS=800 -DSKYWEAVER_NCHANS=64 -DSKYWEAVER_IB_SUBTRACTION=1 -DBUILD_SUBMODULES=1 \ + -DENABLE_TESTING=1 -DENABLE_BENCHMARK=1 -DSKYWEAVER_CB_TSCRUNCH=4 -DSKYWEAVER_IB_TSCRUNCH=4 -DSKYWEAVER_CB_FSCRUNCH=1 -DSKYWEAVER_IB_FSCRUNCH=1 &&\ make -C build/ -j 16 && make -C build/ install diff --git a/Dockerfile.skyweaverpy b/Dockerfile.skyweaverpy index 27077fd..ac7686a 100644 --- a/Dockerfile.skyweaverpy +++ b/Dockerfile.skyweaverpy @@ -20,7 +20,8 @@ RUN apt-get update && apt-get install -y \ # Install Python dependencies RUN pip install --upgrade pip && \ - pip install pytest + pip install pytest && \ + pip install pandas # Copy the rest of your application code into the container diff --git a/cmake/.bdm.db b/cmake/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..dbda81b78cb98217787608da6b38e96752797c7b GIT binary patch literal 12288 zcmeHLL2ukd6!v;I+1=3129+oaRV7zd6jFuYnXx@K6%q+rMTJmGl0u^jA!pawSXtYP z?M)L7NDG1+AdXx(Rfr3Jf-4d~00H9AD+dlp9C@~996L#+UMVxm#Pi;p_r3SN8L#BG z{m!NvJIa0#dUmW3X-=wCq}LQhlB7D!Doj46VNv4(JS&CuirJJ5g_Le6X+(x`CJ^q|>hp_>jY+#0XdF2q zhfCHMg-P^J8U@+a`uz3ljY+E*rSIlJV0lptP(Nu1_4AMA`ofhfl`pnqdvEBla@nKG z(0A6>H@o~~UUd7)t=l`@jqa9mXY1BmYg@a@o88?N#kW02+3DWfSy7VHV9V_EGPRc+ zDfk*1>>j%QAs5COW}+XQE+hA&eEZ(;uq+0>!-%`vu?wLYuvZj%MInh;Nb=Z@<1kXz zH{V`|jDvXJh5C5`L7eR7qUZWaimVFzxFEQGl5uV+2(F)IoSO;~L>tD3r?I@EFpRyT zr6dfBmiSaCU_-81)p_8O==vE!xwRy^e#T{PEs3tbH%xiY%_SMCN;%Ea>N}(OAU_mH zDooEJcP-0sO`^#4W5jCm?BlWmz{?7#GgIKRE?kkja7FGy&bn|#?!pzhi+0w9D{>dE zV82GZboTp|)g3t>jvc?}_+u}#FMl$uq%YuSFQ+dc`z2%#x%2`u<RzQIp$p-mtK|57xU#$V3vD7-WVupJ+aj%YdmhH+`F1)MuB8LkjZDG+H%KFG81)TkX;6#| zovP3@(liZeYk$f~zi0+NzLQQ#`c=RD>9(b67D*>`g4ARZszR#)X}YDVD#Dsh5rr0k qilIk1_@z|=CKLGUjn)+lxW literal 0 HcmV?d00001 diff --git a/cpp/.bdm.db b/cpp/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..c91b928366f3b30bfc88e45704c4df8289c98571 GIT binary patch literal 12288 zcmeH~-D(p-6vuay*la5?l@bI?VW5=SQrhiqJ{7@KLMc|QF@6-Uw#g(NlHJ7JNt=2h zM6GvT_#k}=!I$vDH}Dn2+3d`8H&*CHE~K0xVdsC&oR9gURgcL!Dc36KH6ob>R0y>%^_SzgwLH~k)tc&e7$ z?e`)g+)cz0YH1aXD)Jm-g^WfbnOR!WhUsvjzK30r&YxdY#HYs#RXs){uW1d>WiyV7C1uWwH4zTiqc}TZn#;)3>+IryhAHmmhT6 zQQ57xJrQnE!bXD{nbL+#PM6?yRZf! zK^(th=?Mwqj5?OCkYd$3w0&O74vP-8!=8v}g+26KgmSZ9gI&wQ5lNf?kn~0*aRQdo z8CL$ z(F4feg!~eVJb>JIH6K8JJMf2C9Cbt*0%_niqcR(k6c6N`#1{K9xCd7JZ*%t14+97Q z0U!VbfB+Bx0zd!=00AHX1b_e#xWWYd?|(IQrm!CjAOHk_01yBIKmZ5;0U!VbfB+Eq zUkH5KQ|~Er>5Z5gi>dMHL?V$&rPJnUq=OL%(@bmrjQRLaGqrlJ*D!P0LQ&7_W+9g^ zmW*tn{3|}Ub^BXPQ@X6Qi5 zr6nmGF~)deAhl5}EuKDIKCGn&8u(71!?FFWVOalX9YwLnON*yYl@F`wQ5tSC2zIl^ zK=)@ILjK;HW2Iv!P85E)5!kzJC%)bIXFNvVys&a*jYsB-ZbQ2K>gL*|wRP$G`sJ$^ z*0-fs*0#?{p51Yz&9z&b=Oh-5csGh()ZJkr#kEw!*=^U`=f)tyWc{t9UElq7a{6w2 zf7tBS_kDi39k>`e2JWPddfH$krkp&mgP`wAD_5?qz~gSvbaDUu2D$+|o0}ciV=l&b zVIQOn*JB?J{gQ~fX&$)&?K(!}>&FHqup>evUv z9RMHhzXVo7j^q;Fkz7Kayo7fom++3{679)Lct>&x?}*ii-+HWH(S7^Q8w1Cy zJKmrZ9Y6b4yAT%O+vmdqjFm9{85=6Vxbx{Oz&IV^BIcnUGF47#bd(I6TFMNZgCvu1 zGBcwbILqG6rQ+{|+OKMNYtys8oqc__Sp9kRm6?CdJe*mY{&0F9@9ZB;ZB_nL`AKD^ z{MYi^Wm5XM^xe|Zw~?fqRO&PWQ|ysrK=QwnkF3nY+>d2AIr~_1!1Bk zKAvickKSW#=Xp^uY9s^tLaWtxT6hTPdMkrwGX}Rbsu`+o=`vAeQz5EGW9-U5V(d!u zryqXA+MO?m!b_QMU-s%<_W14vD~{b^E3({oq9~Sb5{=5LhRu|9E5@(=2KZ(8JcGu5 z%QsJo!k07gFFI}e!ByL{TM4dCG~KXN)zlP970b}o7h_ z+iKydZ+WlRYbqKsOxZLI*@By8k%M(CjN5@%J(=ZZvRKD?TDi??blH;;+J`Ailz+*YsOt*3A`T{leKR1h`Q z0>Sc8b^JQz^F!wI_0NgIX^vYdrV7uy7aZIj1a8}3-fI#hEFp@GSi8L_3KmBkD&>aLaPiXI_fc>$GOK1}fhud3K{SJ)q77p4#GQBx*yC`M6-1b2 zcb*+paCpTFTt9H@{;J)s588InjnhIQGS!JiDN&IYssVChnydc?ekpc7Ot#k+nDbdW zv^t`kQ6H45I#fc|WNK+d;plT;1v>MZd5=*Zm3E#H1w9j;3EZ_o&>I9BK_7w<)t{^* z;V7{b1W_{;S*Hf~e3^R=+;>^qo@dFYWXN2cAZ&=OKE4k{iA|`i8=4_&iUP4?cMwC* zy%)J1<{rzDhqvb#bQUR*Y&wmLEUsQsh?Lc}Wm4U=WYwgyq3JPr=>vds-H14;jpF2} z5?8xk(~UKWm@3hsOUN08P)kRG$JjNdN#)j(`Ykve7UcFU!_Jca-9(gAi3pEX6Do?r z(V)~pe z(Oh^8cuZI*q92RDVC~L~D10r`^P`F7A}=Sdf7CQB6sclCuP|{Tv@tWqzXKn26ixu2 zph!R2n-+x|neJ!cqg=1MJqK$BP9(I7VyG&l7MVj$nL@v09^b!VB*ZtAgK(r zia|27$63j=Km{Vx1B2)?bQd!Z55d9==Llg4CGmD;l+YYaxkG;I5EzRhe`JlSh6=e& zQs)xDncgNd`)g$u-z>Me5Ijl00q$j4h8h@{HIbLeOi?`#be8PNyuMu;^|*7<@#}qe z*J<3uvL6&I${bZk;us3%HOpf2ro|(>@g+bnF~Hi%d*8W&sls0l4uU2@ z)w7D%RnxLeFw7qs7IuI+)mkDlB;^rn*TzKQ`Ajc2u|yZg3Xh_!wt))$LzzHDVkalw z1R$%z-Z4>vj!vo8SB>;0puz zP^4K{RYIdCmOjz2M8g)C6oudm5|)J&O(QzR5(d`gETU_QLGW=2%l>3UpM`H2vZco7 Xu{8>f;iNaf;2bJrd4*zOTQmL(E=4di literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/CMakeLists.txt b/cpp/skyweaver/CMakeLists.txt index 76d6ae6..d81e740 100644 --- a/cpp/skyweaver/CMakeLists.txt +++ b/cpp/skyweaver/CMakeLists.txt @@ -18,7 +18,7 @@ set(skyweaver_src src/Timer.cpp src/Transposer.cu src/WeightsManager.cu - src/SkyCleaver.cu + src/SkyCleaver.cpp src/SigprocHeader.cpp ) @@ -138,4 +138,4 @@ if(ENABLE_BENCHMARK) COMMAND beamformer_bench --benchmark_counters_tabular=true WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_subdirectory(benchmark) -endif() \ No newline at end of file +endif() diff --git a/cpp/skyweaver/benchmark/.bdm.db b/cpp/skyweaver/benchmark/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..db73c6c42a37555417a366b82c31ab1ae591ec79 GIT binary patch literal 12288 zcmeHNPm9||6qjUot=+^`LNHwtLZ^g~ZdhE6{#j}XbT^o=bknB0rtQ*0k?pZp*pgjo zoOpXEI3c<9&||M9r-XiqLN7h`T=EV29Xcavw9=Bmo|iXxtoM6w-kR&slj@Ub2=XJd);r5 z&>=rT37L`sMfvFV-EEXC!qAUT+CelM5- zckZaGtGtHCd8lzbTg)qQY z7VOs=ty{O$RkNIB7!*mE__iEOeclxMi=UOo2RClWzZ_D(KPCBe^Ghv{-rL#T?~0ZA zI2fY6FMHhw-2?RKVDI71!4dkRd$f%rKP0Hv{jRr-_-f>0wtAU5^Y*uSF z>mqRl22n;(VXeR*%D6176&OVQY07(Hu3$E+l+&!NA(_+TVo_L8ak>_TYZbHGBuRpZ z;(Sh_zpN@)@T!8%GgBC6U4)`=5sJdaJnJG9g^N%WF4kEWp(tF0B7Yi*sprqHY~Bg^ zaY3R1i56jI-+DTg(+&9ilXL^-Ps047R5oC)yqFs>za8>TES9VZfaVKwQDUcZLN>erd`$4Od-BiTTKu2>~C6=AvQ4f?2cvIriC5HGz`z! z`A^{|`Wic8^CS1@uNT+HThC4m-SW(|JFczsZtAXWyAH;NuIskzS-4|hQ};|el|0Mp kc?ZAITa{I|RFzy+rKE~dk3Iecsvv=-M5Lr7p>0%&%GsUWTCu&0 z?QPOZMJxzz95~R5D*_=Q@lS9pXD$f!%!U7fH@3&Iqpj2fr!u3R^?Yw;zW3fY_DX)c z_0hT;6S+SOeJ7SR;iMpn!rQVe2to~d1v+apFsL#BSFtdD-moStD{nMlc4l@*nE&?p z$+=(WuFXCJz#CV9E5H@t3UCFu0$hQYO@WCtQ)`_&SD)Ou?~?ry3F6Qt5xc&#*;(1{ z$T!zIw`69Mok)&|N4l|`tzN#oBu8W4cftqDZZPZ-x@44&`r*)L>+)6d{yh>BIRuUe zBXU(49ILf1U8+w`(HQP$RD*Ch9+f~bn$koeq|0*RG2=MmNbOZ?t&11ylV;J;I3gjV zqvL6AsQpE~Ag5AmUAR!6G>Tq&Ztetz(|myWMO`Rgec9!J8@rk_MxxKw4)6>X?>FK3rpB^df zH88C9++dG^amq>UgTrCuKFpWzdV6I!?CwQuyA!+MIwSUrpj!mgVhZDX3%#78e*1(b5HJN;FNw>XA`=FFzCrDooBI z-&)3Sn?#Wt#3;+jcMi%1ATJvr&(xqE-NFp{7G}t|Xh*j&L%xL>@-6u27G}t|Fhllh z#8S_GztX%T^68iaT@sA_bbj#*PfQ-budXByVD?LxU8K?jm>FNr2Qa%GvWJ)*bqT6W z(sWiULtKC{ImquMSWIDh4=mDm^IGGP(EO>{ng4tK^ZC~re*ol-E5H@t3UCFu0$c&E z09Sx3z!l&Ma0UKL1#Z^b!j-08nUy4|GFz=y8z<_tH5yy)U=$9!?-Qp-!sYI0)JLkO zX_jhg7$MEHlaDa@@$~Aezlo3Pf-q?RH8*Jg{+RZSnYM7gsinMPk+(o-SSm6sT|-#6 zbfl_C#~Gir3w#nK?0?_>h4$OW+5%#MmI$wpz1V#ZDn++K_#YWU9rux@8;Y%BLj`6F zsfulz8FMQD=Htas`_;CvF*Ve+pgW}1%piW3^jxT%hLJlAZnz@?l_)|TskUkBx`qw# zXPTyB8!UrYaKX=aRFXTDwqQ=pV3jyN{rj7QMQ&s$SjV8Zj&<8qEa;3}`5DO10O<+n zv$TIAwS~8)CP8axclQ%%?sgr|+jXE!mz<|HELf zoD<-~8&`lUz!l&Ma0R#mTmh~CSAZ+P72pbR1^x#LOd|13;Y?GLL`f3UYD}ZqY(YD{ z&`jFG`J^>ZHS4K?eb@DRkHn7a^;AVspj4n(22>5OshGC1@{dG|GPVnYTtOekkiS6DL)!$kVO(2WY- literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/src/.bdm.db b/cpp/skyweaver/src/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..d6a1ed30f03308fb941481998247ccff02948828 GIT binary patch literal 12288 zcmeHMO>Er873QvF?~k-vD~cpGf2I+*jvZH?8P4!0L4ZoK;<~Y8N0y^VMp96_+%=(H z?v~`*mJy^?isY66Ii)_eXmco96hVLjxd-i~?X@Tn1Wnyzfn0lS-jFjCSD=8L4Gwm| zdGqtVH{ZNB1mE6x?TQzINr`v!~4Ih7VJz^=AV7Gy5GoZX?v*>Xy)mF?(eDsz5hFth-gVt^+mphRM zss(UAE3~o#*O)Q#o)bkusI6SNwgMIpqK*gqmm3&Fe6|$(p3hAbyKwGh1kdLdrz07` z^J9zCp^OCDdeQEIu6|bNMg8o^n9$9RLohlv^JwG8R9UT)rKjAVR9TPo& zrx){{4v)#OtC-WothPIhZl{L=Mg{R%q)W$TaFa0f{0NIZ^__d;9RNPw0sBlk$l)cj zBV8gp(k1%gC9)%3B0JJ0^zahdkuH%P;x>|77x!1P@6i3}$n{&UKk6srXW#9W;sAW_ zLL7kNCKQkISOAL7M>7D0JBT2bM?EGINog{wkBc-TM((}TlVCCz6CXIszs(n_uPDu5 zH(#Fl@63lYtnqc@=Z!Pd|D3)*{mj%~ro5?o{nPrb+W%|6tG!nJU-c81T$F-=f`Nj8 zf`Nj8f`Nj8gn`WkRk_e)6SZ==JW-mQoNP>2d1*9mc->(zX#L1_+HSDa8V)-c8yHhU zbwn98Y~41Pt?Yj?xAMdn^?y|1?}D!SRlTeJ?U%e>pI4PrxxiUOWFp(bmd+?62GR-B zsX(k@Ks+W8KL*5)w;op&*3@%Mz38?*_(~5#Z{T0=Ieu)dfsh4|6l2?_2BJ2R(2YL< zbcI8A9;mIu<6+{1&jQS(4XJsLGjKCN=`c z4~GK*Y7oYl$$$XeG>J_xmF%AU8Zb`?dmn~<89wCKMeL0_CIMUn4o){rV4c#Gljr^b!1Gz)?Wv=TULEjElOL_P zPM`mInD~c{P2d6i!#3eesflev5OU#jz*fPNr=jx5uHUMwik)k;#7<)|0XN{vuqnY7 zC6;bk7&93z%YO#&Q^N2#z+bJY%E?^t4M>e7-T|I$La-npniyEZa7~;iSL)cn(nI zWyJmM!RE1}Zu(8v>)wvS>~(}}8)BMS$Tks#iGfmoTlg6OgFnFKnb7>7?a8D3O}s+5 zD{ltyk9Bf{gkqv&2oIZp0%}>N48Vm;hp-=RPaJjCanNg{XgG>)L;?77LI}c?!P^Pz zwn-oi!9^q?PrLxwGhp`=kP*Av8)fbudDrN5Z|Fki%v~M$9j`0ieI{iKm$Iv7{n{7*H_5Fp3bin5{#;l3*PYt&Bp;HcX2^-Y&n7k3t$Go@WtC XbplDx@q!I%K}hwUz3e~05jb%~2oU1RiGP4sc2}p{^au`|pi3!rz4~~+ zdcXHxB~`ih!k!noW`7)voXDicd81S^o-s|sFltaMP<2^?MpX;&Ev4FbYu1cS^kfr8 zmzHlDYwtDAul#rA^72<86s2IGV4z^2V4z^2V4z^&x5mI}d8xMX&_ngpm*4Q*+Y{H1 z0?!TgcW1w|b4@UL>QWWe|Y8wwfTt985QF>ki0?j-O=>DY;=pTP8*Y1Dt z!P2|eBIj`E#^%PKO04~0YkRMwJ@c5?Gk2do=v?mXo6qm>UftTiX0GDkADlHq@2|=9!{KpO9CwdHz1)dBfQ}W9 z(n2>aD8$qzPn;+ULUViXxoz0-IO=;~zcyeTso7c_dA{0WY=v`@5!{Lng zWH=*(RWnZK+v>TK=qPatY$}-fB3YW1Axy&1^CKMFG;&U|9iTkh0d~$i$l?;+ku1?2 z$r8G_M0X@hbVss;EiTa=$r9ZWpGMmB`23pN9lEbiUBB!4)6snV!rzCbSpvTO#4G{h zlQ902GYJ@X{x%aZ4u?32wW~8C(Ui_dX;x$@F>+6mNP@{qoX5ZgbvIvMe#dBi(0X$1 z>$Nx69%%l*d1Lj5)qk!&(fF+4HI`QXwX(hZ4NNIY!9c-4!9c-4!9c-4!9c;lugbu| zdfT|rvMS5va=B8fR;$g`x+=})&Q<5w-SfgI+>B15K4y$aMhRyUBg7d&0vV?t-Lw6` z$Cc0PhEX20zg+3R-~P9%ubpojJ1v@{^|?cJJ%{_u*#C>`^xWXUg+B~~&7;YrFC>=| z)H5b2#X@jOVubR2F+u~h|3_7)r4aJ=y*=&s;W?sr$y4thfEFWkr$=A}=Zq7}V$>o6 z+tQ|38)45OK3yHOKV4Dv)>_;6Q!d2S=`iyC3>Ve?vZcKvQl)g`y593F%aSveLlB9I`&28*bm6#TA9JGTUI z6(Fsl2vhaivdW3P=#3y)Z%rd_7;Y+W3kz{9q~z3M7~06v0ek)&C|!mqQqj~uo|L5dvebk^%|KmRu9{&LUAS*V=~6-T5VV=-j#L_|71sol|6SAq0h_ zAQ%hAwDR)aqVn!i+qlxgIZ2)4g$#;iNx_9J!O`GvN-5J8tj8cKuV*=N>)g*K-Sq(e zcbv&;(8-VusZA^?SiCV~@;;B1AU zq6mIh#00H;2PjMFV@cirORa_hKSe1RC>SUhC>SUhC>SUhC>SUhC>SUhC>SUh_&qRi z8kC+gHd?e?Dwj)@_glt^5BtE6}P#O_jP;R}WG>0xxL-ZrsR1 zluALRMYg`Ls+ZY&;o{wY@r&EDN2m*L-N80IOv2S1Dp!wJP%X+BMi}0gU=SffiICWa tS2852WkE}F^(KMu(l;L^vw4Crxb#cVk}eotT;aJ=!sY~{oO2snKLVuF5oiDa literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/test/data/.bdm.db b/cpp/skyweaver/test/data/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..a6ad13ad3ab19348c3edb844363b2884bacb0401 GIT binary patch literal 12288 zcmeHM-HzKt6t5?OlX4P3y3$#V$G4n9!DhJ#|I?H?gDb*l%^2b#47fd+FyFuUz`|(w+6c z7=Qr`00Y1PFaQhy1Hi!Z#=y#0>-4T)H&upfP6bY8i3P5-E z_0w~)AYa$hkCW3ToSmE&(mkexr51rx6`ZJmXH<$@#+2p-+B^7Yk5xRQ6T$X0J(6kc>K_r!-zR z6)bpD!TPKe?nM`=s9mI@cJVH{NJZ@;6}5|h(M2k17pV|mBiXw6epP)J_^SoZPH?tJ zEBn@WiC%8NAKolCp!gDshtb%8Liv1dK=C?=O)QtX0bNOHW%ZiSS0Kg9`b}b!Ijr7+ zEq*rFt;ZVs!2kw;0bl?a00w{oU;r2Z27m!z02lxUfPrVrz&-t{_J$R-)(xZ8GTQBS z_p-@Hw@YzBzZ~NvJ}Y*PNj7mpCyLxK4Ew(4M1x?^j~s3F%f{ZTzqP*o(PXFnjp@}t zFHb)GW67`T{NL6r^`HG`+bd)T27m!z02lxUfB|3t7yt%<0bl?a00#aS23BYK+gi^G z4BasF);d33cP*>O#^$C~j?LG~v9V#9l|tT4&m6}z9y|Te9}I%X^TRN5Z7+&^d-v}) zKXtd+nY%n~|NaO~x4wVu*nZ@dS2zf4ez`6?u#3CxIJPZhr|*Q3?FO#>Uy>Vz1K0O` dr!OV9AFv+_r4HAQ+(F>kc3^v6H1MLp{RcVRgHr$i literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/test/data/dada_files/.bdm.db b/cpp/skyweaver/test/data/dada_files/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..c9d5502c5359a6820a0328b4aff41e41d9c146f8 GIT binary patch literal 12288 zcmeHN&u`;I6n30!leS5{Wff7VR(MsdN|&lTGaip;SRv7(XjN^2rCUKO4wc(Dsnysi ziL=|~fRsh_#(@I|t_X1f{s|;5ocIG)NQg5MH#qVf|EOJUqnyi(GV*+HX1@34n;A=f z%A*gq+`!RW-JWL$x}jWG3I*kDT~`#P47CU~DO1oWB>^=($cmdA_sV zSl?;rA8s~2(i59*_w~NhaSj7L9=&#FRqqcx&+a{4bNz1935T3StyZt=CF9aj`jaE4 z=jc7)czWWzh1%E3_2uQta6XLTaZJ_jbq6Ookc}qSkqPOJKJplITyR91rE>lD?aENg zIvVtyUP2ePU*smVPwEBR#d3XVsWPl(y)@m_2{d2i1I&}UKxdz+<(C&13tv78?1PRI zFE@S^WA>f(jjcwqGq1Z%ee=DY#{I^&{&0Kq-SzD~{Xt`IRrhVr(RUiVJF9xQ8*ww* zy~yl^TMBa-h0z_?KTg0P;tcKW^Jd@uG#!4>InKlG;c-8iZU-*7j)}c2ILw03V^j&soUBZ1Q19CZCi%d|HKUH_mn(mn0YVOZ5j(l8*8@Qb^n3mCr%Kr7Aq*1Sthcp8N=QKS@FgS$K8CVIw&1b7iiuS$sVD9g^ujlU7{-~YR z-l+ao9e`ODsen{KDj*e*3P=T{0#X5~fK=fBrNG141?4r(C{C+tv8a|xrCPNTmRhZ8 z2liUiZrUwk5{n3CnS=^%31Km+48Oav@#@dZx>8Y;m(=#c^J?qcU;hm2-pqn>S2HhR z+&?@T_{aNJ#Ec2HIHuI1h;u}_g$a+Ti~okyh37y$n^I3tKuuz512NMO1|ghU$RbE! zF^;-20X2%Lv4BtoCJ33rFa@EOHIBMG0X0seHnCwa!A-*u)F9L}#!&B0h18erUaYkd zQG*^78J58)H<3j-;$xYwO~A}zW)OpOVPK13K`g`s8x!+hX#zRSSPp_HMu-za7*tFV zLgT256Htqo+7v=yW-$&Y!w>`x`PlQJPC(5QYQ`}!xM5-@EKadyjyoT_g$bml5w*!U z0ktU?oSJYSIURRj!tejW+&>EZWRVI;1*8H}0jYpgKq?>=kP1izqyka_slZiJ;Mr8+ zo^new)q<)Piqj=XYSreoI+RxywfT8X(-v=y%8j~K0hB2fvl6Df_D6q@X<}?KVi}xR z96>e-a$qJ||4R)sUsXs>wZ2*SvGvQ@a(m_TA23E#7^BJ{zY$Yi5b356s~lr$OF67b1SX*e!wLAH!h22}Y1p;FEvA4M4x1Q|k5^dEecbBq80 literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/test/data/dedispersion/.bdm.db b/cpp/skyweaver/test/data/dedispersion/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..e72a2446636c9637f0144a7c4aa0253343f92d4c GIT binary patch literal 12288 zcmeHM&u`;I6tZOMbck z?k@lSh4DWPYtkluxdGm#Re{;x^<&C~_z6AV%5j%{vhG<1m;_OQ5)$lB2MtJ1Aky)>t6YhSgeYZLL0U7Kvt&8}jAY z_$(S`KQD{wRBEkTx9amok!4785*VJv0p{~k;3uCbwHr5Y%3s`%o&Lnl(#2dogIT&JvyV@Q5&bxy-k%(oVK6w3c(@Z&pw1TdieOL#jIprE6DN+t2yO4ay$w4K z;t_@Xc>+PqW^?FKpKUSA!Z|4b>a!i^o&unLvg6!UK)^PMk59S0BACQp(NhBBq9;BD zg|%U?Y1TP$2~s~5DEF2i^;0c#ZwXSrKS^}Y-6afJC7Pz&8oJZ?C_fa~RG6Ga9$Ln5 zn?w=yV>Qdkb56(9K@zy5e4ClBD~FC`CP_9o0OO6dX2j2H6(%x;J5A?8P2f*O}J_3C9v z3NUt0@;eD8Gnn23>+IXSy5dUBubSG*FE9{IC?FIN3J3*+0zv_yfKWgvAQTV^2nB=! z|5t&2wJp8YG%CxAQmrV}YPE5-&RU}}fC|`Tb9ewlbAFPinZRuXhQ8@?4p*lEn{FF&2*y?Du zVQ98(63bF`J7a>MyMai43Pe{%U$?(!y|1*Tt$!y9X7OS+IwXKq($P#}Vyqfwhh!YV zr;J0lipd9#$bsaKNxo$r)Ch7NtDa(d^|H~_XN$@9{P(Uak z6c7ps1%v`Z0il3UKqw#-5DEwdE|vn1>+&noTGLQuMUgAZRVYX_n#~rpXIGm^TYWxh z*KaiIse%8m8}+ne;f4_n{2>%63~X5)(>9^$x`lOZ>raK1DK5dsc2YF>`N<38_0PT| znAm#KP0PTnt13}DQ0HL94b{X%(=gTu6bLLR5r9d@wr-egWD{axn0%(qHZ6$K!JCd? b9UV#;tZq_jzy;L}!`6vu+8Wkvo8Z3z2~^yS literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/test/src/.bdm.db b/cpp/skyweaver/test/src/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..e2283a31f947f644e618d2b4b00633b33def48a3 GIT binary patch literal 12288 zcmeHM&2QX96!&hn*-f$=Ck-Jhs!B$slvY*ncs!mlZ6%U)DG_a2(zGRNMdR#xw^nw& zi|x$^2T(voNC*jzy>LKC98e|x0WKUkz=boaIKioKK!`hU{Izzvm3pOkl#%`B z9a|SBKcAQyzdwFq>{k%7mNAeqkTH-kkTH-kkTLMIF|eN>EzTZ2TH3$4ZJ9eg)Ak(8 zbff3eYHeYyCS6~y-H@U&$#5mtY@2mY3RlmcIxV^VPRDTe<}JJ1F!_>R@YZy?ooHRW zO4_+&I;P}+j=i3FPHr74&Yn0?+Mnh&Y=^2^PPgAn2}x^G6A6<}Nj_qz;{cG-C=_Rp zA202TNua)KI*~g5J`9GBKPd__MvAk?j+OQ+NtA{agTU)B3>bb=2>GM;^2NC$M{=KD z^^DE78M+%jMndgt3yT+PQDk1S8q)I0TJ1t@Rl2ged}(2IU3#Ooep<4Pjw!9x-dZ~? z@o0pvLG*&*4i72p)l^vBw(PA)=m$Facx&+LTJOfoH``ljv0LAAqwR)g0d%OilN9Pn zfg>i2+%-JUaiztJmlwg}uGh4{|Hy%^$5%&U$FjMJp%=z(La=OZaXga{EZes@o=Ql7 z*7mjrSe>NM_BzR#l+a4fcmpA!8*B}{j)7C6Wd{VsvnkQC11{s)lxW$TZJ+meIwgZw zKBs|M4YTLniGu=0IeskSt!Ww3#C0v(L!nO{V>dkj%F_ejXE2}~+!77MTcUw@3qH6d z8i=<<1MwF1;Ff40-VzOjw^8JJcz*?cyXHH6)2^F#zZ1-#{Ggrl58&OG{R0@@gyAEf zI)LHe(|G{HbO;Y|6zY_yM3e@zI4!CP(KC1BOoGK+407Nk|2Cf*dnkxsh-W7sOzun` zu6$d$R{pE}S$Tfq=Lu(GcKo~X6m^HyakzK^)iW+E=We3E!1w5{_z&C^FJz zl`)-Bjc6fU`4fOEwyF=x%`boajrZ%5RRM`O@MtXQQOg4dZo6B1JEpN6IHFONX+$Q9 ziWCLuib^BJ!d+18>%I2NZWH>=N>!jk-8gTyjlD~TZM4!HQ6?)ylMz-hVT55h(mnPM z=suJ*`c}CroQaH1h2e&2weEQCBkD0`L{l}YU|FUzVTvAUf97M*&Y3R&9l!H_YoaQg z6VZ?$U+T9#>oxeMUp0+J(s~UMU6ysEG9v515XK?lGk=5j@uck=<2;2z!c(>nQs>c_ zLds}}Ayoy-DF!$eAw2s7AVBs!;2Cs}_uFGtVPQBzzuAN=ZJg)XV?)_!kTeidO-GuB zDd+}v5K%%Gs#^e10mX5OZkDRTg`ogY@$oFzv+C}W(XRK~hSyDCAWU@y(qCnWs7Ti_ zi-It>8aOcs!g{eP;Gy}m<`}ly>!z%S6Q=5_EGsg_DnUr6G~(y*`#}T-sy9X-7lF&0 zpmiHu=-RKRBHZgWG1fE&^vi^5I@Jjrg^-|90}z7zyjQ*7gkC?y2^w}y49@T%+Jhh< zhE<3IQ*}xdCR2@45`w(&2Y{q5ih7|cEDw#s03^OBlA&fQoCc~e@Q4yfR-i7*))zyr z;!wXm!q37mPvWz1!+~GFlToIMj49EHs$v;wgkV_>^%qw_e<7Kx8~LhG8+z8F3w5Xq zDkLeDDIrJ&G{!U(BAk5~&@{N2)pE?1AMHi|U?c*8WXLFf9jUrP`GuiI zH+E$n5P~Z-j5h!N&xsQP{ADd;AY&k7AY&k7AY&k7AY&k7AY&k7AY&k7;QzqDzLPs6 z%!oLj%ja_=emSu+EzUxlnG>g{MNyo4-f!ZxSOO{0O1R`Y^#&9rWSuEcw?tG^pqxg@ z!k>9wDxHS!ZeCgR%UmZv{08xlbEqvqhjORi5yps6h7lq#3k5j^Dq2t*(sU>-LP3;5 nVT&>aRt+8VXJw}P>zJZKEe2|+>85~E1|8``mL?#2(SOQ!c*G_j})8pYvI|JA5?oQ>1uN z9Jr|hh3kSS3hye4AP5cIYq-<4f(La9@Ka>Q|1@j}J@u_N)~>XUg=^nky}tU(>iyOW z40z)Va0WO7oB_@NXMi*Csxh#XRvMj~H=E1Pp84o(io!JZQ9_^Z9rSmP`pV<|{-+9+ zDQ==9Xo7~RQq1n%-BFTR5V-N8=ZDb8_GREPPxDq>bv0&h6XHrOXDJB`Buu$oR4_ME`ycHd94jC8k9U;N4Ulrwe{!^=kku%L`Re83fGjDtHFZ3D;)kac z%yOSZK0Y5N{ukx+lgVinM#IyDn!BlwLl=$%77STHA~H$N-8798WpD6s4`&>uo{#IN z1w<*CO<~}NB*mf%cg_GmBpH_)2KZr?aj9Yujy6e8&tnBFn4|%#sem!7NzVb}8|Ru= zT?Vc|KgrT+; zGZYR{I16(9_E!@z+koHP%Qj%~B`h9NWdjz%S91dvuS2nk=~7pqMk&p;unKhsrf6Qi zNq91W`8%*p&gS)NzY6lt@)LO#4|wAYa0WO7oB_@NXMi)n8Q=_X1~>zp0nWgGhk-}+ zuCOJ;T1%2@HL+f=x7V7awcGDU^98xN!WX8V11#KZ&9n?%*Pv>fKv@2`vG@9K;`c9_ zg7Cq(`{$bX$sY^S2eq!SnY)y#E{9R*^`;9?*MMqUrfsN>Vc1YrRkLun{sZ>b8sGZq zjCy+@b%k59K@yZ|EjB?v9HDtHPErp5&4v&_!+@Hl8@2^BmF8z-hirC_#jbGg((FjV zV-ybDMKnu$^I+m>jsq5u-i84mz@7a`Ko|_ym1CN1DpZQ0B3+R zz!~5Sa0WO7oB_@NXW;+9z%mx!7B*x<5+zB@&+Khk?%=j@L(bZbH?k(L%gtOs|C_Km ziiRVgL7SYtp<~-7v{YNQ;qG4&xp26G58n80VDA0?To`YEeWuUoV)B zr30wrs|5fo$FP6}Oh?rWe4784k>;4Trt3PepbGGHf`uIaZN+`wpm8RO_)i#-=L$aH=yJ=G| zgsAn-3m?KuUqbLDyzmWt1i{(t%yhS{P$VFfGbG*lpEKuUezUs}=+^U1>JilJI+p1n zU71!?Re6Arq9{rBjkB*96Ks+Yf?d_n{BP5wvV!lXnRjAxSGo52>h#ptsrAX9OuzsF zKmZ5;0U!VbfWT!VFpNzkv$M0Q;j05m4tvD*97c@ zs+QC3_aY+PO~esuX$g%g@*JlMX^liOJ3pTqX2ON~E^$OUe|}yW-+s|5IWwNj&dsHU z>2Q?>l||U}yc*1W(GvLM+gS4E%#8YJ%QN>ngm0H$;yin;ytY{pCv%%N(8iNmWxY~G z&#D`b%hg@5MhuZ`som1kRie#LyXfQ!XQJ0fxnH!*7^G@*tbhw_laF6cHav8i|;z> z=m9*wGkO5|n~+~(kq3|)FXscuZwLMmi=&Q6O&|@tdQ=)A>5+lFlh|Tk2KT_C|J$7X z=!XFWfB+Bx0zd!=00AHX1b_e#00KY&2wY+U{_lS^{Y7CP7(f6B00AHX1b_e#00KY& z2mk>f@V^Os*ir8(vzdHMjm6aXWFnDBr!yISG%~>mgsx{&{*3wfC#9#zz_fboUpX4N zoPqIbf&I;4L(At&80X8s5;xXveT%0QrMdm%sQLZ)cWd$eHpV4Q)AfRe_0?kDC=~KV rTsUhgo;CgZDjbY)rW|Lku@bppvN12y^BOktMhR<1NiUR&xP<=zBuBc} literal 0 HcmV?d00001 diff --git a/examples/config_files/.bdm.db b/examples/config_files/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..c6df408cbd73cc91d39da0d324300d6e1e323099 GIT binary patch literal 12288 zcmeHN-HO{r6qaOntxe)OrI?0Gp5qwpn@+wD#B;R+3$5 zoZan3!7a^QFZvFpcYTRMU!s)0LEoV>(#)*1BqVo%h&vsTzRnm zF9$Gy0bl?a00w{oU;r4nYz(ZdwO04`ZF}|YQ%X)2BxM;Txw?LQFx(jp(bs#!Z;;ZV zC`UPo$(SLzdi&lM%9lwJWoO$ooll5ZvM8l#Hc!+#y$YQ?AsIm#k8!pjA3C!)THU*M z?bTbNhNm*rESoPEHBhalW~h{O4;2|@j0=g}Nwd}6*sxcfYNBOMG8In9&r2ipubKr% zjaK*09edTTW|>f(gp1G1!PT#t!g==YG78iuyas}&1)^Y)`@G7 zrX@kzT7xt#xzyGgr0G#y@UG1@%)2T$Ez6pa1$&}5g%_0-d!b$HnBOLOPE&?uPf2uI zSMcC<1@E&|xEEcNLc1u1cIjPoQ3~y%6xzkR=%N(bMJeRlNVP8CUuEAp`EE(lF-ez6 zY2W-YHj4-F=l6>TP`(M}#i~7kQh7NaKzTdJhgfZO4Z4cb(%P#-uL2o4)prtKEMR#L zY>KnF+49*^hgl+GSI+hx!b(*i~VdXmiY{3c&7a!Tv9^rmTZ@gGJ}&_mqWiU%Gky z`|FS9|71Wmsen{KDj*e*3P=T{0#{9enKoBzt*q2%pPhK_lOs2XL(h%G{o!tBv)j=> z-tK&&3!m;pdgPAXeynG`*YBUn8VQ!XdT6Iobf?kjclt)X@3sNofXbE~jZ=C3!hfA1Mm1vr#HE@sOgM3j~QemWZ?wGg|uN#_*%x&-fKqI3>M2 zB}8os+SuCfQNzk8p@}J>(4ZItiU_nV17)1;w@(=7E58&ukF^4)dWkY2;GD$RHxV*v zS|-G{X&8{&*oL-g5rinsIF;{yV4SZPDI+ykwpyuNf|4l<%{H<2$$*eyN{JAp(1H}( zEDI*BWq^!P{cVFwtNi+7k(7V`s|$ZC>`ykSfK)&#AQg}bNCl(1QWgXGvPua~c0dUfgk z>fa2&00w{oU;r2Z27m!z;9X;2sjW0xw{GdnZ=MF^Sxh3D1SFOBPy4;SK@UAX=zWEx z4*4lc$&8FB60>*j?4WcJhJNyVH;CqA!l%T!G)d;6oL9_3&z_KkpoGPE9+OY-^u0!F zYfE3Q@fx0rP}5|-h%2C6O~p_u=?=;=iWuh-S>t-6b^EryG|GtYT@I#x+Z6ccAGF5$&70~kkElPK5z(%=)kO5c-u_`vuFQwQ z7#(~*=-u!2(U<*$&-VI9=t1vj2St8J(4hDAU`Dn6A2c1 zOs=1viD~d%F?~2Yslxf_B$e(y4OnUsI4pxv8SspyB2RrvlN9YAezDIgp3_Od`j-vN zDW5H27(~28(S?6n0zt$pE-WPwL|Mg!sf1Xz89h166_&w_hGk0yOv@H}21>D6t$Ei) z;tC9+oS?#5fkBjWSy(GDh=#L__rhGktgDREysR;a>62nnSW!u~7KLjSv&SS&gNT}< zr_evGDp>HUg7ujztn)5XQMgD&;bNb6k&41aDhe0pyo*#6E>a=hMzVGB{>uAK$+rs< zjYzZzbNlv>Gd0_QFFwjPpm-CCn^xI?LiuiPK=C+;O)QtX0xd~tZndk>DM3n3izkUq z<}iN-w)xpyHC`(04+by*3;+Yb05AXy00Y1PFaQhy1Hb?<01R9-14P|W9vE(IRnxSZ zTCdleS9Cs_O*)($)5-8SCQ0^hd-nCoF(oNw-@NUP;&@^@4(@g>yX$n^w%f67&$N`~ zZ`by3{IT`Bt|-^18-HDyyxe%jul)S4DTa2@4MIL(02lxUfB|3t7yt%<0bl?a00w{o zVBjJdSbneGRW=P*Q#DPkt@4w0(=b|Wtgjo{SpOg!Yiov{E99qc>v6(8sZ*4Vi_J97W|;@|1^Pkj?Ro}FFou8a9)V$*GBAJ{Z8 s7Pw=XHuh|{W4=l9+Fi?W95zo#7PeTJ%CKDvJFeqli)HV4UF_li0G>vRWB>pF literal 0 HcmV?d00001 diff --git a/python/.bdm.db b/python/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..5b29949d35bde710bb557cfeb875939822c80532 GIT binary patch literal 12288 zcmeI0&u`pB6vu7vHtQ4?w?ZodRn$RMRZ>NZf39s15(!vUL_#6m0ud)CJG-&6wikQ0 z-Q|F^Ah>bhUqFZ(2mS?)@E>sC#*sTG#Ek8k*ijKxkL??+*YkaE-g`5ju`Sv6KHQH8 zM&mS#LxL>niX_X@+XzXL)S*X<9=xs5PMZsQmTUe0>UN~Q{zjL2*Vd1u7ruM$%H^Lf z-&y~I3TQw82mk>f00e*l5O~@MtdzA*Z*x;!efl864`(4*)$xrDcwdz#VW^zLguL5>D|1kuC#igd5$xlj`bf^hS?vt ziV?Oty&E^wRkvPc6p13V`=}b!{J16bZ@y4EFI~GPe|wLF$5UJ_w>q_|?8BYi{ebVx zYtabpy*~`@1c&H@!@Yx@!z1)waI}q*Fve&Yd@|fdY&WWIxqGEKW?M>Q2R0o&jgm7i z=4DQ1pPYB|=yNgrczV{9>B(8n!$T6$(pBQPE>7x#Rjf^N5fYN+Xm|haE}d~o#t~gV zUqDLOXfER@VNMtDX}#ir0nWyM~I(59xBNuEau zF{(Ai;i74v$(shc&eC9B4B-Y5!VMzCx){O@B7_@6hLlt~%|RNJqfFm(zpRD^P7&q)p9Y>6DgSPnTmdpY>=8w?!Ja6QMWl6C%}+1iuqKQ4GW z_WmzxnnZulfB+Bx0zd!=00AHX1b_e#00KY&2s|qUR*C$k^s;6tvUpqXYFdxBt39n~ zy;qAy`)X<6|Ei_VmgF=|Muy}1hV5CN@0-5o`1G{%*CqBczeZob$M=6ZMc?hm@W-5Y?`FrES8*o OTQF%^`lUhl^?w10^pYz8 literal 0 HcmV?d00001 diff --git a/python/skyweaver/.bdm.db b/python/skyweaver/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..8f3911772db3f250d64458fb92d49b1d5256f6d8 GIT binary patch literal 12288 zcmeHLO>Y}T7~T&zc1q2*s;s0`RR=|(xQfhvjdvG>M1rg+YJifMmI#SxV~^dDy=!-O zo!A_Zj06V|2aa43;)3`S9Lt#t2W}ks56tY&Y<8m{^&HK};oCvMjv^fFwy((pA!VTOpGQ2V|5B^Z%PxrI!A7ooH89kEB&bj z{!M^rf&syRU_dY+7!V8y2Cf+X$cgoDJcpiJp^e+7UM!9vtOw;HaBZat(a&Qqlky2^UKmu`O9W8 ze6`xx*r+Y*#VmcCCn3|zauD&$rqI9nQmL-py(@oznD{4S#Fonjl|}Dv@9cN@&b*2H zVDH0jXSZ_z9v|#|uzhd@9(9hkK2|cZ(Fp}^S`WB zCGrzZFd!HZ3uJ+&X|+ti|7%t)KY#ZP-SpbzYQTn$>A0?=w@q*RUzOetTq8&C!6$z$_{rPS zZ7=I>7wWV_%YlYx(qp@>>upOj-@*exh!zW)x#w+OE*l17&1H8hU9{R Vgef)yI-X&>mg{+@=R%JR{{d;eq+|d9 literal 0 HcmV?d00001 diff --git a/python/skyweaver/__pycache__/.bdm.db b/python/skyweaver/__pycache__/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..e0ff9839ce4f6d500ed074352837584012b31339 GIT binary patch literal 12288 zcmeHLO>Y}T7~WmGu~TrjRb?fjDjF0;ZK`1Q`vX)#jI1gmP)H+61gf>MC(g*;wb@-K zHb@8~!2whxu5jVNl{@GDjY=Fiae?2!JG(oZ-6%-8Hky&gGtc|Z`}NGuO0$nX+95GU zhvU%qVx$Y#1yK~80yef4}-* z`A-16aRxX8oB_@NXMi)n8Mtf=ETpAc^Y-of;?rY-PbN5sLxLkVzPsDm+UuZ?w>ys! zlOZoc5gy@QjI!0Odz&bl`o0&=TO=6wF9(b2SLZ(@6;wvneSGDcwU!#LUN;Kw7_AYp$=?7jiMsG{PZsN9X5-q4JAH zQN2p7xwckcG>TFBBo6}9^J0Mbq9K&;zLIJyw{D3~AI09m2xsMHLnZURx7FV1u${R} z`e^%uz0QNqF8XkH`~9umee_;ue-j0skI`P|lf6wucO#poyO)Z6x}{)i8L)apf*}Lb z)F+h>PNxz1B42(m8kXU>H;kCM7ZV7bIrfX7R|GU-A<3B+$6t8z0w4j+IM);a36hL+RRKY?Q9L}2m2d+QQoS9Du7Y$(98RhIoPU#Hps_$p-n zJ?-@VFUnU0`0>UW;0$mEI0Kvk&H!hCGr$?(3~&ZG1Dt`&&A?(Nz9p>5x+F@HnBL(V zvfPArb45b9Lvvim)wceT=$C^k zz1Xil57@<9R}44pmSZZk8?K?dj^e1At|*G_I)Jpp f(sh;vUoT7x+!ULtfHzyy;p2v8I}Uu|(A@t3dBL`5 literal 0 HcmV?d00001 diff --git a/python/skyweaver/skyweaver.py b/python/skyweaver/skyweaver.py index 61eb04d..263999a 100644 --- a/python/skyweaver/skyweaver.py +++ b/python/skyweaver/skyweaver.py @@ -14,8 +14,6 @@ from collections import defaultdict from matplotlib.patches import Ellipse import matplotlib.patches as mpatches -from adjustText import adjust_text - # 3rd party imports import h5py import yaml @@ -703,6 +701,7 @@ def from_file(cls, config_file: str) -> Self: bfc = data["beamformer_config"] beam_sets = [] for bs in data["beam_sets"]: + print(bs) beam_sets.append( BeamSet( bs["name"], @@ -927,9 +926,9 @@ def create_delays( coords = SkyCoord(coords, unit=u.deg) mosaic_command+=f" --tiling_plot {output_prefix}_bid_{beam_set_id}.png --tiling_coordinate {output_prefix}_bid_{beam_set_id}.csv" log.info("Mosaic command written to %s", f"{outfile}.mosaic") - log.info(f"Writing PSF of BeamSet {bs.name} for target {target.name} to {output_prefix}_bid_{beam_set_id}.fits") + log.info(f"Writing PSF of BeamSet {bs.name} to {output_prefix}_bid_{beam_set_id}.fits") psf_beamshape.psf.write_fits(f"{output_prefix}_bid_{beam_set_id}.fits") - log.info(f"PSF Plot of BeamSet {bs.name} for target {target.name} to {output_prefix}_bid_{beam_set_id}.png") + log.info(f"Writing PSF Plot of BeamSet {bs.name} to {output_prefix}_bid_{beam_set_id}.png") psf_beamshape.plot_psf(f"{output_prefix}_bid_{beam_set_id}.png") with open(f"{outfile}.mosaic", "a") as f: f.write(mosaic_command + "\n") @@ -937,7 +936,7 @@ def create_delays( ra_hms = coord.ra.to_string(unit=u.hour, sep=':', precision=2, pad=True) dec_dms = coord.dec.to_string(unit=u.degree, sep=':', precision=1, alwayssign=True, pad=True) plot_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(cb_beamshape[0], 5), round(cb_beamshape[1], 5), round(cb_beamshape[2], 5), current_beam_set_id, overlap, len(sorted_antennas), 'tiling')) - neighbouring_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(psf_beam_shape.axisH, 5), round(psf_beam_shape.axisV, 5), round(psf_beam_shape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'tiling')) + neighbouring_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(psf_beamshape.axisH, 5), round(psf_beamshape.axisV, 5), round(psf_beamshape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'tiling')) bs_tilings.append(tilings) target = tiling_desc.get("target", None) diff --git a/python/skyweaver_test_utils/.bdm.db b/python/skyweaver_test_utils/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..bfaf50da17f0f961ca0965e65c4c716a1776eadb GIT binary patch literal 12288 zcmeHMPjBNy6t|twxC>eD3Pn_`Rt#FLN>^2}$98P9LZU^{s-i4xH)6LSRJq9{9VK>x zJ!umTD+R>fIB?{O5Vw8_BrbdiBn}+;3@~HQxOP=U&((}NnfH5d-kOHozj0{|37Y0zGyU9=S{)CHiUT9W@uY$#jidrfLxSv^p0{_yIp8)r8j zt^PrQY?1-VfMh^2AQ_MhNCvJO150hC(YkwAUw(0d(diV$B*7>Ze3CvuOpCtEm`DB|QLHMjqorBG|7tTKDhkOQW1HhTqL35WjUz&swvEuFSN!RckZa)KPBPO81d!up~j8D@!8A1w9F;+@3|Piek@Jv{Bn3NrpX|_zCuD%>`o#h! zgv}N(!Z9n6uOgh60FGJ3g{1^=oK;+yN{D6~lhJvus0_v=DqAXGShkRJP>M}!%~xF{ zu0R~;1Qpf_#Bt7LVXZ(MAB{8K3v&h2Rb`y!W%bdN92c8Hi%PP+C|s+UJ|=03V`A|& zMd7@vpuwvOy3SkyFS-at;UW};i+#~WC<+&$C|sP2E<#bb2nBx|iKX-RSHA8PeKkXI z55==6w{LtiRyhvG(!?%7}41W6kFV>%^Ys$J|T@t>J`j~!PPEy2z+JWyt%V~Qy^a8K#nvjQF zOK8X@4LKJf+5KO=`Hw>XWRna?1|$QL0m*=5Kr$d1kPJu$Bme-+m5FC$O{L_Fa>8Xj)zyy1ogaX_`LtT+4%2 n;5fedzmjcQ(4!MAk4u(g*)&SIS*{mYj&J+)J44{R0UiDYG&Z8w literal 0 HcmV?d00001 diff --git a/python/tests/.bdm.db b/python/tests/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..ae7ee8a9a50558dd59fb68b09f72265a4d0d79fa GIT binary patch literal 12288 zcmeHK-Hy{n6t{bz4b^Esj=#~rQ-Asa&*a`Mz zcf&=cfXZDj`VLil)t9K$m*_>`ptrs5jP04VS0K155;~(ye7lyv&Nz@BPJex2A*Hq6^iW?fp|Q$G0l-1)P=pS`>M zHv=$$0bl?a00w{oU;r3+-xyeEOO4jmtNP;6VMvaqB&JD7Qt|wJx4YTvq6gdEhe+s9 zkfM}~$pJ;h>W!Q0D4j)7kjyv2c+w|)$uyS+$s`i%XFdu0 zX#1;P_ilF=eY3m$<>u}l`l7qHj^ZF9sMr0jw~qL36k@)6xjN!o%6vVCtsaN*NPt-$ zlk0~kVj4b^%lF5lDx4gQQsExZkfkmHM`dtO20UY_$m4+0Bt=_0UvIIBCv*_9{zU^5 z%2x{*g)uL&=pr~SfiUJ3OG^oaaaOT3l@QA|rlXTwQ5lSBRJK&Wuxz0xpcI?cns+S| zS6~?D1W9WJhH=iNv{qmk?~gOyrMZGxR~e^yS$#64PvoYsqLOScq-zzk*Cb8DnBtF1~32&00Y1PFaQhy1Hb?<01N;FzyL4+47^(g9;%m=TZUU()-xwhGMOt0-&u4``oqw%}9 z#;)Gkt^3cHXNGGpzQ?BH+u8Iwu9;2G@jNz7j7`(z5^iJ9H!at~Z<1^iTWzOf;kJ}) XY_l-sf!(%`osQk{ear8-K70HJ$Jc`7 literal 0 HcmV?d00001 From 0595fa99ea765213501d31f2b68658ee04315f66 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Wed, 6 Nov 2024 19:07:29 +0100 Subject: [PATCH 32/65] Remove .bdm.db files and update .gitignore --- .bdm.db | Bin 12288 -> 0 bytes .nextflow/.bdm.db | Bin 12288 -> 0 bytes .nextflow/plr/.bdm.db | Bin 12288 -> 0 bytes cmake/.bdm.db | Bin 12288 -> 0 bytes cpp/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/benchmark/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/detail/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/src/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/data/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/data/dada_files/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/data/dedispersion/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/src/.bdm.db | Bin 12288 -> 0 bytes doc/.bdm.db | Bin 12288 -> 0 bytes examples/.bdm.db | Bin 12288 -> 0 bytes examples/config_files/.bdm.db | Bin 12288 -> 0 bytes examples/metadata_files/.bdm.db | Bin 12288 -> 0 bytes prototyping/.bdm.db | Bin 12288 -> 0 bytes python/.bdm.db | Bin 12288 -> 0 bytes python/skyweaver/.bdm.db | Bin 12288 -> 0 bytes python/skyweaver/__pycache__/.bdm.db | Bin 12288 -> 0 bytes python/skyweaver_test_utils/.bdm.db | Bin 12288 -> 0 bytes python/tests/.bdm.db | Bin 12288 -> 0 bytes 24 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .bdm.db delete mode 100644 .nextflow/.bdm.db delete mode 100644 .nextflow/plr/.bdm.db delete mode 100644 cmake/.bdm.db delete mode 100644 cpp/.bdm.db delete mode 100644 cpp/skyweaver/.bdm.db delete mode 100644 cpp/skyweaver/benchmark/.bdm.db delete mode 100644 cpp/skyweaver/detail/.bdm.db delete mode 100644 cpp/skyweaver/src/.bdm.db delete mode 100644 cpp/skyweaver/test/.bdm.db delete mode 100644 cpp/skyweaver/test/data/.bdm.db delete mode 100644 cpp/skyweaver/test/data/dada_files/.bdm.db delete mode 100644 cpp/skyweaver/test/data/dedispersion/.bdm.db delete mode 100644 cpp/skyweaver/test/src/.bdm.db delete mode 100644 doc/.bdm.db delete mode 100644 examples/.bdm.db delete mode 100644 examples/config_files/.bdm.db delete mode 100644 examples/metadata_files/.bdm.db delete mode 100644 prototyping/.bdm.db delete mode 100644 python/.bdm.db delete mode 100644 python/skyweaver/.bdm.db delete mode 100644 python/skyweaver/__pycache__/.bdm.db delete mode 100644 python/skyweaver_test_utils/.bdm.db delete mode 100644 python/tests/.bdm.db diff --git a/.bdm.db b/.bdm.db deleted file mode 100644 index ad9cfd5b69b9ad4dccece7d52f8f73c40ab5ce9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHM&2QX96!(54yW4iNEhy@Ws=`%OwTaTj9?$rL3W=m$RZ)`=lC&i%2)S9?wX*gu zwzo+*AQlNuaDW>pB*XzB^-ti!5h3nSgv1|!gv1pI-mGU_JK9P;ml9)je&7_wl;tMe0_ZHq3;|FoiK@fC+5%V zTixZIu6$>;dspT<*^cGd={tLg%vN_6m*jX91a@@P@xwvSp-YD8$c+X8Uze?t4<0y? zBS(O7G<055yqVhk<;(T)85+Yw2IWPA(Xb%oqbW?}LRyq39vQ|lN2ph=&0o4yAGh+3 zMzIrdI69u@hU%Zyi)vSD^A|7H$IZN#o}W2^;WQthepVOCCm)n+XV0B0eYTz0_xlb@ zmpv*BeP?-Pt;=`jirFiJ70-g`2P{r9uw_xt-raj>@^b8|cK!8HR8^1@zTpdNEU9@R#=xQzoen>-PS=fg;!4GM~*-TFG!%4*1R89hK{bc`%uOKh< zlOR7+5WM_M@g@ZK<{dlljht}L2}eOXzwlANG&z7DzcM+1>`TZV<-!4E%IEU{ve$tf zV!qV{5phb>S*d%zZnz+5EftRdbW-MH37N z1_T3w0l|P^KrkQ}5DW+g1Ov|#0~?LDbfKkH8s&1i(x_Ie%~N&Sn$3>qC%zXBBFDvs zLM&ZFilQP@Hx*(i()gPfS6=!-`nfJi-j$y|b-(`c7uwhBZRzC}p?IYt9#s5ptam#> z&s7PwOjA`cA*P{eNJkc7c(*BB`?rRB``3u}ty){U%)=`c!VBRN=k^B&o&LacO~W*a zj!lbTtP=>`Dcenk6L^{GVc_3hBRn*ImdMHcK3!n$E11rba%!(^DXF95R& zu~s;_>y@^2ZfdM+tJk|5+g(>x3{_WD!$8EsnxPv2&)}*{6JB$B^-j4hU7HFwjn}>5 z&{eTwLE;D!2PP24s--gc`u6}o<1Laqyzi9S(!x~u>+ANuv*yQ~S7e!3C8mxIqyf8F z(R7B|{0mTP-r`ciIiUCaQtPAyf1(Kn1OtKr!GK^uFd!HZ3D?HqL``$_xb5MWa%ct ZP)NwF1C)Fts-`L!X@;h28qvUk{vT9hANv3R diff --git a/.nextflow/.bdm.db b/.nextflow/.bdm.db deleted file mode 100644 index 4ecbe7d1786bb1b6bd37c0995fe46d526466ef5b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI1-D(p-6vuay*rb)1iUh$@94Muxl(^Z=ry`h0D8>4f1~v3*n@rLnyPLSXX;Ux6 z_;u%n@8GR3;e+@FzJs&bndxp?p-4a|XUMYiKWEN4Gr!qcShDwGm$(>p2ezrZNRyT& zS(Y9mBuP@7UQv4Swm>^EF6dnj^#9h4OIhV!l6n^wThg^}SC^N5ENw6Tq5>Ka00KY& z2mk>f00b@@fk|W`o?cx|O!kimJ|1GrwF!3k{ZXS{ZPwAto%$=pZAf>JgALqv5gVP| z+(ORCG!{vq;_KQ^!j>Yk_r|YIoRguy#9G*Jo`nfUj^J-A@MN24O-bLa!S61Y&d#-+9V76R#i?Zy^YHhd9cjgZ1pq;19`gXm6o;P-$ zR2wbyxZc`AmTqFytiNts~DK%Sa)69LABjyH9F&g+a+}Ud;tU38_lIjEN_Zz75X?J zh~>>#cmjf0(-{j_K+$T3+dr*k28H38K~G5Zf*$u&1ai~4`l}X&Ly}lNAmI&3V)-nE zHzbL5U`)9ScSzG!O_}=B>foV!C^m&o%AW3p2n}ocm^cox+#FkzsgJ`3nmlZv>+}uk zYzQ}q5N;45+H43nh!AcNA^F)5ZV(~dz}`lD>FoXGuiL?IM%Zd&Yh?QV%m+iB9>C9c zrw5R|3E3?YI)Kc0IS(Lv9M~b|TOE=rN9udEu*?Ufi^t+gqJur@KLZ)>+nj#%LjwXp z00;m9AOHk_01yBIKmZ5;0U!VbE-?Y``(I9em*@`~5C8%|00;m9AOHk_01yBIKmZ8* zX9AyE@_p%cst}PQ5jnaTi^Y=3R7#t+l;3=zX{m(QqdxXaXo;*v|LyG>gX2z4DHn?Q zQc2U)qLR-QbH!r0`a61~cI#&(AxXW=`~6<#W7E47rJ|~uR#FwMST4|*LRl%DHI>hr m{(Y3&k}u>dikh!zrAk?;D1QJa;JHr# diff --git a/.nextflow/plr/.bdm.db b/.nextflow/plr/.bdm.db deleted file mode 100644 index 2211b5050431e4053ce512252953f14d51e6884f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeH~?Q7FO9LLiK*K|@l8AWgm4#t=rVwF>B zr+2>a-w^yK{6qW?{2#m|cekWd7}6Iqen(60`}^JR;XaoHvioeCco-e_9n0{LCe2BS zgmfPvNs==3pQ3+0X6Pi%1-&LB^S@0q(z>3_#j>{=&262Z%q`METaR1%roMxo?rc43 z?Chb3`raC{4GW`|{<5`({L^65@bp5naNa+6g2Kt`AnoEn=ev51u#wa3) z?RP9Z5kc&rW8sP@TFvx&XSJ-TFg+{kiHUC1&O(I7HN8z-m)M@l_GB5OxPc~*8|XemLuoRE z8$<{^BffR^{tEZ);@3lLx3N96LVxkSnFt=hPq%{y zki7}nB^i4FnelQyfb4c)4>3RLm@ILmp;wE`azuJ~B<>`-*q7lwQ1rje=|?{dAOHk_ z01yBIKmZ5;0U!VbfB+Bx0zlvr6Y#(P6S;2^{lNeNKmZ5;0U!VbfB+Bx0zd!=0D=F4 zz{lOhU1>gFNhXrXY$}^hr*pY{UJFJ(9HG#(yzI}YkNxC|JTRS(TB<3UR<2j-RaM)l zmDP<(x$!GG-@N%lmZWa+-K%c#!@mDll#NnJ(`qF}tJW*!TCGx7YUfS$^QM1arG#Ue gDW{ohszffBY|6`2rCzDh!_>4=RVk}gwOXzJ21JCnKL7v# diff --git a/cmake/.bdm.db b/cmake/.bdm.db deleted file mode 100644 index dbda81b78cb98217787608da6b38e96752797c7b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHLL2ukd6!v;I+1=3129+oaRV7zd6jFuYnXx@K6%q+rMTJmGl0u^jA!pawSXtYP z?M)L7NDG1+AdXx(Rfr3Jf-4d~00H9AD+dlp9C@~996L#+UMVxm#Pi;p_r3SN8L#BG z{m!NvJIa0#dUmW3X-=wCq}LQhlB7D!Doj46VNv4(JS&CuirJJ5g_Le6X+(x`CJ^q|>hp_>jY+#0XdF2q zhfCHMg-P^J8U@+a`uz3ljY+E*rSIlJV0lptP(Nu1_4AMA`ofhfl`pnqdvEBla@nKG z(0A6>H@o~~UUd7)t=l`@jqa9mXY1BmYg@a@o88?N#kW02+3DWfSy7VHV9V_EGPRc+ zDfk*1>>j%QAs5COW}+XQE+hA&eEZ(;uq+0>!-%`vu?wLYuvZj%MInh;Nb=Z@<1kXz zH{V`|jDvXJh5C5`L7eR7qUZWaimVFzxFEQGl5uV+2(F)IoSO;~L>tD3r?I@EFpRyT zr6dfBmiSaCU_-81)p_8O==vE!xwRy^e#T{PEs3tbH%xiY%_SMCN;%Ea>N}(OAU_mH zDooEJcP-0sO`^#4W5jCm?BlWmz{?7#GgIKRE?kkja7FGy&bn|#?!pzhi+0w9D{>dE zV82GZboTp|)g3t>jvc?}_+u}#FMl$uq%YuSFQ+dc`z2%#x%2`u<RzQIp$p-mtK|57xU#$V3vD7-WVupJ+aj%YdmhH+`F1)MuB8LkjZDG+H%KFG81)TkX;6#| zovP3@(liZeYk$f~zi0+NzLQQ#`c=RD>9(b67D*>`g4ARZszR#)X}YDVD#Dsh5rr0k qilIk1_@z|=CKLGUjn)+lxW diff --git a/cpp/.bdm.db b/cpp/.bdm.db deleted file mode 100644 index c91b928366f3b30bfc88e45704c4df8289c98571..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeH~-D(p-6vuay*la5?l@bI?VW5=SQrhiqJ{7@KLMc|QF@6-Uw#g(NlHJ7JNt=2h zM6GvT_#k}=!I$vDH}Dn2+3d`8H&*CHE~K0xVdsC&oR9gURgcL!Dc36KH6ob>R0y>%^_SzgwLH~k)tc&e7$ z?e`)g+)cz0YH1aXD)Jm-g^WfbnOR!WhUsvjzK30r&YxdY#HYs#RXs){uW1d>WiyV7C1uWwH4zTiqc}TZn#;)3>+IryhAHmmhT6 zQQ57xJrQnE!bXD{nbL+#PM6?yRZf! zK^(th=?Mwqj5?OCkYd$3w0&O74vP-8!=8v}g+26KgmSZ9gI&wQ5lNf?kn~0*aRQdo z8CL$ z(F4feg!~eVJb>JIH6K8JJMf2C9Cbt*0%_niqcR(k6c6N`#1{K9xCd7JZ*%t14+97Q z0U!VbfB+Bx0zd!=00AHX1b_e#xWWYd?|(IQrm!CjAOHk_01yBIKmZ5;0U!VbfB+Eq zUkH5KQ|~Er>5Z5gi>dMHL?V$&rPJnUq=OL%(@bmrjQRLaGqrlJ*D!P0LQ&7_W+9g^ zmW*tn{3|}Ub^BXPQ@X6Qi5 zr6nmGF~)deAhl5}EuKDIKCGn&8u(71!?FFWVOalX9YwLnON*yYl@F`wQ5tSC2zIl^ zK=)@ILjK;HW2Iv!P85E)5!kzJC%)bIXFNvVys&a*jYsB-ZbQ2K>gL*|wRP$G`sJ$^ z*0-fs*0#?{p51Yz&9z&b=Oh-5csGh()ZJkr#kEw!*=^U`=f)tyWc{t9UElq7a{6w2 zf7tBS_kDi39k>`e2JWPddfH$krkp&mgP`wAD_5?qz~gSvbaDUu2D$+|o0}ciV=l&b zVIQOn*JB?J{gQ~fX&$)&?K(!}>&FHqup>evUv z9RMHhzXVo7j^q;Fkz7Kayo7fom++3{679)Lct>&x?}*ii-+HWH(S7^Q8w1Cy zJKmrZ9Y6b4yAT%O+vmdqjFm9{85=6Vxbx{Oz&IV^BIcnUGF47#bd(I6TFMNZgCvu1 zGBcwbILqG6rQ+{|+OKMNYtys8oqc__Sp9kRm6?CdJe*mY{&0F9@9ZB;ZB_nL`AKD^ z{MYi^Wm5XM^xe|Zw~?fqRO&PWQ|ysrK=QwnkF3nY+>d2AIr~_1!1Bk zKAvickKSW#=Xp^uY9s^tLaWtxT6hTPdMkrwGX}Rbsu`+o=`vAeQz5EGW9-U5V(d!u zryqXA+MO?m!b_QMU-s%<_W14vD~{b^E3({oq9~Sb5{=5LhRu|9E5@(=2KZ(8JcGu5 z%QsJo!k07gFFI}e!ByL{TM4dCG~KXN)zlP970b}o7h_ z+iKydZ+WlRYbqKsOxZLI*@By8k%M(CjN5@%J(=ZZvRKD?TDi??blH;;+J`Ailz+*YsOt*3A`T{leKR1h`Q z0>Sc8b^JQz^F!wI_0NgIX^vYdrV7uy7aZIj1a8}3-fI#hEFp@GSi8L_3KmBkD&>aLaPiXI_fc>$GOK1}fhud3K{SJ)q77p4#GQBx*yC`M6-1b2 zcb*+paCpTFTt9H@{;J)s588InjnhIQGS!JiDN&IYssVChnydc?ekpc7Ot#k+nDbdW zv^t`kQ6H45I#fc|WNK+d;plT;1v>MZd5=*Zm3E#H1w9j;3EZ_o&>I9BK_7w<)t{^* z;V7{b1W_{;S*Hf~e3^R=+;>^qo@dFYWXN2cAZ&=OKE4k{iA|`i8=4_&iUP4?cMwC* zy%)J1<{rzDhqvb#bQUR*Y&wmLEUsQsh?Lc}Wm4U=WYwgyq3JPr=>vds-H14;jpF2} z5?8xk(~UKWm@3hsOUN08P)kRG$JjNdN#)j(`Ykve7UcFU!_Jca-9(gAi3pEX6Do?r z(V)~pe z(Oh^8cuZI*q92RDVC~L~D10r`^P`F7A}=Sdf7CQB6sclCuP|{Tv@tWqzXKn26ixu2 zph!R2n-+x|neJ!cqg=1MJqK$BP9(I7VyG&l7MVj$nL@v09^b!VB*ZtAgK(r zia|27$63j=Km{Vx1B2)?bQd!Z55d9==Llg4CGmD;l+YYaxkG;I5EzRhe`JlSh6=e& zQs)xDncgNd`)g$u-z>Me5Ijl00q$j4h8h@{HIbLeOi?`#be8PNyuMu;^|*7<@#}qe z*J<3uvL6&I${bZk;us3%HOpf2ro|(>@g+bnF~Hi%d*8W&sls0l4uU2@ z)w7D%RnxLeFw7qs7IuI+)mkDlB;^rn*TzKQ`Ajc2u|yZg3Xh_!wt))$LzzHDVkalw z1R$%z-Z4>vj!vo8SB>;0puz zP^4K{RYIdCmOjz2M8g)C6oudm5|)J&O(QzR5(d`gETU_QLGW=2%l>3UpM`H2vZco7 Xu{8>f;iNaf;2bJrd4*zOTQmL(E=4di diff --git a/cpp/skyweaver/benchmark/.bdm.db b/cpp/skyweaver/benchmark/.bdm.db deleted file mode 100644 index db73c6c42a37555417a366b82c31ab1ae591ec79..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNPm9||6qjUot=+^`LNHwtLZ^g~ZdhE6{#j}XbT^o=bknB0rtQ*0k?pZp*pgjo zoOpXEI3c<9&||M9r-XiqLN7h`T=EV29Xcavw9=Bmo|iXxtoM6w-kR&slj@Ub2=XJd);r5 z&>=rT37L`sMfvFV-EEXC!qAUT+CelM5- zckZaGtGtHCd8lzbTg)qQY z7VOs=ty{O$RkNIB7!*mE__iEOeclxMi=UOo2RClWzZ_D(KPCBe^Ghv{-rL#T?~0ZA zI2fY6FMHhw-2?RKVDI71!4dkRd$f%rKP0Hv{jRr-_-f>0wtAU5^Y*uSF z>mqRl22n;(VXeR*%D6176&OVQY07(Hu3$E+l+&!NA(_+TVo_L8ak>_TYZbHGBuRpZ z;(Sh_zpN@)@T!8%GgBC6U4)`=5sJdaJnJG9g^N%WF4kEWp(tF0B7Yi*sprqHY~Bg^ zaY3R1i56jI-+DTg(+&9ilXL^-Ps047R5oC)yqFs>za8>TES9VZfaVKwQDUcZLN>erd`$4Od-BiTTKu2>~C6=AvQ4f?2cvIriC5HGz`z! z`A^{|`Wic8^CS1@uNT+HThC4m-SW(|JFczsZtAXWyAH;NuIskzS-4|hQ};|el|0Mp kc?ZAITa{I|RFzy+rKE~dk3Iecsvv=-M5Lr7p>0%&%GsUWTCu&0 z?QPOZMJxzz95~R5D*_=Q@lS9pXD$f!%!U7fH@3&Iqpj2fr!u3R^?Yw;zW3fY_DX)c z_0hT;6S+SOeJ7SR;iMpn!rQVe2to~d1v+apFsL#BSFtdD-moStD{nMlc4l@*nE&?p z$+=(WuFXCJz#CV9E5H@t3UCFu0$hQYO@WCtQ)`_&SD)Ou?~?ry3F6Qt5xc&#*;(1{ z$T!zIw`69Mok)&|N4l|`tzN#oBu8W4cftqDZZPZ-x@44&`r*)L>+)6d{yh>BIRuUe zBXU(49ILf1U8+w`(HQP$RD*Ch9+f~bn$koeq|0*RG2=MmNbOZ?t&11ylV;J;I3gjV zqvL6AsQpE~Ag5AmUAR!6G>Tq&Ztetz(|myWMO`Rgec9!J8@rk_MxxKw4)6>X?>FK3rpB^df zH88C9++dG^amq>UgTrCuKFpWzdV6I!?CwQuyA!+MIwSUrpj!mgVhZDX3%#78e*1(b5HJN;FNw>XA`=FFzCrDooBI z-&)3Sn?#Wt#3;+jcMi%1ATJvr&(xqE-NFp{7G}t|Xh*j&L%xL>@-6u27G}t|Fhllh z#8S_GztX%T^68iaT@sA_bbj#*PfQ-budXByVD?LxU8K?jm>FNr2Qa%GvWJ)*bqT6W z(sWiULtKC{ImquMSWIDh4=mDm^IGGP(EO>{ng4tK^ZC~re*ol-E5H@t3UCFu0$c&E z09Sx3z!l&Ma0UKL1#Z^b!j-08nUy4|GFz=y8z<_tH5yy)U=$9!?-Qp-!sYI0)JLkO zX_jhg7$MEHlaDa@@$~Aezlo3Pf-q?RH8*Jg{+RZSnYM7gsinMPk+(o-SSm6sT|-#6 zbfl_C#~Gir3w#nK?0?_>h4$OW+5%#MmI$wpz1V#ZDn++K_#YWU9rux@8;Y%BLj`6F zsfulz8FMQD=Htas`_;CvF*Ve+pgW}1%piW3^jxT%hLJlAZnz@?l_)|TskUkBx`qw# zXPTyB8!UrYaKX=aRFXTDwqQ=pV3jyN{rj7QMQ&s$SjV8Zj&<8qEa;3}`5DO10O<+n zv$TIAwS~8)CP8axclQ%%?sgr|+jXE!mz<|HELf zoD<-~8&`lUz!l&Ma0R#mTmh~CSAZ+P72pbR1^x#LOd|13;Y?GLL`f3UYD}ZqY(YD{ z&`jFG`J^>ZHS4K?eb@DRkHn7a^;AVspj4n(22>5OshGC1@{dG|GPVnYTtOekkiS6DL)!$kVO(2WY- diff --git a/cpp/skyweaver/src/.bdm.db b/cpp/skyweaver/src/.bdm.db deleted file mode 100644 index d6a1ed30f03308fb941481998247ccff02948828..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHMO>Er873QvF?~k-vD~cpGf2I+*jvZH?8P4!0L4ZoK;<~Y8N0y^VMp96_+%=(H z?v~`*mJy^?isY66Ii)_eXmco96hVLjxd-i~?X@Tn1Wnyzfn0lS-jFjCSD=8L4Gwm| zdGqtVH{ZNB1mE6x?TQzINr`v!~4Ih7VJz^=AV7Gy5GoZX?v*>Xy)mF?(eDsz5hFth-gVt^+mphRM zss(UAE3~o#*O)Q#o)bkusI6SNwgMIpqK*gqmm3&Fe6|$(p3hAbyKwGh1kdLdrz07` z^J9zCp^OCDdeQEIu6|bNMg8o^n9$9RLohlv^JwG8R9UT)rKjAVR9TPo& zrx){{4v)#OtC-WothPIhZl{L=Mg{R%q)W$TaFa0f{0NIZ^__d;9RNPw0sBlk$l)cj zBV8gp(k1%gC9)%3B0JJ0^zahdkuH%P;x>|77x!1P@6i3}$n{&UKk6srXW#9W;sAW_ zLL7kNCKQkISOAL7M>7D0JBT2bM?EGINog{wkBc-TM((}TlVCCz6CXIszs(n_uPDu5 zH(#Fl@63lYtnqc@=Z!Pd|D3)*{mj%~ro5?o{nPrb+W%|6tG!nJU-c81T$F-=f`Nj8 zf`Nj8f`Nj8gn`WkRk_e)6SZ==JW-mQoNP>2d1*9mc->(zX#L1_+HSDa8V)-c8yHhU zbwn98Y~41Pt?Yj?xAMdn^?y|1?}D!SRlTeJ?U%e>pI4PrxxiUOWFp(bmd+?62GR-B zsX(k@Ks+W8KL*5)w;op&*3@%Mz38?*_(~5#Z{T0=Ieu)dfsh4|6l2?_2BJ2R(2YL< zbcI8A9;mIu<6+{1&jQS(4XJsLGjKCN=`c z4~GK*Y7oYl$$$XeG>J_xmF%AU8Zb`?dmn~<89wCKMeL0_CIMUn4o){rV4c#Gljr^b!1Gz)?Wv=TULEjElOL_P zPM`mInD~c{P2d6i!#3eesflev5OU#jz*fPNr=jx5uHUMwik)k;#7<)|0XN{vuqnY7 zC6;bk7&93z%YO#&Q^N2#z+bJY%E?^t4M>e7-T|I$La-npniyEZa7~;iSL)cn(nI zWyJmM!RE1}Zu(8v>)wvS>~(}}8)BMS$Tks#iGfmoTlg6OgFnFKnb7>7?a8D3O}s+5 zD{ltyk9Bf{gkqv&2oIZp0%}>N48Vm;hp-=RPaJjCanNg{XgG>)L;?77LI}c?!P^Pz zwn-oi!9^q?PrLxwGhp`=kP*Av8)fbudDrN5Z|Fki%v~M$9j`0ieI{iKm$Iv7{n{7*H_5Fp3bin5{#;l3*PYt&Bp;HcX2^-Y&n7k3t$Go@WtC XbplDx@q!I%K}hwUz3e~05jb%~2oU1RiGP4sc2}p{^au`|pi3!rz4~~+ zdcXHxB~`ih!k!noW`7)voXDicd81S^o-s|sFltaMP<2^?MpX;&Ev4FbYu1cS^kfr8 zmzHlDYwtDAul#rA^72<86s2IGV4z^2V4z^2V4z^&x5mI}d8xMX&_ngpm*4Q*+Y{H1 z0?!TgcW1w|b4@UL>QWWe|Y8wwfTt985QF>ki0?j-O=>DY;=pTP8*Y1Dt z!P2|eBIj`E#^%PKO04~0YkRMwJ@c5?Gk2do=v?mXo6qm>UftTiX0GDkADlHq@2|=9!{KpO9CwdHz1)dBfQ}W9 z(n2>aD8$qzPn;+ULUViXxoz0-IO=;~zcyeTso7c_dA{0WY=v`@5!{Lng zWH=*(RWnZK+v>TK=qPatY$}-fB3YW1Axy&1^CKMFG;&U|9iTkh0d~$i$l?;+ku1?2 z$r8G_M0X@hbVss;EiTa=$r9ZWpGMmB`23pN9lEbiUBB!4)6snV!rzCbSpvTO#4G{h zlQ902GYJ@X{x%aZ4u?32wW~8C(Ui_dX;x$@F>+6mNP@{qoX5ZgbvIvMe#dBi(0X$1 z>$Nx69%%l*d1Lj5)qk!&(fF+4HI`QXwX(hZ4NNIY!9c-4!9c-4!9c-4!9c;lugbu| zdfT|rvMS5va=B8fR;$g`x+=})&Q<5w-SfgI+>B15K4y$aMhRyUBg7d&0vV?t-Lw6` z$Cc0PhEX20zg+3R-~P9%ubpojJ1v@{^|?cJJ%{_u*#C>`^xWXUg+B~~&7;YrFC>=| z)H5b2#X@jOVubR2F+u~h|3_7)r4aJ=y*=&s;W?sr$y4thfEFWkr$=A}=Zq7}V$>o6 z+tQ|38)45OK3yHOKV4Dv)>_;6Q!d2S=`iyC3>Ve?vZcKvQl)g`y593F%aSveLlB9I`&28*bm6#TA9JGTUI z6(Fsl2vhaivdW3P=#3y)Z%rd_7;Y+W3kz{9q~z3M7~06v0ek)&C|!mqQqj~uo|L5dvebk^%|KmRu9{&LUAS*V=~6-T5VV=-j#L_|71sol|6SAq0h_ zAQ%hAwDR)aqVn!i+qlxgIZ2)4g$#;iNx_9J!O`GvN-5J8tj8cKuV*=N>)g*K-Sq(e zcbv&;(8-VusZA^?SiCV~@;;B1AU zq6mIh#00H;2PjMFV@cirORa_hKSe1RC>SUhC>SUhC>SUhC>SUhC>SUhC>SUh_&qRi z8kC+gHd?e?Dwj)@_glt^5BtE6}P#O_jP;R}WG>0xxL-ZrsR1 zluALRMYg`Ls+ZY&;o{wY@r&EDN2m*L-N80IOv2S1Dp!wJP%X+BMi}0gU=SffiICWa tS2852WkE}F^(KMu(l;L^vw4Crxb#cVk}eotT;aJ=!sY~{oO2snKLVuF5oiDa diff --git a/cpp/skyweaver/test/data/.bdm.db b/cpp/skyweaver/test/data/.bdm.db deleted file mode 100644 index a6ad13ad3ab19348c3edb844363b2884bacb0401..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHM-HzKt6t5?OlX4P3y3$#V$G4n9!DhJ#|I?H?gDb*l%^2b#47fd+FyFuUz`|(w+6c z7=Qr`00Y1PFaQhy1Hi!Z#=y#0>-4T)H&upfP6bY8i3P5-E z_0w~)AYa$hkCW3ToSmE&(mkexr51rx6`ZJmXH<$@#+2p-+B^7Yk5xRQ6T$X0J(6kc>K_r!-zR z6)bpD!TPKe?nM`=s9mI@cJVH{NJZ@;6}5|h(M2k17pV|mBiXw6epP)J_^SoZPH?tJ zEBn@WiC%8NAKolCp!gDshtb%8Liv1dK=C?=O)QtX0bNOHW%ZiSS0Kg9`b}b!Ijr7+ zEq*rFt;ZVs!2kw;0bl?a00w{oU;r2Z27m!z02lxUfPrVrz&-t{_J$R-)(xZ8GTQBS z_p-@Hw@YzBzZ~NvJ}Y*PNj7mpCyLxK4Ew(4M1x?^j~s3F%f{ZTzqP*o(PXFnjp@}t zFHb)GW67`T{NL6r^`HG`+bd)T27m!z02lxUfB|3t7yt%<0bl?a00#aS23BYK+gi^G z4BasF);d33cP*>O#^$C~j?LG~v9V#9l|tT4&m6}z9y|Te9}I%X^TRN5Z7+&^d-v}) zKXtd+nY%n~|NaO~x4wVu*nZ@dS2zf4ez`6?u#3CxIJPZhr|*Q3?FO#>Uy>Vz1K0O` dr!OV9AFv+_r4HAQ+(F>kc3^v6H1MLp{RcVRgHr$i diff --git a/cpp/skyweaver/test/data/dada_files/.bdm.db b/cpp/skyweaver/test/data/dada_files/.bdm.db deleted file mode 100644 index c9d5502c5359a6820a0328b4aff41e41d9c146f8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHN&u`;I6n30!leS5{Wff7VR(MsdN|&lTGaip;SRv7(XjN^2rCUKO4wc(Dsnysi ziL=|~fRsh_#(@I|t_X1f{s|;5ocIG)NQg5MH#qVf|EOJUqnyi(GV*+HX1@34n;A=f z%A*gq+`!RW-JWL$x}jWG3I*kDT~`#P47CU~DO1oWB>^=($cmdA_sV zSl?;rA8s~2(i59*_w~NhaSj7L9=&#FRqqcx&+a{4bNz1935T3StyZt=CF9aj`jaE4 z=jc7)czWWzh1%E3_2uQta6XLTaZJ_jbq6Ookc}qSkqPOJKJplITyR91rE>lD?aENg zIvVtyUP2ePU*smVPwEBR#d3XVsWPl(y)@m_2{d2i1I&}UKxdz+<(C&13tv78?1PRI zFE@S^WA>f(jjcwqGq1Z%ee=DY#{I^&{&0Kq-SzD~{Xt`IRrhVr(RUiVJF9xQ8*ww* zy~yl^TMBa-h0z_?KTg0P;tcKW^Jd@uG#!4>InKlG;c-8iZU-*7j)}c2ILw03V^j&soUBZ1Q19CZCi%d|HKUH_mn(mn0YVOZ5j(l8*8@Qb^n3mCr%Kr7Aq*1Sthcp8N=QKS@FgS$K8CVIw&1b7iiuS$sVD9g^ujlU7{-~YR z-l+ao9e`ODsen{KDj*e*3P=T{0#X5~fK=fBrNG141?4r(C{C+tv8a|xrCPNTmRhZ8 z2liUiZrUwk5{n3CnS=^%31Km+48Oav@#@dZx>8Y;m(=#c^J?qcU;hm2-pqn>S2HhR z+&?@T_{aNJ#Ec2HIHuI1h;u}_g$a+Ti~okyh37y$n^I3tKuuz512NMO1|ghU$RbE! zF^;-20X2%Lv4BtoCJ33rFa@EOHIBMG0X0seHnCwa!A-*u)F9L}#!&B0h18erUaYkd zQG*^78J58)H<3j-;$xYwO~A}zW)OpOVPK13K`g`s8x!+hX#zRSSPp_HMu-za7*tFV zLgT256Htqo+7v=yW-$&Y!w>`x`PlQJPC(5QYQ`}!xM5-@EKadyjyoT_g$bml5w*!U z0ktU?oSJYSIURRj!tejW+&>EZWRVI;1*8H}0jYpgKq?>=kP1izqyka_slZiJ;Mr8+ zo^new)q<)Piqj=XYSreoI+RxywfT8X(-v=y%8j~K0hB2fvl6Df_D6q@X<}?KVi}xR z96>e-a$qJ||4R)sUsXs>wZ2*SvGvQ@a(m_TA23E#7^BJ{zY$Yi5b356s~lr$OF67b1SX*e!wLAH!h22}Y1p;FEvA4M4x1Q|k5^dEecbBq80 diff --git a/cpp/skyweaver/test/data/dedispersion/.bdm.db b/cpp/skyweaver/test/data/dedispersion/.bdm.db deleted file mode 100644 index e72a2446636c9637f0144a7c4aa0253343f92d4c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHM&u`;I6tZOMbck z?k@lSh4DWPYtkluxdGm#Re{;x^<&C~_z6AV%5j%{vhG<1m;_OQ5)$lB2MtJ1Aky)>t6YhSgeYZLL0U7Kvt&8}jAY z_$(S`KQD{wRBEkTx9amok!4785*VJv0p{~k;3uCbwHr5Y%3s`%o&Lnl(#2dogIT&JvyV@Q5&bxy-k%(oVK6w3c(@Z&pw1TdieOL#jIprE6DN+t2yO4ay$w4K z;t_@Xc>+PqW^?FKpKUSA!Z|4b>a!i^o&unLvg6!UK)^PMk59S0BACQp(NhBBq9;BD zg|%U?Y1TP$2~s~5DEF2i^;0c#ZwXSrKS^}Y-6afJC7Pz&8oJZ?C_fa~RG6Ga9$Ln5 zn?w=yV>Qdkb56(9K@zy5e4ClBD~FC`CP_9o0OO6dX2j2H6(%x;J5A?8P2f*O}J_3C9v z3NUt0@;eD8Gnn23>+IXSy5dUBubSG*FE9{IC?FIN3J3*+0zv_yfKWgvAQTV^2nB=! z|5t&2wJp8YG%CxAQmrV}YPE5-&RU}}fC|`Tb9ewlbAFPinZRuXhQ8@?4p*lEn{FF&2*y?Du zVQ98(63bF`J7a>MyMai43Pe{%U$?(!y|1*Tt$!y9X7OS+IwXKq($P#}Vyqfwhh!YV zr;J0lipd9#$bsaKNxo$r)Ch7NtDa(d^|H~_XN$@9{P(Uak z6c7ps1%v`Z0il3UKqw#-5DEwdE|vn1>+&noTGLQuMUgAZRVYX_n#~rpXIGm^TYWxh z*KaiIse%8m8}+ne;f4_n{2>%63~X5)(>9^$x`lOZ>raK1DK5dsc2YF>`N<38_0PT| znAm#KP0PTnt13}DQ0HL94b{X%(=gTu6bLLR5r9d@wr-egWD{axn0%(qHZ6$K!JCd? b9UV#;tZq_jzy;L}!`6vu+8Wkvo8Z3z2~^yS diff --git a/cpp/skyweaver/test/src/.bdm.db b/cpp/skyweaver/test/src/.bdm.db deleted file mode 100644 index e2283a31f947f644e618d2b4b00633b33def48a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHM&2QX96!&hn*-f$=Ck-Jhs!B$slvY*ncs!mlZ6%U)DG_a2(zGRNMdR#xw^nw& zi|x$^2T(voNC*jzy>LKC98e|x0WKUkz=boaIKioKK!`hU{Izzvm3pOkl#%`B z9a|SBKcAQyzdwFq>{k%7mNAeqkTH-kkTH-kkTLMIF|eN>EzTZ2TH3$4ZJ9eg)Ak(8 zbff3eYHeYyCS6~y-H@U&$#5mtY@2mY3RlmcIxV^VPRDTe<}JJ1F!_>R@YZy?ooHRW zO4_+&I;P}+j=i3FPHr74&Yn0?+Mnh&Y=^2^PPgAn2}x^G6A6<}Nj_qz;{cG-C=_Rp zA202TNua)KI*~g5J`9GBKPd__MvAk?j+OQ+NtA{agTU)B3>bb=2>GM;^2NC$M{=KD z^^DE78M+%jMndgt3yT+PQDk1S8q)I0TJ1t@Rl2ged}(2IU3#Ooep<4Pjw!9x-dZ~? z@o0pvLG*&*4i72p)l^vBw(PA)=m$Facx&+LTJOfoH``ljv0LAAqwR)g0d%OilN9Pn zfg>i2+%-JUaiztJmlwg}uGh4{|Hy%^$5%&U$FjMJp%=z(La=OZaXga{EZes@o=Ql7 z*7mjrSe>NM_BzR#l+a4fcmpA!8*B}{j)7C6Wd{VsvnkQC11{s)lxW$TZJ+meIwgZw zKBs|M4YTLniGu=0IeskSt!Ww3#C0v(L!nO{V>dkj%F_ejXE2}~+!77MTcUw@3qH6d z8i=<<1MwF1;Ff40-VzOjw^8JJcz*?cyXHH6)2^F#zZ1-#{Ggrl58&OG{R0@@gyAEf zI)LHe(|G{HbO;Y|6zY_yM3e@zI4!CP(KC1BOoGK+407Nk|2Cf*dnkxsh-W7sOzun` zu6$d$R{pE}S$Tfq=Lu(GcKo~X6m^HyakzK^)iW+E=We3E!1w5{_z&C^FJz zl`)-Bjc6fU`4fOEwyF=x%`boajrZ%5RRM`O@MtXQQOg4dZo6B1JEpN6IHFONX+$Q9 ziWCLuib^BJ!d+18>%I2NZWH>=N>!jk-8gTyjlD~TZM4!HQ6?)ylMz-hVT55h(mnPM z=suJ*`c}CroQaH1h2e&2weEQCBkD0`L{l}YU|FUzVTvAUf97M*&Y3R&9l!H_YoaQg z6VZ?$U+T9#>oxeMUp0+J(s~UMU6ysEG9v515XK?lGk=5j@uck=<2;2z!c(>nQs>c_ zLds}}Ayoy-DF!$eAw2s7AVBs!;2Cs}_uFGtVPQBzzuAN=ZJg)XV?)_!kTeidO-GuB zDd+}v5K%%Gs#^e10mX5OZkDRTg`ogY@$oFzv+C}W(XRK~hSyDCAWU@y(qCnWs7Ti_ zi-It>8aOcs!g{eP;Gy}m<`}ly>!z%S6Q=5_EGsg_DnUr6G~(y*`#}T-sy9X-7lF&0 zpmiHu=-RKRBHZgWG1fE&^vi^5I@Jjrg^-|90}z7zyjQ*7gkC?y2^w}y49@T%+Jhh< zhE<3IQ*}xdCR2@45`w(&2Y{q5ih7|cEDw#s03^OBlA&fQoCc~e@Q4yfR-i7*))zyr z;!wXm!q37mPvWz1!+~GFlToIMj49EHs$v;wgkV_>^%qw_e<7Kx8~LhG8+z8F3w5Xq zDkLeDDIrJ&G{!U(BAk5~&@{N2)pE?1AMHi|U?c*8WXLFf9jUrP`GuiI zH+E$n5P~Z-j5h!N&xsQP{ADd;AY&k7AY&k7AY&k7AY&k7AY&k7AY&k7;QzqDzLPs6 z%!oLj%ja_=emSu+EzUxlnG>g{MNyo4-f!ZxSOO{0O1R`Y^#&9rWSuEcw?tG^pqxg@ z!k>9wDxHS!ZeCgR%UmZv{08xlbEqvqhjORi5yps6h7lq#3k5j^Dq2t*(sU>-LP3;5 nVT&>aRt+8VXJw}P>zJZKEe2|+>85~E1|8``mL?#2(SOQ!c*G_j})8pYvI|JA5?oQ>1uN z9Jr|hh3kSS3hye4AP5cIYq-<4f(La9@Ka>Q|1@j}J@u_N)~>XUg=^nky}tU(>iyOW z40z)Va0WO7oB_@NXMi*Csxh#XRvMj~H=E1Pp84o(io!JZQ9_^Z9rSmP`pV<|{-+9+ zDQ==9Xo7~RQq1n%-BFTR5V-N8=ZDb8_GREPPxDq>bv0&h6XHrOXDJB`Buu$oR4_ME`ycHd94jC8k9U;N4Ulrwe{!^=kku%L`Re83fGjDtHFZ3D;)kac z%yOSZK0Y5N{ukx+lgVinM#IyDn!BlwLl=$%77STHA~H$N-8798WpD6s4`&>uo{#IN z1w<*CO<~}NB*mf%cg_GmBpH_)2KZr?aj9Yujy6e8&tnBFn4|%#sem!7NzVb}8|Ru= zT?Vc|KgrT+; zGZYR{I16(9_E!@z+koHP%Qj%~B`h9NWdjz%S91dvuS2nk=~7pqMk&p;unKhsrf6Qi zNq91W`8%*p&gS)NzY6lt@)LO#4|wAYa0WO7oB_@NXMi)n8Q=_X1~>zp0nWgGhk-}+ zuCOJ;T1%2@HL+f=x7V7awcGDU^98xN!WX8V11#KZ&9n?%*Pv>fKv@2`vG@9K;`c9_ zg7Cq(`{$bX$sY^S2eq!SnY)y#E{9R*^`;9?*MMqUrfsN>Vc1YrRkLun{sZ>b8sGZq zjCy+@b%k59K@yZ|EjB?v9HDtHPErp5&4v&_!+@Hl8@2^BmF8z-hirC_#jbGg((FjV zV-ybDMKnu$^I+m>jsq5u-i84mz@7a`Ko|_ym1CN1DpZQ0B3+R zz!~5Sa0WO7oB_@NXW;+9z%mx!7B*x<5+zB@&+Khk?%=j@L(bZbH?k(L%gtOs|C_Km ziiRVgL7SYtp<~-7v{YNQ;qG4&xp26G58n80VDA0?To`YEeWuUoV)B zr30wrs|5fo$FP6}Oh?rWe4784k>;4Trt3PepbGGHf`uIaZN+`wpm8RO_)i#-=L$aH=yJ=G| zgsAn-3m?KuUqbLDyzmWt1i{(t%yhS{P$VFfGbG*lpEKuUezUs}=+^U1>JilJI+p1n zU71!?Re6Arq9{rBjkB*96Ks+Yf?d_n{BP5wvV!lXnRjAxSGo52>h#ptsrAX9OuzsF zKmZ5;0U!VbfWT!VFpNzkv$M0Q;j05m4tvD*97c@ zs+QC3_aY+PO~esuX$g%g@*JlMX^liOJ3pTqX2ON~E^$OUe|}yW-+s|5IWwNj&dsHU z>2Q?>l||U}yc*1W(GvLM+gS4E%#8YJ%QN>ngm0H$;yin;ytY{pCv%%N(8iNmWxY~G z&#D`b%hg@5MhuZ`som1kRie#LyXfQ!XQJ0fxnH!*7^G@*tbhw_laF6cHav8i|;z> z=m9*wGkO5|n~+~(kq3|)FXscuZwLMmi=&Q6O&|@tdQ=)A>5+lFlh|Tk2KT_C|J$7X z=!XFWfB+Bx0zd!=00AHX1b_e#00KY&2wY+U{_lS^{Y7CP7(f6B00AHX1b_e#00KY& z2mk>f@V^Os*ir8(vzdHMjm6aXWFnDBr!yISG%~>mgsx{&{*3wfC#9#zz_fboUpX4N zoPqIbf&I;4L(At&80X8s5;xXveT%0QrMdm%sQLZ)cWd$eHpV4Q)AfRe_0?kDC=~KV rTsUhgo;CgZDjbY)rW|Lku@bppvN12y^BOktMhR<1NiUR&xP<=zBuBc} diff --git a/examples/config_files/.bdm.db b/examples/config_files/.bdm.db deleted file mode 100644 index c6df408cbd73cc91d39da0d324300d6e1e323099..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHN-HO{r6qaOntxe)OrI?0Gp5qwpn@+wD#B;R+3$5 zoZan3!7a^QFZvFpcYTRMU!s)0LEoV>(#)*1BqVo%h&vsTzRnm zF9$Gy0bl?a00w{oU;r4nYz(ZdwO04`ZF}|YQ%X)2BxM;Txw?LQFx(jp(bs#!Z;;ZV zC`UPo$(SLzdi&lM%9lwJWoO$ooll5ZvM8l#Hc!+#y$YQ?AsIm#k8!pjA3C!)THU*M z?bTbNhNm*rESoPEHBhalW~h{O4;2|@j0=g}Nwd}6*sxcfYNBOMG8In9&r2ipubKr% zjaK*09edTTW|>f(gp1G1!PT#t!g==YG78iuyas}&1)^Y)`@G7 zrX@kzT7xt#xzyGgr0G#y@UG1@%)2T$Ez6pa1$&}5g%_0-d!b$HnBOLOPE&?uPf2uI zSMcC<1@E&|xEEcNLc1u1cIjPoQ3~y%6xzkR=%N(bMJeRlNVP8CUuEAp`EE(lF-ez6 zY2W-YHj4-F=l6>TP`(M}#i~7kQh7NaKzTdJhgfZO4Z4cb(%P#-uL2o4)prtKEMR#L zY>KnF+49*^hgl+GSI+hx!b(*i~VdXmiY{3c&7a!Tv9^rmTZ@gGJ}&_mqWiU%Gky z`|FS9|71Wmsen{KDj*e*3P=T{0#{9enKoBzt*q2%pPhK_lOs2XL(h%G{o!tBv)j=> z-tK&&3!m;pdgPAXeynG`*YBUn8VQ!XdT6Iobf?kjclt)X@3sNofXbE~jZ=C3!hfA1Mm1vr#HE@sOgM3j~QemWZ?wGg|uN#_*%x&-fKqI3>M2 zB}8os+SuCfQNzk8p@}J>(4ZItiU_nV17)1;w@(=7E58&ukF^4)dWkY2;GD$RHxV*v zS|-G{X&8{&*oL-g5rinsIF;{yV4SZPDI+ykwpyuNf|4l<%{H<2$$*eyN{JAp(1H}( zEDI*BWq^!P{cVFwtNi+7k(7V`s|$ZC>`ykSfK)&#AQg}bNCl(1QWgXGvPua~c0dUfgk z>fa2&00w{oU;r2Z27m!z;9X;2sjW0xw{GdnZ=MF^Sxh3D1SFOBPy4;SK@UAX=zWEx z4*4lc$&8FB60>*j?4WcJhJNyVH;CqA!l%T!G)d;6oL9_3&z_KkpoGPE9+OY-^u0!F zYfE3Q@fx0rP}5|-h%2C6O~p_u=?=;=iWuh-S>t-6b^EryG|GtYT@I#x+Z6ccAGF5$&70~kkElPK5z(%=)kO5c-u_`vuFQwQ z7#(~*=-u!2(U<*$&-VI9=t1vj2St8J(4hDAU`Dn6A2c1 zOs=1viD~d%F?~2Yslxf_B$e(y4OnUsI4pxv8SspyB2RrvlN9YAezDIgp3_Od`j-vN zDW5H27(~28(S?6n0zt$pE-WPwL|Mg!sf1Xz89h166_&w_hGk0yOv@H}21>D6t$Ei) z;tC9+oS?#5fkBjWSy(GDh=#L__rhGktgDREysR;a>62nnSW!u~7KLjSv&SS&gNT}< zr_evGDp>HUg7ujztn)5XQMgD&;bNb6k&41aDhe0pyo*#6E>a=hMzVGB{>uAK$+rs< zjYzZzbNlv>Gd0_QFFwjPpm-CCn^xI?LiuiPK=C+;O)QtX0xd~tZndk>DM3n3izkUq z<}iN-w)xpyHC`(04+by*3;+Yb05AXy00Y1PFaQhy1Hb?<01R9-14P|W9vE(IRnxSZ zTCdleS9Cs_O*)($)5-8SCQ0^hd-nCoF(oNw-@NUP;&@^@4(@g>yX$n^w%f67&$N`~ zZ`by3{IT`Bt|-^18-HDyyxe%jul)S4DTa2@4MIL(02lxUfB|3t7yt%<0bl?a00w{o zVBjJdSbneGRW=P*Q#DPkt@4w0(=b|Wtgjo{SpOg!Yiov{E99qc>v6(8sZ*4Vi_J97W|;@|1^Pkj?Ro}FFou8a9)V$*GBAJ{Z8 s7Pw=XHuh|{W4=l9+Fi?W95zo#7PeTJ%CKDvJFeqli)HV4UF_li0G>vRWB>pF diff --git a/python/.bdm.db b/python/.bdm.db deleted file mode 100644 index 5b29949d35bde710bb557cfeb875939822c80532..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI0&u`pB6vu7vHtQ4?w?ZodRn$RMRZ>NZf39s15(!vUL_#6m0ud)CJG-&6wikQ0 z-Q|F^Ah>bhUqFZ(2mS?)@E>sC#*sTG#Ek8k*ijKxkL??+*YkaE-g`5ju`Sv6KHQH8 zM&mS#LxL>niX_X@+XzXL)S*X<9=xs5PMZsQmTUe0>UN~Q{zjL2*Vd1u7ruM$%H^Lf z-&y~I3TQw82mk>f00e*l5O~@MtdzA*Z*x;!efl864`(4*)$xrDcwdz#VW^zLguL5>D|1kuC#igd5$xlj`bf^hS?vt ziV?Oty&E^wRkvPc6p13V`=}b!{J16bZ@y4EFI~GPe|wLF$5UJ_w>q_|?8BYi{ebVx zYtabpy*~`@1c&H@!@Yx@!z1)waI}q*Fve&Yd@|fdY&WWIxqGEKW?M>Q2R0o&jgm7i z=4DQ1pPYB|=yNgrczV{9>B(8n!$T6$(pBQPE>7x#Rjf^N5fYN+Xm|haE}d~o#t~gV zUqDLOXfER@VNMtDX}#ir0nWyM~I(59xBNuEau zF{(Ai;i74v$(shc&eC9B4B-Y5!VMzCx){O@B7_@6hLlt~%|RNJqfFm(zpRD^P7&q)p9Y>6DgSPnTmdpY>=8w?!Ja6QMWl6C%}+1iuqKQ4GW z_WmzxnnZulfB+Bx0zd!=00AHX1b_e#00KY&2s|qUR*C$k^s;6tvUpqXYFdxBt39n~ zy;qAy`)X<6|Ei_VmgF=|Muy}1hV5CN@0-5o`1G{%*CqBczeZob$M=6ZMc?hm@W-5Y?`FrES8*o OTQF%^`lUhl^?w10^pYz8 diff --git a/python/skyweaver/.bdm.db b/python/skyweaver/.bdm.db deleted file mode 100644 index 8f3911772db3f250d64458fb92d49b1d5256f6d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHLO>Y}T7~T&zc1q2*s;s0`RR=|(xQfhvjdvG>M1rg+YJifMmI#SxV~^dDy=!-O zo!A_Zj06V|2aa43;)3`S9Lt#t2W}ks56tY&Y<8m{^&HK};oCvMjv^fFwy((pA!VTOpGQ2V|5B^Z%PxrI!A7ooH89kEB&bj z{!M^rf&syRU_dY+7!V8y2Cf+X$cgoDJcpiJp^e+7UM!9vtOw;HaBZat(a&Qqlky2^UKmu`O9W8 ze6`xx*r+Y*#VmcCCn3|zauD&$rqI9nQmL-py(@oznD{4S#Fonjl|}Dv@9cN@&b*2H zVDH0jXSZ_z9v|#|uzhd@9(9hkK2|cZ(Fp}^S`WB zCGrzZFd!HZ3uJ+&X|+ti|7%t)KY#ZP-SpbzYQTn$>A0?=w@q*RUzOetTq8&C!6$z$_{rPS zZ7=I>7wWV_%YlYx(qp@>>upOj-@*exh!zW)x#w+OE*l17&1H8hU9{R Vgef)yI-X&>mg{+@=R%JR{{d;eq+|d9 diff --git a/python/skyweaver/__pycache__/.bdm.db b/python/skyweaver/__pycache__/.bdm.db deleted file mode 100644 index e0ff9839ce4f6d500ed074352837584012b31339..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHLO>Y}T7~WmGu~TrjRb?fjDjF0;ZK`1Q`vX)#jI1gmP)H+61gf>MC(g*;wb@-K zHb@8~!2whxu5jVNl{@GDjY=Fiae?2!JG(oZ-6%-8Hky&gGtc|Z`}NGuO0$nX+95GU zhvU%qVx$Y#1yK~80yef4}-* z`A-16aRxX8oB_@NXMi)n8Mtf=ETpAc^Y-of;?rY-PbN5sLxLkVzPsDm+UuZ?w>ys! zlOZoc5gy@QjI!0Odz&bl`o0&=TO=6wF9(b2SLZ(@6;wvneSGDcwU!#LUN;Kw7_AYp$=?7jiMsG{PZsN9X5-q4JAH zQN2p7xwckcG>TFBBo6}9^J0Mbq9K&;zLIJyw{D3~AI09m2xsMHLnZURx7FV1u${R} z`e^%uz0QNqF8XkH`~9umee_;ue-j0skI`P|lf6wucO#poyO)Z6x}{)i8L)apf*}Lb z)F+h>PNxz1B42(m8kXU>H;kCM7ZV7bIrfX7R|GU-A<3B+$6t8z0w4j+IM);a36hL+RRKY?Q9L}2m2d+QQoS9Du7Y$(98RhIoPU#Hps_$p-n zJ?-@VFUnU0`0>UW;0$mEI0Kvk&H!hCGr$?(3~&ZG1Dt`&&A?(Nz9p>5x+F@HnBL(V zvfPArb45b9Lvvim)wceT=$C^k zz1Xil57@<9R}44pmSZZk8?K?dj^e1At|*G_I)Jpp f(sh;vUoT7x+!ULtfHzyy;p2v8I}Uu|(A@t3dBL`5 diff --git a/python/skyweaver_test_utils/.bdm.db b/python/skyweaver_test_utils/.bdm.db deleted file mode 100644 index bfaf50da17f0f961ca0965e65c4c716a1776eadb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHMPjBNy6t|twxC>eD3Pn_`Rt#FLN>^2}$98P9LZU^{s-i4xH)6LSRJq9{9VK>x zJ!umTD+R>fIB?{O5Vw8_BrbdiBn}+;3@~HQxOP=U&((}NnfH5d-kOHozj0{|37Y0zGyU9=S{)CHiUT9W@uY$#jidrfLxSv^p0{_yIp8)r8j zt^PrQY?1-VfMh^2AQ_MhNCvJO150hC(YkwAUw(0d(diV$B*7>Ze3CvuOpCtEm`DB|QLHMjqorBG|7tTKDhkOQW1HhTqL35WjUz&swvEuFSN!RckZa)KPBPO81d!up~j8D@!8A1w9F;+@3|Piek@Jv{Bn3NrpX|_zCuD%>`o#h! zgv}N(!Z9n6uOgh60FGJ3g{1^=oK;+yN{D6~lhJvus0_v=DqAXGShkRJP>M}!%~xF{ zu0R~;1Qpf_#Bt7LVXZ(MAB{8K3v&h2Rb`y!W%bdN92c8Hi%PP+C|s+UJ|=03V`A|& zMd7@vpuwvOy3SkyFS-at;UW};i+#~WC<+&$C|sP2E<#bb2nBx|iKX-RSHA8PeKkXI z55==6w{LtiRyhvG(!?%7}41W6kFV>%^Ys$J|T@t>J`j~!PPEy2z+JWyt%V~Qy^a8K#nvjQF zOK8X@4LKJf+5KO=`Hw>XWRna?1|$QL0m*=5Kr$d1kPJu$Bme-+m5FC$O{L_Fa>8Xj)zyy1ogaX_`LtT+4%2 n;5fedzmjcQ(4!MAk4u(g*)&SIS*{mYj&J+)J44{R0UiDYG&Z8w diff --git a/python/tests/.bdm.db b/python/tests/.bdm.db deleted file mode 100644 index ae7ee8a9a50558dd59fb68b09f72265a4d0d79fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHK-Hy{n6t{bz4b^Esj=#~rQ-Asa&*a`Mz zcf&=cfXZDj`VLil)t9K$m*_>`ptrs5jP04VS0K155;~(ye7lyv&Nz@BPJex2A*Hq6^iW?fp|Q$G0l-1)P=pS`>M zHv=$$0bl?a00w{oU;r3+-xyeEOO4jmtNP;6VMvaqB&JD7Qt|wJx4YTvq6gdEhe+s9 zkfM}~$pJ;h>W!Q0D4j)7kjyv2c+w|)$uyS+$s`i%XFdu0 zX#1;P_ilF=eY3m$<>u}l`l7qHj^ZF9sMr0jw~qL36k@)6xjN!o%6vVCtsaN*NPt-$ zlk0~kVj4b^%lF5lDx4gQQsExZkfkmHM`dtO20UY_$m4+0Bt=_0UvIIBCv*_9{zU^5 z%2x{*g)uL&=pr~SfiUJ3OG^oaaaOT3l@QA|rlXTwQ5lSBRJK&Wuxz0xpcI?cns+S| zS6~?D1W9WJhH=iNv{qmk?~gOyrMZGxR~e^yS$#64PvoYsqLOScq-zzk*Cb8DnBtF1~32&00Y1PFaQhy1Hb?<01N;FzyL4+47^(g9;%m=TZUU()-xwhGMOt0-&u4``oqw%}9 z#;)Gkt^3cHXNGGpzQ?BH+u8Iwu9;2G@jNz7j7`(z5^iJ9H!at~Z<1^iTWzOf;kJ}) XY_l-sf!(%`osQk{ear8-K70HJ$Jc`7 From 00fa1373005ef21c67d3b78a2d78ef6e8d7e67e2 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Wed, 6 Nov 2024 19:27:25 +0100 Subject: [PATCH 33/65] typo fixes in include flags, migration from cuh to hpp --- .gitignore | 2 ++ cpp/skyweaver/src/SkyCleaver.cpp | 2 +- cpp/skyweaver/src/skycleaver_cli.cpp | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d230542..01066c7 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,6 @@ eggs/ core* report* CMakeCache.txt +.nextflow +*.bdm.db diff --git a/cpp/skyweaver/src/SkyCleaver.cpp b/cpp/skyweaver/src/SkyCleaver.cpp index 4420950..36bd663 100644 --- a/cpp/skyweaver/src/SkyCleaver.cpp +++ b/cpp/skyweaver/src/SkyCleaver.cpp @@ -7,7 +7,7 @@ #include #include #include "skyweaver/types.cuh" -#include "skyweaver/SkyCleaver.cuh" +#include "skyweaver/SkyCleaver.hpp" namespace fs = std::filesystem; using SkyCleaver = skyweaver::SkyCleaver; diff --git a/cpp/skyweaver/src/skycleaver_cli.cpp b/cpp/skyweaver/src/skycleaver_cli.cpp index 0b56bac..dcdcc1a 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cpp +++ b/cpp/skyweaver/src/skycleaver_cli.cpp @@ -1,4 +1,4 @@ -#include "skyweaver/SkyCleaver.cuh" +#include "skyweaver/SkyCleaver.hpp" #include "skyweaver/SkyCleaverConfig.hpp" #include "skyweaver/logging.hpp" @@ -301,4 +301,4 @@ int main(int argc, char** argv) skyweaver::SkyCleaver skycleaver(config); skycleaver.cleave(); -} \ No newline at end of file +} From 4b06cb3864a74876209609c7cee386a48e69c4ec Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Tue, 12 Nov 2024 14:35:40 +0000 Subject: [PATCH 34/65] More compact implementation of tdb output spitting --- cpp/skyweaver/PipelineConfig.hpp | 11 +++++++ .../detail/IncoherentDedispersionPipeline.cu | 32 +++++++++++++++++-- cpp/skyweaver/src/PipelineConfig.cpp | 12 ++++++- cpp/skyweaver/src/skyweaver_cli.cu | 12 +++++-- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 1656b0c..c751b9c 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -273,6 +273,16 @@ class PipelineConfig */ std::size_t nantennas() const { return SKYWEAVER_NANTENNAS; } + /** + * @brief Set the number of beams per output file + */ + void nbeams_per_file(std::size_t nbeams_per_file); + + /** + * @brief Get the number of beams per output file + */ + std::size_t nbeams_per_file() const; + /** * @brief Return the total number of frequency channels in the input * data @@ -350,6 +360,7 @@ class PipelineConfig float _output_level; DedispersionPlan _ddplan; mutable std::vector _channel_frequencies; + std::size_t _nbeams_per_file; }; } // namespace skyweaver diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index f295f5b..f033d19 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -47,6 +47,7 @@ IncoherentDedispersionPipeline:: // TODO: give these sensible numbers // TODO: work out how to get the max delay into the handler } + _n_tdb_files = _config.nbeams() / _config.nbeams_per_file(); } template @@ -79,7 +80,34 @@ void IncoherentDedispersionPipeline:: BOOST_LOG_TRIVIAL(debug) << "Passing output buffer to handler: " << _output_buffers[ref_dm_idx].describe(); _timer.start("file writing"); - _handler(_output_buffers[ref_dm_idx], ref_dm_idx); + + if (_n_tdb_files > 1) + { + const std::size_t ndms = _output_buffers[ref_dm_idx].ndms(); + const std::size_t nbeams = _output_buffers[ref_dm_idx].nbeams(); + const std::size_t nsamples = _output_buffers[ref_dm_idx].nsamples(); + const std::size_t nbeams_per_file = _config.nbeams_per_file(); + + _beamsplit_buffer.metalike(_output_buffers[ref_dm_idx]); + _beamsplit_buffer.resize({nsamples, ndms, nbeams_per_file}); + + for (std::size_t tdb_file_idx = 0; tdb_file_idx < _n_tdb_files; ++tdb_file_idx) + { + for (std::size_t tdidx = 0; tdidx < nsamples * ndms; ++tdidx) + { + for (std::size_t bidx = 0; bidx < nbeams_per_file ; ++bidx) + { + _beamsplit_buffer[tdidx + bidx] = _output_buffers[ref_dm_idx][tdidx + bidx + tdb_file_idx * nbeams_per_file]; + } + + } + _handler(_beamsplit_buffer, ref_dm_idx * _n_tdb_files + tdb_file_idx); + } + } + else + { + _handler(_output_buffers[ref_dm_idx], ref_dm_idx); + } _timer.stop("file writing"); } @@ -113,4 +141,4 @@ void IncoherentDedispersionPipeline::operator()( _agg_buffers[ref_dm_idx]->push_back(data.vector()); } -} // namespace skyweaver \ No newline at end of file +} // namespace skyweaver diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index c2cdf3b..919132f 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -15,7 +15,8 @@ PipelineConfig::PipelineConfig() _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f) + _stokes_mode("I"), _output_level(24.0f), + _nbeams_per_file(SKYWEAVER_NBEAMS) { } @@ -244,4 +245,13 @@ void PipelineConfig::total_nchans(std::size_t nchans) _total_nchans = nchans; } +std::size_t PipelineConfig::nbeams_per_file() const +{ + return _nbeams_per_file; +} + +void PipelineConfig::nbeams_per_file(std::size_t nbeams_per_file) +{ + _nbeams_per_file = nbeams_per_file; +} } // namespace skyweaver diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 8204ca3..4bce8f2 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -476,9 +476,15 @@ int main(int argc, char** argv) ("log-level", po::value()->default_value("info")->notifier( [](std::string level) { skyweaver::set_log_level(level); }), - "The logging level to use (debug, info, warning, error)"); + "The logging level to use (debug, info, warning, error)") - // set options allowed on command line + + ("nbeams_per_file", + po::value()->default_value(SKYWEAVER_NBEAMS)->notifier( + [&config](std::size_t key) { config.nbeams_per_file(key); }), + "The number of beams per output file. Default = all beams."); + +// set options allowed on command line po::options_description cmdline_options; cmdline_options.add(generic).add(main_options); @@ -617,4 +623,4 @@ int main(int argc, char** argv) return ERROR_UNHANDLED_EXCEPTION; } return SUCCESS; -} \ No newline at end of file +} From e697b31d27b0e6f6a626ace318155d729769cfaf Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Mon, 25 Nov 2024 12:44:25 +0100 Subject: [PATCH 35/65] minor merge changes --- cpp/skyweaver/CMakeLists.txt | 3 +- cpp/skyweaver/DescribedVector.hpp | 10 +- cpp/skyweaver/MultiFileWriter.cuh | 10 +- cpp/skyweaver/SkyCleaver.hpp | 30 +- cpp/skyweaver/SkyCleaverConfig.hpp | 10 +- cpp/skyweaver/detail/MultiFileWriter.cu | 17 +- cpp/skyweaver/{src => detail}/SkyCleaver.cpp | 435 ++++++++++-------- .../detail/file_writer_callbacks.cpp | 2 +- cpp/skyweaver/src/skycleaver_cli.cpp | 93 +++- 9 files changed, 353 insertions(+), 257 deletions(-) rename cpp/skyweaver/{src => detail}/SkyCleaver.cpp (50%) diff --git a/cpp/skyweaver/CMakeLists.txt b/cpp/skyweaver/CMakeLists.txt index d81e740..e2a605a 100644 --- a/cpp/skyweaver/CMakeLists.txt +++ b/cpp/skyweaver/CMakeLists.txt @@ -18,7 +18,6 @@ set(skyweaver_src src/Timer.cpp src/Transposer.cu src/WeightsManager.cu - src/SkyCleaver.cpp src/SigprocHeader.cpp ) @@ -43,6 +42,8 @@ set(skyweaver_inc WeightsManager.cuh MultiFileWriter.cuh SigprocHeader.hpp + SkyCleaver.hpp + SkyCleaverConfig.hpp ) set(SKYWEAVER_LIBRARIES ${CMAKE_PROJECT_NAME} ${DEPENDENCY_LIBRARIES}) diff --git a/cpp/skyweaver/DescribedVector.hpp b/cpp/skyweaver/DescribedVector.hpp index 4950f30..9b11578 100644 --- a/cpp/skyweaver/DescribedVector.hpp +++ b/cpp/skyweaver/DescribedVector.hpp @@ -692,14 +692,16 @@ template using FPAStatsD = DescribedVector, FreqDim, PolnDim, AntennaDim>; -//skycleaver output vectors + +//skycleaver vectors template -using TDBPowersStdH = DescribedVector>, +using TDBPowersStdH = DescribedVector, TimeDim, DispersionDim, - BeamDim>; + BeamDim, + PolnDim>; template -using TFPowersStdH = DescribedVector>, +using TFPowersStdH = DescribedVector, TimeDim, FreqDim>; diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 6533d6a..d107713 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -22,22 +22,24 @@ struct MultiFileWriterConfig { std::string prefix; std::string extension; std::string output_basename; + std::string suffix; MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), - output_dir("default/"), prefix(""), extension("") {}; + output_dir("default/"), prefix(""), extension(""), output_basename(""), suffix("") {}; MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, - std::string extension) + std::string extension, + std::string suffix) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), - extension(extension), output_basename("") {}; - + extension(extension), suffix(suffix), output_basename("") {}; + std::string to_string() { return "header_size: " + std::to_string(header_size) + diff --git a/cpp/skyweaver/SkyCleaver.hpp b/cpp/skyweaver/SkyCleaver.hpp index 46449de..8c99caa 100644 --- a/cpp/skyweaver/SkyCleaver.hpp +++ b/cpp/skyweaver/SkyCleaver.hpp @@ -29,18 +29,14 @@ struct BridgeReader { std::string freq; }; // BridgeReader - - +template class SkyCleaver { public: - using InputType = int8_t; - using OutputType = uint8_t; - using InputVectorType = TDBPowersStdH; - using OutputVectorType = TFPowersStdH; using FreqType = std::size_t; // up to the nearest Hz using BeamNumberType = std::size_t; using DMNumberType = std::size_t; + using StokesNumberType = std::size_t; private: SkyCleaverConfig const& _config; @@ -53,17 +49,24 @@ class SkyCleaver ObservationHeader _header; std::vector _beam_filenames; - std::map>>> + std::map< + StokesNumberType, + std::map>>>> _beam_writers; - std::map>> - _beam_data; + + std::map >>> + _beam_data; std::size_t _total_beam_writers; + int _nthreads_read; void init_readers(); void init_writers(); + void read(std::size_t gulp_samples); + void write(); Timer _timer; @@ -71,11 +74,12 @@ class SkyCleaver SkyCleaver(SkyCleaverConfig const& config); SkyCleaver(SkyCleaver const&) = delete; void operator=(SkyCleaver const&) = delete; - void cleave(); }; // class SkyCleaver + } // namespace skyweaver +#include "skyweaver/detail/SkyCleaver.cpp" #endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index 8b77dcc..6e4f76f 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -25,6 +25,9 @@ class SkyCleaverConfig std::vector _required_beams; std::vector _required_dms; + std::string _out_stokes; + std::vector _stokes_positions; + public: SkyCleaverConfig() @@ -32,7 +35,8 @@ class SkyCleaverConfig _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096), - _start_sample(0), _nsamples_to_read(0), _required_beams({}), _required_dms({}) + _start_sample(0), _nsamples_to_read(0), _required_beams({}), _required_dms({}), + _out_stokes("I"), _stokes_positions({}) { } SkyCleaverConfig(SkyCleaverConfig const&) = delete; @@ -68,6 +72,8 @@ class SkyCleaverConfig } void required_beams(std::vector required_beams) { _required_beams = required_beams; } void required_dms(std::vector required_dms) { _required_dms = required_dms; } + void out_stokes(std::string out_stokes) { _out_stokes = out_stokes; } + void stokes_positions(std::vector stokes_positions) { _stokes_positions = stokes_positions; } @@ -91,6 +97,8 @@ class SkyCleaverConfig std::size_t nsamples_to_read() const { return _nsamples_to_read; } std::vector required_beams() const { return _required_beams; } std::vector required_dms() const { return _required_dms; } + std::string out_stokes() const { return _out_stokes; } + std::vector stokes_positions() const { return _stokes_positions; } diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index cfe8a9e..16b6c05 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -43,13 +43,11 @@ MultiFileWriter::MultiFileWriter( CreateStreamCallBackType create_stream_callback) : _tag(tag), _create_stream_callback(create_stream_callback) { - MultiFileWriterConfig writer_config; - writer_config.header_size = config.dada_header_size(); - writer_config.max_file_size = config.max_output_filesize(); - writer_config.stokes_mode = config.stokes_mode(); - writer_config.base_output_dir = config.output_dir(); + _config.header_size = config.dada_header_size(); + _config.max_file_size = config.max_output_filesize(); + _config.stokes_mode = config.stokes_mode(); + _config.base_output_dir = config.output_dir(); - _config = writer_config; } template @@ -125,7 +123,12 @@ MultiFileWriter::get_basefilename(VectorType const& stream_data, base_filename << get_formatted_time(_header.utc_start) << "_" << stream_idx << "_" << std::fixed << std::setprecision(3) << std::setfill('0') << std::setw(9) - << stream_data.reference_dm(); + << stream_data.reference_dm(); + + if(!_config.suffix.empty()) { + base_filename << "_" << _config.suffix; + } + if(!_tag.empty()) { base_filename << "_" << _tag; } diff --git a/cpp/skyweaver/src/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp similarity index 50% rename from cpp/skyweaver/src/SkyCleaver.cpp rename to cpp/skyweaver/detail/SkyCleaver.cpp index 36bd663..8a4945e 100644 --- a/cpp/skyweaver/src/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -1,23 +1,22 @@ +#include "skyweaver/SkyCleaver.hpp" + +#include "skyweaver/types.cuh" +#include "skyweaver/beamformer_utils.cuh" + #include #include #include #include -#include #include +#include #include -#include "skyweaver/types.cuh" -#include "skyweaver/SkyCleaver.hpp" - -namespace fs = std::filesystem; -using SkyCleaver = skyweaver::SkyCleaver; +namespace fs = std::filesystem; -using FreqType = skyweaver::SkyCleaver::FreqType; using BridgeReader = skyweaver::BridgeReader; using MultiFileReader = skyweaver::MultiFileReader; -using OutputVectorType = skyweaver::SkyCleaver::OutputVectorType; -using InputVectorType = skyweaver::SkyCleaver::InputVectorType; + namespace { @@ -91,14 +90,13 @@ std::vector get_files(std::string directory_path, } } // namespace - -void SkyCleaver::init_readers() +template +void skyweaver::SkyCleaver::init_readers() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_readers") - std::string root_dir = _config.root_dir(); - std::string root_prefix = _config.root_prefix(); - std::size_t stream_id = _config.stream_id(); + std::string root_dir = _config.root_dir(); + std::size_t stream_id = _config.stream_id(); // get the list of directories in root/stream_id(for the nex) std::vector freq_dirs = @@ -108,7 +106,7 @@ void SkyCleaver::init_readers() << "Found " << freq_dirs.size() << " frequency directories in root directory: " << root_dir; - std::map bridge_timestamps; + std::map::FreqType, long double> bridge_timestamps; long double latest_timestamp = 0.0; for(const auto& freq_dir: freq_dirs) { @@ -142,8 +140,8 @@ void SkyCleaver::init_readers() int nbridges = _config.nbridges(); - std::size_t gulp_size = - _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); + // std::size_t gulp_size = + // _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); ObservationHeader const& header = (*_bridge_readers.begin()).second->get_header(); @@ -151,15 +149,13 @@ void SkyCleaver::init_readers() << "Read header from first file: " << header.to_string(); long double obs_centre_freq = header.obs_frequency; - long double obs_bandwidth = header.obs_bandwidth; + long double obs_bandwidth = header.obs_bandwidth; long double start_freq = obs_centre_freq - obs_bandwidth / 2; - - for(int i = 0; i < nbridges; i++) { - - int ifreq = std::lround(std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); + int ifreq = std::lround( + std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) << "Expected frequency [" << i << "]: " << ifreq; @@ -172,13 +168,16 @@ void SkyCleaver::init_readers() _bridge_data[ifreq] = std::make_unique( std::initializer_list{_config.nsamples_per_block(), _config.ndms(), - _config.nbeams()}, + _config.nbeams(), + _config.stokes_mode().size()}, 0); } // print _expected_freqs std::size_t smallest_data_size = std::numeric_limits::max(); + std::size_t dbp_factor = + _config.ndms() * _config.nbeams() * _config.stokes_mode().size(); for(const auto& [freq, reader]: _bridge_readers) { // at this point, all non-existed frequencies have been added with zero @@ -205,15 +204,13 @@ void SkyCleaver::init_readers() << " Number of samples to skip: " << nsamples; BOOST_LOG_TRIVIAL(debug) - << "Seeking " << nsamples * _config.ndms() * _config.nbeams() + << "Seeking " << nsamples * dbp_factor << " bytes in bridge reader for frequency: " << freq; - std::size_t bytes_seeking = (nsamples * _config.ndms() * - _config.nbeams() * - sizeof(InputVectorType::value_type)); + std::size_t bytes_seeking = + (nsamples * dbp_factor * sizeof(typename InputVectorType::value_type)); - _bridge_readers[freq]->seekg(bytes_seeking, - std::ios_base::beg); + _bridge_readers[freq]->seekg(bytes_seeking, std::ios_base::beg); std::size_t data_size = _bridge_readers[freq]->get_total_size() - bytes_seeking; @@ -224,16 +221,16 @@ void SkyCleaver::init_readers() } } - - if(smallest_data_size % (_config.ndms() * _config.nbeams()) != 0) { - std::runtime_error("Data size is not a multiple of ndms * nbeams"); + if(smallest_data_size % dbp_factor != 0) { + std::runtime_error( + "Data size is not a multiple of ndms * nbeams * nstokes"); } - std::size_t smallest_nsamples = - std::floor(smallest_data_size / _config.ndms() / _config.nbeams()); + std::size_t smallest_nsamples = std::floor(smallest_data_size / dbp_factor); - if (smallest_nsamples < _config.start_sample()) { - std::runtime_error("start_sample is greater than the smallest_nsamples in the data."); + if(smallest_nsamples < _config.start_sample()) { + std::runtime_error( + "start_sample is greater than the smallest_nsamples in the data."); } smallest_nsamples = smallest_nsamples - _config.start_sample(); @@ -248,22 +245,18 @@ void SkyCleaver::init_readers() } if(_config.nsamples_to_read() > 0) { - if(smallest_nsamples < _config.nsamples_to_read()) { std::runtime_error( "Smallest data size is less than nsamples_to_read"); } - _nsamples_to_read = _config.nsamples_to_read(); - } - else{ + } else { _nsamples_to_read = smallest_nsamples; } - std::size_t bytes_seeking = - (_config.start_sample() * _config.ndms() * _config.nbeams() * - sizeof(InputVectorType::value_type)); + std::size_t bytes_seeking = (_config.start_sample() * dbp_factor * + sizeof(typename InputVectorType::value_type)); if(bytes_seeking > 0) { BOOST_LOG_TRIVIAL(info) << "Seeking " << bytes_seeking @@ -278,93 +271,102 @@ void SkyCleaver::init_readers() << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; _header = _bridge_readers[_available_freqs[0]]->get_header(); - BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver: " << _header.to_string(); + BOOST_LOG_TRIVIAL(info) + << "Adding first header to SkyCleaver: " << _header.to_string(); _header.nchans = _header.nchans * _config.nbridges(); _header.nbeams = _config.nbeams(); - -} -void SkyCleaver::init_writers() + _nthreads_read = _config.nthreads() > _config.nbridges() + ? _config.nbridges() + : _config.nthreads(); +} +template +void skyweaver::SkyCleaver::init_writers() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_writers") BOOST_LOG_TRIVIAL(debug) << "_config.output_dir(); " << _config.output_dir(); - if(!fs::exists(_config.output_dir())) { fs::create_directories(_config.output_dir()); } - std::string out_prefix = _config.out_prefix().empty() - ? "" - : _config.out_prefix() + "_"; std::string output_dir = _config.output_dir(); - - - for(int idm = 0; idm < _config.ndms(); idm++) { - - std::string prefix = _config.ndms() > 1 ? out_prefix + "idm_" + - to_string_with_padding(_header.dms[idm], 9, 3) + "_": out_prefix; - - if(! _config.required_dms().empty()) { - const auto& required_dms = _config.required_dms(); - if(std::ranges::find(required_dms, _header.dms[idm]) == required_dms.end()) { - BOOST_LOG_TRIVIAL(info) - << "DM " << _header.dms[idm] << " is not required, skipping from writing"; - continue; - } - } - for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - - // skip if beam is not used - if (! _config.required_beams().empty()) { - const auto& required_beams = _config.required_beams(); - std::cerr << "required_beams: " << required_beams.size() << "ibeam: " << ibeam << std::endl; - if(std::ranges::find(required_beams, ibeam) == required_beams.end()) { + for(std::size_t istokes = 0; istokes < _config.stokes_positions().size(); + istokes++) { + for(int idm = 0; idm < _config.ndms(); idm++) { + if(!_config.required_dms().empty()) { + const auto& required_dms = _config.required_dms(); + if(std::ranges::find(required_dms, _header.dms[idm]) == + required_dms.end()) { BOOST_LOG_TRIVIAL(info) - << "Beam " << ibeam << " is not required, skipping from writing"; + << "DM " << _header.dms[idm] + << " is not required, skipping from writing"; continue; } } - MultiFileWriterConfig writer_config; - writer_config.header_size = _config.dada_header_size(); - writer_config.max_file_size = _config.max_output_filesize(); - writer_config.stokes_mode = _config.stokes_mode(); - writer_config.base_output_dir = output_dir; - writer_config.prefix = prefix + "cb_" + to_string_with_padding(ibeam, 5);; - writer_config.extension = ".fil"; - - BOOST_LOG_TRIVIAL(info) - << "Writer config: " << writer_config.to_string(); - - typename MultiFileWriter::CreateStreamCallBackType - create_stream_callback_sigproc = - skyweaver::detail::create_sigproc_file_stream< - OutputVectorType>; - _beam_writers[idm][ibeam] = - std::make_unique>( - writer_config, - "", - create_stream_callback_sigproc); - _header.ibeam = ibeam; - _beam_writers[idm][ibeam]->init(_header); - - _beam_data[idm][ibeam] = std::make_shared( - std::initializer_list{_config.nsamples_per_block(), - _config.nbridges()}, - 0); - - _beam_data[idm][ibeam]->reference_dm(_header.refdm); - - _total_beam_writers++; + for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + // skip if beam is not used + if(!_config.required_beams().empty()) { + const auto& required_beams = _config.required_beams(); + std::cerr << "required_beams: " << required_beams.size() + << "ibeam: " << ibeam << std::endl; + if(std::ranges::find(required_beams, ibeam) == + required_beams.end()) { + BOOST_LOG_TRIVIAL(info) + << "Beam " << ibeam + << " is not required, skipping from writing"; + continue; + } + } + MultiFileWriterConfig writer_config; + writer_config.header_size = _config.dada_header_size(); + writer_config.max_file_size = _config.max_output_filesize(); + writer_config.stokes_mode = _config.out_stokes().at(istokes); + writer_config.base_output_dir = output_dir; + writer_config.prefix = _config.out_prefix(); + std::string suffix = + _config.ndms() > 1 + ? "_idm_" + + to_string_with_padding(_header.dms[idm], 9, 3) + : ""; + writer_config.suffix = suffix + "_cb_" + + to_string_with_padding(ibeam, 5) + "_" + + _config.out_stokes().at(istokes); + writer_config.extension = ".fil"; + + BOOST_LOG_TRIVIAL(info) + << "Writer config: " << writer_config.to_string(); + + typename MultiFileWriter:: + CreateStreamCallBackType create_stream_callback_sigproc = + skyweaver::detail::create_sigproc_file_stream< + OutputVectorType>; + _beam_writers[istokes][idm][ibeam] = + std::make_unique>( + writer_config, + "", + create_stream_callback_sigproc); + _header.ibeam = ibeam; + _beam_writers[istokes][idm][ibeam]->init(_header); + + _beam_data[istokes][idm][ibeam] = + std::make_shared( + std::initializer_list{_config.nsamples_per_block(), + _config.nbridges()}, + 0); + + _beam_data[istokes][idm][ibeam]->reference_dm(_header.refdm); + _total_beam_writers++; + } } } BOOST_LOG_TRIVIAL(info) << "Added " << _total_beam_writers << " beam writers to SkyCleaver"; } - -SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) +template +skyweaver::SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) { _timer.start("skycleaver::init_readers"); init_readers(); @@ -373,13 +375,64 @@ SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) init_writers(); _timer.stop("skycleaver::init_writers"); } +template +void skyweaver::SkyCleaver::read(std::size_t gulp_samples) +{ + std::size_t gulp_size = gulp_samples * _config.ndms() * _config.nbeams() * + _config.stokes_mode().size(); + BOOST_LOG_TRIVIAL(info) << "Reading gulp samples: " << gulp_samples + << " with size: " << gulp_size; + + omp_set_num_threads(_nthreads_read); + + std::vector read_failures( + _available_freqs.size(), + false); // since we cannot throw exceptions in parallel regions +#pragma omp parallel for + for(std::size_t i = 0; i < _available_freqs.size(); i++) { + skyweaver::SkyCleaver::FreqType freq = _available_freqs[i]; + if(_bridge_readers.find(freq) == _bridge_readers.end()) { + read_failures[i] = true; + } + const auto& reader = _bridge_readers[freq]; + if(reader->eof()) { + BOOST_LOG_TRIVIAL(warning) + << "End of file reached for bridge " << freq; + read_failures[i] = true; + } + + std::streamsize read_size = + reader->read(reinterpret_cast(thrust::raw_pointer_cast( + _bridge_data[freq]->data())), + gulp_size); // read a big chunk of data + BOOST_LOG_TRIVIAL(debug) + << "Read " << read_size << " bytes from bridge" << freq; + if(read_size < gulp_size * sizeof(typename InputVectorType::value_type)) { + BOOST_LOG_TRIVIAL(warning) + << "Read less data than expected from bridge " << freq; + read_failures[i] = true; + } + } + bool failed = false; + for(int i = 0; i < read_failures.size(); i++) { + if(read_failures[i]) { + BOOST_LOG_TRIVIAL(error) + << "Reading bridge [" << i << "]: failed " << std::endl; + failed = true; + } + } + if(failed) { + std::runtime_error("Failed to read data from bridge readers."); + } + + BOOST_LOG_TRIVIAL(info) << "Read data from bridge readers"; +} -void SkyCleaver::cleave() +template +void skyweaver::SkyCleaver::cleave() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::cleave") - /** read data **/ - for(std::size_t nsamples_read = 0; nsamples_read < _nsamples_to_read; nsamples_read += _config.nsamples_per_block()) { std::size_t gulp_samples = @@ -390,116 +443,96 @@ void SkyCleaver::cleave() BOOST_LOG_TRIVIAL(info) << "Cleaving samples: " << nsamples_read << " to " << nsamples_read + gulp_samples; - std::size_t gulp_size = - gulp_samples * _config.ndms() * _config.nbeams(); - - int nthreads_read = _config.nthreads() > _config.nbridges() - ? _config.nbridges() - : _config.nthreads(); - - omp_set_num_threads(nthreads_read); - _timer.start("skyweaver::read_data"); - - std::vector read_status( - _available_freqs.size(), - false); // since we cannot throw exceptions in parallel regions -#pragma omp parallel for - for(std::size_t i = 0; i < _available_freqs.size(); i++) { - FreqType freq = _available_freqs[i]; - if(_bridge_readers.find(freq) == _bridge_readers.end()) { - read_status[i] = true; - } - const auto& reader = _bridge_readers[freq]; - if(reader->eof()) { - read_status[i] = true; - } - - std::streamsize read_size = - reader->read(reinterpret_cast(thrust::raw_pointer_cast( - _bridge_data[freq]->data())), - gulp_size); // read a big chunk of data - BOOST_LOG_TRIVIAL(debug) - << "Read " << read_size << " bytes from bridge" << freq; - if(read_size < gulp_size * sizeof(InputVectorType::value_type)) { - BOOST_LOG_TRIVIAL(warning) - << "Read less data than expected from bridge " << freq; - read_status[i] = true; - } - } - - if(std::any_of(read_status.begin(), read_status.end(), [](bool status) { - return status; - })) { - std::runtime_error("Some bridges have had unexpected reads"); - } - - BOOST_LOG_TRIVIAL(info) << "Read data from bridge readers"; - + read(gulp_samples); _timer.stop("skyweaver::read_data"); - _timer.start("skyweaver::process_data"); - - /** Process data **/ + _timer.start("skyweaver::process_data"); omp_set_num_threads(_config.nthreads()); - std::size_t nbridges = _config.nbridges(); - std::size_t nsamples_per_block = _config.nsamples_per_block(); - std::size_t ndms = _config.ndms(); - std::size_t nbeams = _config.nbeams(); - -#pragma omp parallel for schedule(static) collapse(2) - for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { - for(std::size_t idm = 0; idm < ndms; idm++) { // cannot separate loops, so do checks later - - // _beam_data[idm][ibeam] is not found, skip - if(_beam_data.find(idm) == _beam_data.end()) { - continue; - } - if(_beam_data[idm].find(ibeam) == _beam_data[idm].end()) { - continue; - } - + std::size_t nbridges = _config.nbridges(); + std::size_t ndms = _config.ndms(); + std::size_t nbeams = _config.nbeams(); + std::size_t nstokes_out = _config.stokes_positions().size(); + std::size_t nstokes_in = _config.stokes_mode().size(); + +#pragma omp parallel for schedule(static) collapse(3) + for(std::size_t istokes = 0; istokes < nstokes_out; istokes++) { + for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { + for(std::size_t idm = 0; idm < ndms; + idm++) { // cannot separate loops, so do checks later + BOOST_LOG_TRIVIAL(debug) + << "Processing data for stokes: " << istokes + << " beam: " << ibeam << " dm: " << idm; + if(_beam_data.find(istokes) == _beam_data.end()) { + continue; + } + if(_beam_data[istokes].find(idm) == + _beam_data[istokes].end()) { + continue; + } + if(_beam_data[istokes][idm].find(ibeam) == + _beam_data[istokes][idm].end()) { + continue; + } + const int stokes_position = _config.stokes_positions()[istokes]; + #pragma omp simd - for(std::size_t isample = 0; isample < nsamples_per_block; isample++) { - const std::size_t base_index = - isample * ndms * nbeams + idm * nbeams + ibeam; - - std::size_t ifreq = 0; - const std::size_t out_offset = isample * nbridges; - for(const auto& [freq, ifreq_data]: - _bridge_data) { // for each frequency - _beam_data[idm][ibeam]->at(out_offset + nbridges - 1 - - ifreq) = clamp(127 + ifreq_data->at(base_index)); - ++ifreq; + for(std::size_t isample = 0; isample < gulp_samples; + isample++) { + // This is the input, so use nstokes_in + const std::size_t base_index = + isample * ndms * nbeams * nstokes_in + + idm * nbeams * nstokes_in + + ibeam * nstokes_in + + stokes_position; + + std::size_t ifreq = 0; + const std::size_t out_offset = isample * nbridges; + for(const auto& [freq, ifreq_data]: + _bridge_data) { // for each frequency + _beam_data[istokes][idm][ibeam]->at( + out_offset + nbridges - 1 - ifreq) = + clamp(127 + + ifreq_data->at(base_index)); + ++ifreq; + } } } } } BOOST_LOG_TRIVIAL(info) << "Processed data"; _timer.stop("skyweaver::process_data"); - - /** Write data **/ - - _timer.start("skyweaver::write_data"); + write(); + _timer.stop("skyweaver::write_data"); + } -#pragma omp parallel for schedule(static) collapse(2) - for(int idm = 0; idm < _config.ndms(); idm++) { - for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - - if(_beam_data.find(idm) == _beam_data.end()) { + _timer.show_all_timings(); +} +template +void skyweaver::SkyCleaver::write() +{ + omp_set_num_threads(_config.nthreads()); +#pragma omp parallel for schedule(static) collapse(3) + for(std::size_t istokes = 0; istokes < _config.stokes_positions().size(); + istokes++) { + for(std::size_t idm = 0; idm < _config.ndms(); idm++) { + for(std::size_t ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + if(_beam_data.find(istokes) == _beam_data.end()) { + continue; + } + if(_beam_data[istokes].find(idm) == _beam_data[istokes].end()) { continue; } - if(_beam_data[idm].find(ibeam) == _beam_data[idm].end()) { + if(_beam_data[istokes][idm].find(ibeam) == + _beam_data[istokes][idm].end()) { continue; } - _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], - _config.stream_id()); + _beam_writers[istokes][idm][ibeam]->write( + *_beam_data[istokes][idm][ibeam], + _config.stream_id()); } } - - _timer.stop("skyweaver::write_data"); } - _timer.show_all_timings(); } diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index f2b9601..882d91b 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -227,7 +227,7 @@ create_sigproc_file_stream(MultiFileWriterConfig const& config, // uint32_t ibeam = 0; double az = 0.0; double za = 0.0; - uint32_t nifs = header.npol; + uint32_t nifs = 1; header.sigproc_params = true; header.rawfile = std::string("unset"); diff --git a/cpp/skyweaver/src/skycleaver_cli.cpp b/cpp/skyweaver/src/skycleaver_cli.cpp index dcdcc1a..62f995f 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cpp +++ b/cpp/skyweaver/src/skycleaver_cli.cpp @@ -1,3 +1,4 @@ +#include "skyweaver/DescribedVector.hpp" #include "skyweaver/SkyCleaver.hpp" #include "skyweaver/SkyCleaverConfig.hpp" #include "skyweaver/logging.hpp" @@ -7,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -18,10 +21,8 @@ #include #include #include -#include -#include -#include #include +#include #define BOOST_LOG_DYN_LINK 1 @@ -42,7 +43,6 @@ const size_t ERROR_UNHANDLED_EXCEPTION = 2; const char* build_time = __DATE__ " " __TIME__; - template std::vector get_list_from_string(const std::string& value, @@ -99,6 +99,12 @@ std::ostream& operator<<(std::ostream& os, const std::vector& vec) return os; } } // namespace std +template +void run_pipeline(skyweaver::SkyCleaverConfig const& config) +{ + skyweaver::SkyCleaver skycleaver(config); + skycleaver.cleave(); +} int main(int argc, char** argv) { @@ -126,11 +132,6 @@ int main(int argc, char** argv) ->default_value(config.output_dir()) ->notifier([&config](std::string key) { config.output_dir(key); }), "The output directory for all results")( - "root-prefix", - po::value() - ->default_value(config.root_prefix()) - ->notifier([&config](std::string key) { config.root_prefix(key); }), - "The prefix for all output files")( "out-prefix", po::value() ->default_value(config.out_prefix()) @@ -212,21 +213,29 @@ int main(int argc, char** argv) [&config](std::size_t key) { config.nsamples_to_read(key); }), "total number of samples to read from start_sample")( "required_beams", - po::value() - ->default_value("") - ->notifier([&config](std::string key) { + po::value()->default_value("")->notifier( + [&config](std::string key) { config.required_beams(get_list_from_string(key)); }), "Comma separated list of beams to process. Syntax - beam1, beam2, " "beam3:beam4:step, beam5 etc..")( "required_dms", - po::value() - ->default_value("") - ->notifier([&config](std::string key) { + po::value()->default_value("")->notifier( + [&config](std::string key) { config.required_dms(get_list_from_string(key)); }), "Comma separated list of DMs to process. Syntax - dm1, dm2, " - "dm1:dm2:step, etc.."); + "dm1:dm2:step, etc..")( + "out-stokes", + po::value() + ->default_value(config.out_stokes()) + ->notifier([&config](std::string key) { + if(key.find_first_not_of("IQUV") != std::string::npos) { + throw std::runtime_error("Invalid Stokes mode: " + key); + } + config.out_stokes(key); + }), + "The list of stokes needed - these will be separte output files"); po::options_description cmdline_options; cmdline_options.add(generic).add(main_options); @@ -271,7 +280,6 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); BOOST_LOG_TRIVIAL(info) << "output_dir: " << config.output_dir(); - BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); BOOST_LOG_TRIVIAL(info) @@ -283,22 +291,57 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); - BOOST_LOG_TRIVIAL(info) << "dada_header_size: " << config.dada_header_size(); + BOOST_LOG_TRIVIAL(info) + << "dada_header_size: " << config.dada_header_size(); BOOST_LOG_TRIVIAL(info) << "start_sample: " << config.start_sample(); - BOOST_LOG_TRIVIAL(info) << "nsamples_to_read: " << config.nsamples_to_read(); + BOOST_LOG_TRIVIAL(info) + << "nsamples_to_read: " << config.nsamples_to_read(); + BOOST_LOG_TRIVIAL(info) << "out_stokes: " << config.out_stokes(); + if(config.required_beams().size() > 0) { - for (auto beam : config.required_beams()) { - BOOST_LOG_TRIVIAL(info) << "required_beam: " << beam; - } + for(auto beam: config.required_beams()) { + BOOST_LOG_TRIVIAL(info) << "required_beam: " << beam; + } } if(config.required_dms().size() > 0) { - for (auto dm : config.required_dms()) { - BOOST_LOG_TRIVIAL(info) << "required_dm: " << dm; - } + for(auto dm: config.required_dms()) { + BOOST_LOG_TRIVIAL(info) << "required_dm: " << dm; + } + } + std::vector stokes_positions; + // make sure that every character in out_stokes is a valid stokes mode + // if present add the relevant position to the stokes_positions vector + for(auto stokes: config.out_stokes()) { + std::size_t pos = config.stokes_mode().find(stokes); + if(pos == std::string::npos) { + throw std::runtime_error("Invalid Stokes mode: " + stokes); + } + stokes_positions.push_back(pos); } + config.stokes_positions(stokes_positions); + + run_pipeline, + skyweaver::TFPowersStdH>(config); skyweaver::SkyCleaver skycleaver(config); skycleaver.cleave(); + + // if(config.stokes_mode() == "I" || config.stokes_mode() == "Q" || + // config.stokes_mode() == "U" || config.stokes_mode() == "V") { + // run_pipeline>, + // skyweaver::TFPowersStdH>(config); + // } else if(config.stokes_mode() == "IV" || config.stokes_mode() == "QU") { + // run_pipeline, + // skyweaver::TFPowersStdH>(config); + + // } else if(config.stokes_mode() == "IQUV") { + // run_pipeline, + // skyweaver::TFPowersStdH>(config); + // } else { + // throw std::runtime_error("Invalid Stokes mode: " + + // config.stokes_mode()); + // } + } From a1a2c94f30790b03aaca862c444f7e1106507f29 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Tue, 26 Nov 2024 17:32:34 +0100 Subject: [PATCH 36/65] get header values from header instead of user --- cpp/skyweaver/ObservationHeader.hpp | 3 +- cpp/skyweaver/SkyCleaver.hpp | 4 +- cpp/skyweaver/detail/SkyCleaver.cpp | 129 ++++++++++++++++++------ cpp/skyweaver/src/ObservationHeader.cpp | 9 +- cpp/skyweaver/src/skycleaver_cli.cpp | 39 +------ 5 files changed, 114 insertions(+), 70 deletions(-) diff --git a/cpp/skyweaver/ObservationHeader.hpp b/cpp/skyweaver/ObservationHeader.hpp index d09564d..6e2fbb9 100644 --- a/cpp/skyweaver/ObservationHeader.hpp +++ b/cpp/skyweaver/ObservationHeader.hpp @@ -41,8 +41,9 @@ struct ObservationHeader { std::string telescope; // Telescope name std::string instrument; // Name of the recording instrument std::string order; // Order of the dimensions in the data - std::string ndms; // Number of DMs + std::size_t ndms; // Number of DMs std::vector dms; // DMs + std::string stokes_mode; // Stokes mode std::string to_string() const; // Convert the header to a string diff --git a/cpp/skyweaver/SkyCleaver.hpp b/cpp/skyweaver/SkyCleaver.hpp index 8c99caa..6ff993f 100644 --- a/cpp/skyweaver/SkyCleaver.hpp +++ b/cpp/skyweaver/SkyCleaver.hpp @@ -39,7 +39,7 @@ class SkyCleaver using StokesNumberType = std::size_t; private: - SkyCleaverConfig const& _config; + SkyCleaverConfig& _config; std::map> _bridge_readers; std::map> _bridge_data; @@ -71,7 +71,7 @@ class SkyCleaver Timer _timer; public: - SkyCleaver(SkyCleaverConfig const& config); + SkyCleaver(SkyCleaverConfig& config); SkyCleaver(SkyCleaver const&) = delete; void operator=(SkyCleaver const&) = delete; void cleave(); diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index 8a4945e..a17b669 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -1,8 +1,8 @@ #include "skyweaver/SkyCleaver.hpp" -#include "skyweaver/types.cuh" #include "skyweaver/beamformer_utils.cuh" +#include "skyweaver/types.cuh" #include #include @@ -12,11 +12,10 @@ #include #include -namespace fs = std::filesystem; - -using BridgeReader = skyweaver::BridgeReader; -using MultiFileReader = skyweaver::MultiFileReader; +namespace fs = std::filesystem; +using BridgeReader = skyweaver::BridgeReader; +using MultiFileReader = skyweaver::MultiFileReader; namespace { @@ -90,6 +89,45 @@ std::vector get_files(std::string directory_path, } } // namespace + +void compare_bridge_headers(const skyweaver::ObservationHeader& first, + const skyweaver::ObservationHeader& second) +{ + if(first.nchans != second.nchans) { + throw std::runtime_error("Number of channels in bridge readers do not " + "match. Expected: " + + std::to_string(first.nchans) + + " Found: " + std::to_string(second.nchans)); + } + if(first.nbeams != second.nbeams) { + throw std::runtime_error( + "Number of beams in bridge readers do not match. " + "Expected: " + + std::to_string(first.nbeams) + + " Found: " + std::to_string(second.nbeams)); + } + if(first.nbits != second.nbits) { + throw std::runtime_error( + "Number of bits in bridge readers do not match. " + "Expected: " + + std::to_string(first.nbits) + + " Found: " + std::to_string(second.nbits)); + } + if(first.tsamp != second.tsamp) { + throw std::runtime_error( + "Sampling time in bridge readers do not match. " + "Expected: " + + std::to_string(first.tsamp) + + " Found: " + std::to_string(second.tsamp)); + } + if(first.stokes_mode != second.stokes_mode) { + throw std::runtime_error("Stokes mode in bridge readers do not match. " + "Expected: " + + first.stokes_mode + + " Found: " + second.stokes_mode); + } +} + template void skyweaver::SkyCleaver::init_readers() { @@ -106,7 +144,9 @@ void skyweaver::SkyCleaver::init_readers() << "Found " << freq_dirs.size() << " frequency directories in root directory: " << root_dir; - std::map::FreqType, long double> bridge_timestamps; + std::map::FreqType, + long double> + bridge_timestamps; long double latest_timestamp = 0.0; for(const auto& freq_dir: freq_dirs) { @@ -140,22 +180,49 @@ void skyweaver::SkyCleaver::init_readers() int nbridges = _config.nbridges(); - // std::size_t gulp_size = - // _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); + _header = _bridge_readers[_available_freqs[0]]->get_header(); + for(const auto& [freq, reader]: _bridge_readers) { + compare_bridge_headers(_header, reader->get_header()); + } + BOOST_LOG_TRIVIAL(info) + << "Number of beams: " << _header.nbeams + << " Number of DMS: " << _header.ndms + << " Stokes mode: " << _header.stokes_mode + << " Number of channels: " << _header.nchans; + + + _config.nbeams(_header.nbeams); + _config.ndms(_header.ndms); + _config.stokes_mode(_header.stokes_mode); + _config.nchans(_header.nchans); - ObservationHeader const& header = - (*_bridge_readers.begin()).second->get_header(); BOOST_LOG_TRIVIAL(info) - << "Read header from first file: " << header.to_string(); + << "Number of beams: " << _config.nbeams() + << " Number of DMS: " << _config.ndms() + << " Stokes mode: " << _config.stokes_mode() + << " Number of channels: " << _config.nchans(); + + std::vector stokes_positions; + // make sure that every character in out_stokes is a valid stokes mode + // if present add the relevant position to the stokes_positions vector + for(auto stokes: _config.out_stokes()) { + std::size_t pos = _config.stokes_mode().find(stokes); + if(pos == std::string::npos) { + throw std::runtime_error("Invalid Stokes mode: " + stokes); + } + stokes_positions.push_back(pos); + } + + _config.stokes_positions(stokes_positions); - long double obs_centre_freq = header.obs_frequency; - long double obs_bandwidth = header.obs_bandwidth; + long double obs_centre_freq = _header.obs_frequency; + long double obs_bandwidth = _header.obs_bandwidth; long double start_freq = obs_centre_freq - obs_bandwidth / 2; - for(int i = 0; i < nbridges; i++) { - int ifreq = std::lround( - std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); + for(int i = 0; i < _config.nbridges(); i++) { + int ifreq = std::lround(std::floor( + start_freq + (i + 0.5) * obs_bandwidth / _config.nbridges())); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) << "Expected frequency [" << i << "]: " << ifreq; @@ -168,13 +235,11 @@ void skyweaver::SkyCleaver::init_readers() _bridge_data[ifreq] = std::make_unique( std::initializer_list{_config.nsamples_per_block(), _config.ndms(), - _config.nbeams(), + _config.nbeams(), _config.stokes_mode().size()}, 0); } - // print _expected_freqs - std::size_t smallest_data_size = std::numeric_limits::max(); std::size_t dbp_factor = _config.ndms() * _config.nbeams() * _config.stokes_mode().size(); @@ -208,7 +273,8 @@ void skyweaver::SkyCleaver::init_readers() << " bytes in bridge reader for frequency: " << freq; std::size_t bytes_seeking = - (nsamples * dbp_factor * sizeof(typename InputVectorType::value_type)); + (nsamples * dbp_factor * + sizeof(typename InputVectorType::value_type)); _bridge_readers[freq]->seekg(bytes_seeking, std::ios_base::beg); @@ -366,7 +432,9 @@ void skyweaver::SkyCleaver::init_writers() << "Added " << _total_beam_writers << " beam writers to SkyCleaver"; } template -skyweaver::SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) +skyweaver::SkyCleaver::SkyCleaver( + SkyCleaverConfig& config) + : _config(config) { _timer.start("skycleaver::init_readers"); init_readers(); @@ -376,7 +444,8 @@ skyweaver::SkyCleaver::SkyCleaver(SkyCleaverC _timer.stop("skycleaver::init_writers"); } template -void skyweaver::SkyCleaver::read(std::size_t gulp_samples) +void skyweaver::SkyCleaver::read( + std::size_t gulp_samples) { std::size_t gulp_size = gulp_samples * _config.ndms() * _config.nbeams() * _config.stokes_mode().size(); @@ -390,7 +459,8 @@ void skyweaver::SkyCleaver::read(std::size_t false); // since we cannot throw exceptions in parallel regions #pragma omp parallel for for(std::size_t i = 0; i < _available_freqs.size(); i++) { - skyweaver::SkyCleaver::FreqType freq = _available_freqs[i]; + skyweaver::SkyCleaver::FreqType + freq = _available_freqs[i]; if(_bridge_readers.find(freq) == _bridge_readers.end()) { read_failures[i] = true; } @@ -407,7 +477,8 @@ void skyweaver::SkyCleaver::read(std::size_t gulp_size); // read a big chunk of data BOOST_LOG_TRIVIAL(debug) << "Read " << read_size << " bytes from bridge" << freq; - if(read_size < gulp_size * sizeof(typename InputVectorType::value_type)) { + if(read_size < + gulp_size * sizeof(typename InputVectorType::value_type)) { BOOST_LOG_TRIVIAL(warning) << "Read less data than expected from bridge " << freq; read_failures[i] = true; @@ -475,17 +546,17 @@ void skyweaver::SkyCleaver::cleave() _beam_data[istokes][idm].end()) { continue; } - const int stokes_position = _config.stokes_positions()[istokes]; - + const int stokes_position = + _config.stokes_positions()[istokes]; + #pragma omp simd for(std::size_t isample = 0; isample < gulp_samples; isample++) { // This is the input, so use nstokes_in const std::size_t base_index = isample * ndms * nbeams * nstokes_in + - idm * nbeams * nstokes_in + - ibeam * nstokes_in + - stokes_position; + idm * nbeams * nstokes_in + ibeam * nstokes_in + + stokes_position; std::size_t ifreq = 0; const std::size_t out_offset = isample * nbridges; diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index 7040a4e..e17c282 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -69,8 +69,11 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, "OBS_FREQ", header.frequency)); - header.ndms = parser.get_or_default("NDMS", "0"); - if(header.ndms != "0") { + header.ndms = parser.get_or_default("NDMS", 0); + header.nbeams = parser.get_or_default("NBEAM", 1); + header.stokes_mode = + parser.get_or_default("STOKES_MODE", "I"); + if(header.ndms != 0) { header.dms = parse_float_list(parser.get("DMS")); } } @@ -152,7 +155,7 @@ std::string ObservationHeader::to_string() const << " instrument: " << instrument << "\n" << " chan0_idx: " << chan0_idx << "\n" << " obs_offset: " << obs_offset << "\n"; - if(ndms != "0") { + if(ndms != 0) { oss << " ndms: " << ndms << "\n"; oss << " dms: "; for(auto dm: dms) { oss << dm << " "; } diff --git a/cpp/skyweaver/src/skycleaver_cli.cpp b/cpp/skyweaver/src/skycleaver_cli.cpp index 62f995f..9fec4af 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cpp +++ b/cpp/skyweaver/src/skycleaver_cli.cpp @@ -100,7 +100,7 @@ std::ostream& operator<<(std::ostream& os, const std::vector& vec) } } // namespace std template -void run_pipeline(skyweaver::SkyCleaverConfig const& config) +void run_pipeline(skyweaver::SkyCleaverConfig& config) { skyweaver::SkyCleaver skycleaver(config); skycleaver.cleave(); @@ -148,31 +148,11 @@ int main(int argc, char** argv) ->notifier( [&config](std::size_t key) { config.nsamples_per_block(key); }), "The number of samples per block")( - "nchans", - po::value() - ->default_value(config.nchans()) - ->notifier([&config](std::size_t key) { config.nchans(key); }), - "The number of channels")( "nbridges", po::value() ->default_value(config.nbridges()) ->notifier([&config](std::size_t key) { config.nbridges(key); }), "The number of bridges")( - "nbeams", - po::value() - ->default_value(config.nbeams()) - ->notifier([&config](std::size_t key) { config.nbeams(key); }), - "The number of beams")( - "ndms", - po::value() - ->default_value(config.ndms()) - ->notifier([&config](std::size_t key) { config.ndms(key); }), - "The number of DMs")( - "stokes-mode", - po::value() - ->default_value(config.stokes_mode()) - ->notifier([&config](std::string key) { config.stokes_mode(key); }), - "The stokes mode")( "stream-id", po::value() ->default_value(config.stream_id()) @@ -308,25 +288,14 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "required_dm: " << dm; } } - std::vector stokes_positions; - // make sure that every character in out_stokes is a valid stokes mode - // if present add the relevant position to the stokes_positions vector - for(auto stokes: config.out_stokes()) { - std::size_t pos = config.stokes_mode().find(stokes); - if(pos == std::string::npos) { - throw std::runtime_error("Invalid Stokes mode: " + stokes); - } - stokes_positions.push_back(pos); - } - - config.stokes_positions(stokes_positions); + run_pipeline, skyweaver::TFPowersStdH>(config); - skyweaver::SkyCleaver skycleaver(config); - skycleaver.cleave(); + // skyweaver::SkyCleaver skycleaver(config); + // skycleaver.cleave(); // if(config.stokes_mode() == "I" || config.stokes_mode() == "Q" || // config.stokes_mode() == "U" || config.stokes_mode() == "V") { From cbaf03bca758f85ba14e07600d1a3ac12e036f41 Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 28 Nov 2024 11:48:32 +0100 Subject: [PATCH 37/65] dirty pass on scaling change --- cpp/skyweaver/CoherentBeamformer.cuh | 6 +- cpp/skyweaver/IncoherentBeamformer.cuh | 10 +- cpp/skyweaver/StatisticsCalculator.cuh | 6 +- cpp/skyweaver/beamformer_utils.cuh | 34 +++-- cpp/skyweaver/src/CoherentBeamformer.cu | 10 +- cpp/skyweaver/src/IncoherentBeamformer.cu | 14 +- cpp/skyweaver/src/StatisticsCalculator.cu | 166 ++++++++++++---------- 7 files changed, 137 insertions(+), 109 deletions(-) diff --git a/cpp/skyweaver/CoherentBeamformer.cuh b/cpp/skyweaver/CoherentBeamformer.cuh index dc47b9c..d10cc35 100644 --- a/cpp/skyweaver/CoherentBeamformer.cuh +++ b/cpp/skyweaver/CoherentBeamformer.cuh @@ -33,8 +33,8 @@ __global__ void bf_ftpa_general_k( int2 const* __restrict__ ftpa_voltages, int2 const* __restrict__ fbpa_weights, typename BfTraits::QuantisedPowerType* __restrict__ btf_powers, - float const* __restrict__ output_scale, - float const* __restrict__ output_offset, + float4 const* __restrict__ output_scale, + float4 const* __restrict__ output_offset, int const* __restrict__ beamset_mapping, typename BfTraits::RawPowerType const* __restrict__ ib_powers, int nsamples); @@ -55,7 +55,7 @@ class CoherentBeamformer typedef BTFPowersD RawPowerVectorTypeD; // FBA order (assuming equal weight per polarisation) typedef thrust::device_vector WeightsVectorTypeD; - typedef thrust::device_vector ScalingVectorTypeD; + typedef thrust::device_vector ScalingVectorTypeD; typedef thrust::device_vector MappingVectorTypeD; public: diff --git a/cpp/skyweaver/IncoherentBeamformer.cuh b/cpp/skyweaver/IncoherentBeamformer.cuh index e40d73b..ef18b6d 100644 --- a/cpp/skyweaver/IncoherentBeamformer.cuh +++ b/cpp/skyweaver/IncoherentBeamformer.cuh @@ -34,8 +34,8 @@ __global__ void icbf_ftpa_general_k( char2 const* __restrict__ ftpa_voltages, typename BfTraits::RawPowerType* __restrict__ tf_powers_raw, typename BfTraits::QuantisedPowerType* __restrict__ tf_powers, - float const* __restrict__ output_scale, - float const* __restrict__ output_offset, + float4 const* __restrict__ output_scale, + float4 const* __restrict__ output_offset, float const* __restrict__ antenna_weights, int nsamples, int nbeamsets); @@ -52,7 +52,9 @@ class IncoherentBeamformer // TF order typedef BTFPowersD RawPowerVectorTypeD; // TF order - typedef thrust::device_vector ScalingVectorTypeD; + typedef thrust::device_vector ScalingVectorTypeD; // Always IQUV scalings + // BsA order + typedef thrust::device_vector BeamsetWeightsVectorTypeD; public: /** @@ -86,7 +88,7 @@ class IncoherentBeamformer PowerVectorTypeD& output, ScalingVectorTypeD const& output_scale, ScalingVectorTypeD const& output_offset, - ScalingVectorTypeD const& antenna_weights, + BeamsetWeightsVectorTypeD const& antenna_weights, int nbeamsets, cudaStream_t stream); diff --git a/cpp/skyweaver/StatisticsCalculator.cuh b/cpp/skyweaver/StatisticsCalculator.cuh index 204f82b..5767a78 100644 --- a/cpp/skyweaver/StatisticsCalculator.cuh +++ b/cpp/skyweaver/StatisticsCalculator.cuh @@ -39,9 +39,11 @@ struct StatisticsFileHeader { class StatisticsCalculator { public: - typedef float ScalingType; + typedef float4 ScalingType; // Always caclulate for full stokes typedef thrust::device_vector ScalingVectorTypeD; typedef thrust::host_vector ScalingVectorTypeH; + typedef thrust::device_vector BeamsetWeightsVectorTypeD; + typedef thrust::host_vector BeamsetWeightsVectorTypeH; typedef FPAStatsD StatisticsVectorTypeD; typedef FPAStatsH StatisticsVectorTypeH; @@ -93,7 +95,7 @@ class StatisticsCalculator * @brief Update the scaling arrays based on the last statistics * calculation. */ - void update_scalings(ScalingVectorTypeH const& beamset_weights, + void update_scalings(BeamsetWeightsVectorTypeH const& beamset_weights, int nbeamsets); private: diff --git a/cpp/skyweaver/beamformer_utils.cuh b/cpp/skyweaver/beamformer_utils.cuh index 74d4873..ae165e8 100644 --- a/cpp/skyweaver/beamformer_utils.cuh +++ b/cpp/skyweaver/beamformer_utils.cuh @@ -270,10 +270,20 @@ struct IncoherentBeamSubtract { static inline __host__ __device__ void apply(T const& power, T const& ib_power, float const& ib_mutliplier, // 127^2 as default - float const& scale_factor, + float4 const& scale_factor, T& result) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor); + if constexpr(S == StokesParameter::I) { + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.x); + } constexpr(S == StokesParameter::Q) { + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.y); + } constexpr(S == StokesParameter::U) { + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.z); + } constexpr(S == StokesParameter::V) { + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.w); + } else { + static_no_match(); + } } }; @@ -285,10 +295,16 @@ struct Rescale { T& result) { if constexpr(S == StokesParameter::I) { - AT(result, I) = rintf((AT(power, I) - offset) / scale_factor); + AT(result, I) = rintf((AT(power, I) - offset.x) / scale_factor.x); + } constexpr(S == StokesParameter::Q) { + AT(result, I) = rintf((AT(power, I) - offset.y) / scale_factor.y); + } constexpr(S == StokesParameter::U) { + AT(result, I) = rintf((AT(power, I) - offset.z) / scale_factor.z); + } constexpr(S == StokesParameter::V) { + AT(result, I) = rintf((AT(power, I) - offset.w) / scale_factor.w); } else { - AT(result, I) = rintf(AT(power, I) / scale_factor); - } + static_no_match(); + } } }; @@ -335,16 +351,16 @@ struct StokesTraits ib_subtract(RawPowerType const& power, RawPowerType const& ib_power, float const& ib_mutliplier, - float const& scale_factor) { + float4 const& scale_factor) { RawPowerType result{}; - Iterate<0, Stokes...>::template apply(power, ib_power, ib_mutliplier, scale_factor, result); + Iterate<0, Stokes...>::template apply(power, ib_power, ib_mutliplier, scale_factor, result); return result; } static inline __host__ __device__ RawPowerType rescale(RawPowerType const& power, - float const& offset, - float const& scale_factor) { + float4 const& offset, + float4 const& scale_factor) { RawPowerType result{}; Iterate<0, Stokes...>::template apply(power, offset, scale_factor, result); return result; diff --git a/cpp/skyweaver/src/CoherentBeamformer.cu b/cpp/skyweaver/src/CoherentBeamformer.cu index 43a8688..ff1e563 100644 --- a/cpp/skyweaver/src/CoherentBeamformer.cu +++ b/cpp/skyweaver/src/CoherentBeamformer.cu @@ -14,8 +14,8 @@ __global__ void bf_ftpa_general_k( int2 const* __restrict__ ftpa_voltages, int2 const* __restrict__ fbpa_weights, typename BfTraits::QuantisedPowerType* __restrict__ tfb_powers, - float const* __restrict__ output_scale, - float const* __restrict__ output_offset, + float4 const* __restrict__ output_scale, + float4 const* __restrict__ output_offset, int const* __restrict__ beamset_mapping, typename BfTraits::RawPowerType const* __restrict__ ib_powers, int nsamples) @@ -156,7 +156,7 @@ __global__ void bf_ftpa_general_k( int const ib_power_idx = beamset_idx * nsamps_out * gridDim.y + output_sample_idx * gridDim.y + blockIdx.y; int const scloff_idx = beamset_idx * gridDim.y + blockIdx.y; - float scale = output_scale[scloff_idx]; + float4 scale = output_scale[scloff_idx]; typename BfTraits::RawPowerType ib_power = ib_powers[ib_power_idx]; #if SKYWEAVER_IB_SUBTRACTION /* @@ -246,9 +246,9 @@ void CoherentBeamformer::beamform( char2 const* fbpa_weights_ptr = thrust::raw_pointer_cast(weights.data()); typename BfTraits::QuantisedPowerType* tfb_powers_ptr = thrust::raw_pointer_cast(output.data()); - float const* power_scaling_ptr = + float4 const* power_scaling_ptr = thrust::raw_pointer_cast(output_scale.data()); - float const* power_offset_ptr = + float4 const* power_offset_ptr = thrust::raw_pointer_cast(output_offset.data()); typename BfTraits::RawPowerType const* ib_powers_ptr = thrust::raw_pointer_cast(ib_powers.data()); diff --git a/cpp/skyweaver/src/IncoherentBeamformer.cu b/cpp/skyweaver/src/IncoherentBeamformer.cu index c91618e..ee4a80e 100644 --- a/cpp/skyweaver/src/IncoherentBeamformer.cu +++ b/cpp/skyweaver/src/IncoherentBeamformer.cu @@ -17,8 +17,8 @@ __global__ void icbf_ftpa_general_k( char2 const* __restrict__ ftpa_voltages, typename BfTraits::RawPowerType* __restrict__ tf_powers_raw, typename BfTraits::QuantisedPowerType* __restrict__ tf_powers, - float const* __restrict__ output_scale, - float const* __restrict__ output_offset, + float4 const* __restrict__ output_scale, + float4 const* __restrict__ output_offset, float const* __restrict__ antenna_weights, int nsamples, int nbeamsets) @@ -79,9 +79,9 @@ __global__ void icbf_ftpa_general_k( const typename BfTraits::RawPowerType power_f32 = acc_buffer[0]; const int output_idx = beamset_idx * gridDim.x * gridDim.y + output_t_idx * gridDim.y + output_f_idx; - const float scale = + const float4 scale = output_scale[beamset_idx * gridDim.y + output_f_idx]; - const float offset = + const float4 offset = output_offset[beamset_idx * gridDim.y + output_f_idx]; tf_powers_raw[output_idx] = power_f32; tf_powers[output_idx] = @@ -112,7 +112,7 @@ void IncoherentBeamformer::beamform( PowerVectorTypeD& output, ScalingVectorTypeD const& output_scale, ScalingVectorTypeD const& output_offset, - ScalingVectorTypeD const& antenna_weights, + BeamsetWeightsVectorTypeD const& antenna_weights, int nbeamsets, cudaStream_t stream) { @@ -156,9 +156,9 @@ void IncoherentBeamformer::beamform( dim3 grid(nsamples / _config.ib_tscrunch(), _config.nchans() / _config.ib_fscrunch()); char2 const* ftpa_voltages_ptr = thrust::raw_pointer_cast(input.data()); - float const* output_scale_ptr = + float4 const* output_scale_ptr = thrust::raw_pointer_cast(output_scale.data()); - float const* output_offset_ptr = + float4 const* output_offset_ptr = thrust::raw_pointer_cast(output_offset.data()); float const* antenna_weights_ptr = thrust::raw_pointer_cast(antenna_weights.data()); diff --git a/cpp/skyweaver/src/StatisticsCalculator.cu b/cpp/skyweaver/src/StatisticsCalculator.cu index 4de451a..a2cf68c 100644 --- a/cpp/skyweaver/src/StatisticsCalculator.cu +++ b/cpp/skyweaver/src/StatisticsCalculator.cu @@ -130,7 +130,7 @@ void StatisticsCalculator::calculate_statistics( } void StatisticsCalculator::update_scalings( - ScalingVectorTypeH const& beamset_weights, + BeamsetWeightsVectorTypeH const& beamset_weights, int nbeamsets) { // At this stage we have the standard deviations of each channel @@ -139,99 +139,107 @@ void StatisticsCalculator::update_scalings( // disk. const float weights_amp = 127.0f; - std::size_t reduced_nchans_ib = _config.nchans() / _config.ib_fscrunch(); - std::size_t reduced_nchans_cb = _config.nchans() / _config.cb_fscrunch(); + if ((_config.ib_fscrunch() != _config.cb_fscrunch()) || + (_config.ib_tscrunch() != _config.cb_tscrunch())) { + throw std::invalid_argument("IB and CB must share same F and T scrunch"); + } + const std::size_t output_nchans = _config.nchans() / _config.cb_fscrunch(); + const std::size_t fscrunch = _config.cb_fscrunch(); + const std::size_t tscrunch = _config.cb_tscrunch(); // Offsets for the coherent beams - _cb_offsets_d.resize(reduced_nchans_cb * nbeamsets); - _cb_offsets_h.resize(reduced_nchans_cb * nbeamsets); + _cb_offsets_d.resize(output_nchans * nbeamsets); + _cb_offsets_h.resize(output_nchans * nbeamsets); // Scalings for the coherent beams - _cb_scaling_d.resize(reduced_nchans_cb * nbeamsets); - _cb_scaling_h.resize(reduced_nchans_cb * nbeamsets); + _cb_scaling_d.resize(output_nchans * nbeamsets); + _cb_scaling_h.resize(output_nchans * nbeamsets); // Offsets for the incoherent beam - _ib_offsets_d.resize(reduced_nchans_ib * nbeamsets); - _ib_offsets_h.resize(reduced_nchans_ib * nbeamsets); + _ib_offsets_d.resize(output_nchans * nbeamsets); + _ib_offsets_h.resize(output_nchans * nbeamsets); // Scalings for the incoherent beam - _ib_scaling_d.resize(reduced_nchans_ib * nbeamsets); - _ib_scaling_h.resize(reduced_nchans_ib * nbeamsets); + _ib_scaling_d.resize(output_nchans * nbeamsets); + _ib_scaling_h.resize(output_nchans * nbeamsets); const std::uint32_t pa = _config.npol() * _config.nantennas(); const std::uint32_t a = _config.nantennas(); for(std::uint32_t beamset_idx = 0; beamset_idx < nbeamsets; ++beamset_idx) { - // For each frequency channel we average the std estimates then - // calculate the offsets and scalings. - for(int f_idx = 0; f_idx < _config.nchans(); ++f_idx) { - float sum = 0.0f; - float count = 0.0f; - for(int p_idx = 0; p_idx < _config.npol(); ++p_idx) { - for(int a_idx = 0; a_idx < _config.nantennas(); ++a_idx) { - const float weight = - beamset_weights[_config.nantennas() * beamset_idx + - a_idx]; - sum += - weight * _stats_h[f_idx * pa + p_idx * a + a_idx].std; - count += weight; - } - } - const float avg_std = sum / count; - BOOST_LOG_TRIVIAL(debug) << "Channel " << f_idx; - BOOST_LOG_TRIVIAL(debug) - << "Averaged standard deviation = " << avg_std; - float const effective_nantennas = count / _config.npol(); - - // CB OFFSET - { - float scale = std::pow(weights_amp * avg_std * - std::sqrt(effective_nantennas), - 2); - float dof = 2 * _config.cb_tscrunch() * _config.npol(); - _cb_offsets_h[f_idx] = scale * dof; - BOOST_LOG_TRIVIAL(debug) - << "CB offset = " << _cb_offsets_h[f_idx]; - } - // CB SCALE - { - float scale = std::pow(weights_amp * avg_std * - std::sqrt(effective_nantennas), - 2); - float dof = 2 * _config.cb_tscrunch() * _config.npol(); - _cb_scaling_h[f_idx] = - scale * std::sqrt(2 * dof) / _config.output_level(); - BOOST_LOG_TRIVIAL(debug) - << "CB scaling = " << _cb_scaling_h[f_idx]; - } - - // IB OFFSET - { - float scale = std::pow(avg_std, 2); - float dof = 2 * _config.ib_tscrunch() * effective_nantennas * - _config.npol(); - _ib_offsets_h[f_idx] = scale * dof; - BOOST_LOG_TRIVIAL(debug) - << "IB offset = " << _ib_offsets_h[f_idx]; + // Here we compute the offsets and scaling factors for I, Q, U and V for each beamset + // Statistics are in FPA order + + for(int fo_idx = 0; fo_idx < output_nchans; ++fo_idx) { + + const int output_idx = beamset_idx * nbeamsets + fo_idx; + + // reset main accumulators + float meanI = 0.0f; + float meanQ = 0.0f; + float varIQc = 0.0f; + float varUVc = 0.0f; + float varIQi = 0.0f; + float varUVi = 0.0f; + + for(int fs_idx = 0; fs_idx < fscrunch; ++fs_idx) { + const int f_idx = fo_idx * fscrunch + fs_idx; + + // Per antenna/pol accumulators + float var_p0_sum = 0.0f; // sum(sigma_p0^2) + float var_p1_sum = 0.0f; // sum(sigma_p1^2) + float quad_sum = 0.0f; // sum(sigma_p0^4 + sigma_p1^4) + float sum_mul = 0.0f; // sum(sigma_p0^2 * sigma_p1^2) + for (int a_idx = 0; a_idx < _cofig.npol(); ++a_idx) { + // stats are in FPA order + // weights are in FPBA order + const int input_idx = f_idx * pa + a_idx; + Statistics p0 = _stats_h[input_idx]; + Statistics p1 = _stats_h[input_idx + nantennas]; + const float weight = beamset_weights[beamset_idx * a + a_idx]; + const float std_p0 = p0.std * weight; + const float std_p1 = p1.std * weight; + const float var_p0 = std_p0 * std_p0; + const float var_p1 = std_p1 * std_p1; + meanI += var_p0 + var_p1; + meanQ += var_p0 - var_p1; + var_p0_sum += var_p0; + var_p1_sum += var_p1; + quad_sum += (var_p0 * var_p0) + (var_p1 * var_p1); + sum_mul += var_p0 * var_p1; + } + varIQc += (var_p0_sum * var_p0_sum) + (var_p1_sum * var_p1_sum); + varIQi += quad_sum; + varUVc += var_p0_sum * var_p1_sum; + varUVi += sum_mul; } - // IB SCALE - { - float scale = std::pow(avg_std, 2); - float dof = 2 * _config.ib_tscrunch() * effective_nantennas * - _config.npol(); - _ib_scaling_h[f_idx] = - scale * std::sqrt(2 * dof) / _config.output_level(); - BOOST_LOG_TRIVIAL(debug) - << "IB scaling = " << _ib_scaling_h[f_idx]; - } + // Coherent offsets + _cb_offsets_h[output_idx].x = 2 * tscrunch * meanI; // I + _cb_offsets_h[output_idx].y = 2 * tscrunch * meanQ; // Q + _cb_offsets_h[output_idx].z = 0.0f; // U + _cb_offsets_h[output_idx].w = 0.0f; // V + + // Coherent scales + _cb_scaling_h[output_idx].x = 4 * tscrunch * varIQc / _config.output_level(); // I + _cb_scaling_h[output_idx].y = 4 * tscrunch * varIQc / _config.output_level(); // Q + _cb_scaling_h[output_idx].z = 8 * tscrunch * varUVc / _config.output_level(); // U + _cb_scaling_h[output_idx].w = 8 * tscrunch * varUVc / _config.output_level(); // V + + // Incoherent offsets + _ib_offsets_h[output_idx].x = 2 * tscrunch * meanI; // I + _ib_offsets_h[output_idx].y = 2 * tscrunch * meanQ; // Q + _ib_offsets_h[output_idx].z = 0.0f; // U + _ib_offsets_h[output_idx].w = 0.0f; // V + + // Incoherent scales + _ib_scaling_h[output_idx].x = 4 * tscrunch * varIQi / _config.output_level(); // I + _ib_scaling_h[output_idx].y = 4 * tscrunch * varIQi / _config.output_level(); // Q + _ib_scaling_h[output_idx].z = 8 * tscrunch * varUVi / _config.output_level(); // U + _ib_scaling_h[output_idx].w = 8 * tscrunch * varUVi / _config.output_level(); // V } } - // At this stage, all scaling vectors are available on the host and - // could be written to disk. - - // Copying these back to the device _cb_offsets_d = _cb_offsets_h; _cb_scaling_d = _cb_scaling_h; _ib_offsets_d = _ib_offsets_h; @@ -256,7 +264,7 @@ void StatisticsCalculator::dump_scalings( std::string const& timestamp, std::string const& tag, std::string const& path, - thrust::host_vector const& ar) const + ScalingVectorTypeH const& ar) const { std::ofstream writer; std::string filename = path + "/" + timestamp + "_" + tag + "_" + @@ -271,7 +279,7 @@ void StatisticsCalculator::dump_scalings( BOOST_LOG_TRIVIAL(error) << error_message.str(); return; } - writer.write((char*)ar.data(), ar.size() * sizeof(float)); + writer.write(static_cast(ar.data()), ar.size() * sizeof(ScalingType)); writer.close(); } @@ -305,4 +313,4 @@ StatisticsCalculator::ib_scaling() const return _ib_scaling_d; } -} // namespace skyweaver +} // namespace skyweaver \ No newline at end of file From 820efb994e341feea8119be5ec12b90143905b90 Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 28 Nov 2024 15:25:23 +0100 Subject: [PATCH 38/65] compiling now and beamformer tests running --- cpp/skyweaver/StatisticsCalculator.cuh | 2 +- cpp/skyweaver/beamformer_utils.cuh | 16 ++--- cpp/skyweaver/benchmark/beamformer_bench.cu | 2 +- cpp/skyweaver/src/StatisticsCalculator.cu | 10 +-- .../test/CoherentBeamformerTester.cuh | 17 ++++- .../test/IncoherentBeamformerTester.cuh | 7 +- .../test/src/CoherentBeamformerTester.cu | 71 +++++++++++-------- .../test/src/IncoherentBeamformerTester.cu | 34 +++++---- 8 files changed, 95 insertions(+), 64 deletions(-) diff --git a/cpp/skyweaver/StatisticsCalculator.cuh b/cpp/skyweaver/StatisticsCalculator.cuh index 5767a78..1b9dee3 100644 --- a/cpp/skyweaver/StatisticsCalculator.cuh +++ b/cpp/skyweaver/StatisticsCalculator.cuh @@ -104,7 +104,7 @@ class StatisticsCalculator void dump_scalings(std::string const& timestamp, std::string const& tag, std::string const& path, - thrust::host_vector const& ar) const; + ScalingVectorTypeH const& ar) const; private: PipelineConfig const& _config; diff --git a/cpp/skyweaver/beamformer_utils.cuh b/cpp/skyweaver/beamformer_utils.cuh index ae165e8..15ec773 100644 --- a/cpp/skyweaver/beamformer_utils.cuh +++ b/cpp/skyweaver/beamformer_utils.cuh @@ -275,11 +275,11 @@ struct IncoherentBeamSubtract { { if constexpr(S == StokesParameter::I) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.x); - } constexpr(S == StokesParameter::Q) { + } else if constexpr(S == StokesParameter::Q) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.y); - } constexpr(S == StokesParameter::U) { + } else if constexpr(S == StokesParameter::U) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.z); - } constexpr(S == StokesParameter::V) { + } else if constexpr(S == StokesParameter::V) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.w); } else { static_no_match(); @@ -290,17 +290,17 @@ struct IncoherentBeamSubtract { struct Rescale { template static inline __host__ __device__ void apply(T const& power, - float const& offset, - float const& scale_factor, + float4 const& offset, + float4 const& scale_factor, T& result) { if constexpr(S == StokesParameter::I) { AT(result, I) = rintf((AT(power, I) - offset.x) / scale_factor.x); - } constexpr(S == StokesParameter::Q) { + } else if constexpr(S == StokesParameter::Q) { AT(result, I) = rintf((AT(power, I) - offset.y) / scale_factor.y); - } constexpr(S == StokesParameter::U) { + } else if constexpr(S == StokesParameter::U) { AT(result, I) = rintf((AT(power, I) - offset.z) / scale_factor.z); - } constexpr(S == StokesParameter::V) { + } else if constexpr(S == StokesParameter::V) { AT(result, I) = rintf((AT(power, I) - offset.w) / scale_factor.w); } else { static_no_match(); diff --git a/cpp/skyweaver/benchmark/beamformer_bench.cu b/cpp/skyweaver/benchmark/beamformer_bench.cu index a179741..663cb3b 100644 --- a/cpp/skyweaver/benchmark/beamformer_bench.cu +++ b/cpp/skyweaver/benchmark/beamformer_bench.cu @@ -70,7 +70,7 @@ class BeamformerBencher: public benchmark::Fixture typename CoherentBeamformer::PowerVectorTypeD btf_powers_gpu; typename CoherentBeamformer::ScalingVectorTypeD scales; typename CoherentBeamformer::ScalingVectorTypeD offsets; - typename CoherentBeamformer::ScalingVectorTypeD _antenna_weights; + typename IncoherentBeamformer::BeamsetWeightsVectorTypeD _antenna_weights; typename IncoherentBeamformer::PowerVectorTypeD tf_powers_gpu; typename IncoherentBeamformer::RawPowerVectorTypeD tf_powers_raw_gpu; typename CoherentBeamformer::MappingVectorTypeD _beamset_mapping; diff --git a/cpp/skyweaver/src/StatisticsCalculator.cu b/cpp/skyweaver/src/StatisticsCalculator.cu index a2cf68c..a8dcb33 100644 --- a/cpp/skyweaver/src/StatisticsCalculator.cu +++ b/cpp/skyweaver/src/StatisticsCalculator.cu @@ -191,15 +191,15 @@ void StatisticsCalculator::update_scalings( float var_p1_sum = 0.0f; // sum(sigma_p1^2) float quad_sum = 0.0f; // sum(sigma_p0^4 + sigma_p1^4) float sum_mul = 0.0f; // sum(sigma_p0^2 * sigma_p1^2) - for (int a_idx = 0; a_idx < _cofig.npol(); ++a_idx) { + for (int a_idx = 0; a_idx < _config.npol(); ++a_idx) { // stats are in FPA order // weights are in FPBA order const int input_idx = f_idx * pa + a_idx; Statistics p0 = _stats_h[input_idx]; - Statistics p1 = _stats_h[input_idx + nantennas]; + Statistics p1 = _stats_h[input_idx + a]; const float weight = beamset_weights[beamset_idx * a + a_idx]; - const float std_p0 = p0.std * weight; - const float std_p1 = p1.std * weight; + const float std_p0 = p0.std * weight * weights_amp; + const float std_p1 = p1.std * weight * weights_amp; const float var_p0 = std_p0 * std_p0; const float var_p1 = std_p1 * std_p1; meanI += var_p0 + var_p1; @@ -279,7 +279,7 @@ void StatisticsCalculator::dump_scalings( BOOST_LOG_TRIVIAL(error) << error_message.str(); return; } - writer.write(static_cast(ar.data()), ar.size() * sizeof(ScalingType)); + writer.write(reinterpret_cast(thrust::raw_pointer_cast(ar.data())), ar.size() * sizeof(ScalingType)); writer.close(); } diff --git a/cpp/skyweaver/test/CoherentBeamformerTester.cuh b/cpp/skyweaver/test/CoherentBeamformerTester.cuh index fd60262..e956de1 100644 --- a/cpp/skyweaver/test/CoherentBeamformerTester.cuh +++ b/cpp/skyweaver/test/CoherentBeamformerTester.cuh @@ -21,26 +21,37 @@ class CoherentBeamformerTester: public ::testing::Test typedef CoherentBeamformer CoherentBeamformer; typedef IncoherentBeamformer IncoherentBeamformer; typedef CoherentBeamformer::VoltageVectorTypeD VoltageVectorTypeD; + typedef FTPAVoltagesH VoltageVectorTypeH; + typedef CoherentBeamformer::PowerVectorTypeD PowerVectorTypeD; typedef TFBPowersH HostPowerVectorType; + typedef IncoherentBeamformer::PowerVectorTypeD IBPowerVectorTypeD; typedef BTFPowersH HostIBPowerVectorType; + typedef IncoherentBeamformer::RawPowerVectorTypeD RawIBPowerVectorTypeD; typedef BTFPowersH HostRawIBPowerVectorType; + typedef CoherentBeamformer::WeightsVectorTypeD WeightsVectorTypeD; typedef thrust::host_vector WeightsVectorTypeH; + typedef CoherentBeamformer::ScalingVectorTypeD ScalingVectorTypeD; typedef thrust::host_vector HostScalingVectorType; + typedef CoherentBeamformer::MappingVectorTypeD MappingVectorTypeD; typedef thrust::host_vector MappingVectorTypeH; + + typedef IncoherentBeamformer::BeamsetWeightsVectorTypeD BeamsetWeightsVectorTypeD; + typedef thrust::host_vector + BeamsetWeightsVectorTypeH; protected: void SetUp() override; @@ -61,8 +72,8 @@ class CoherentBeamformerTester: public ::testing::Test int nbeams, int nantennas, int npol, - float const* scales, - float const* offsets, + float4 const* scales, + float4 const* offsets, float const* antenna_weights, int const* beamset_mapping); @@ -70,7 +81,7 @@ class CoherentBeamformerTester: public ::testing::Test WeightsVectorTypeD const& fbpa_weights_gpu, ScalingVectorTypeD const& scales_gpu, ScalingVectorTypeD const& offsets_gpu, - ScalingVectorTypeD const& antenna_weights, + BeamsetWeightsVectorTypeD const& antenna_weights, MappingVectorTypeD const& beamset_mapping, PowerVectorTypeD& btf_powers_gpu, int nsamples); diff --git a/cpp/skyweaver/test/IncoherentBeamformerTester.cuh b/cpp/skyweaver/test/IncoherentBeamformerTester.cuh index 37da649..5983321 100644 --- a/cpp/skyweaver/test/IncoherentBeamformerTester.cuh +++ b/cpp/skyweaver/test/IncoherentBeamformerTester.cuh @@ -31,6 +31,9 @@ class IncoherentBeamformerTester: public ::testing::Test typedef IncoherentBeamformer::ScalingVectorTypeD ScalingVectorTypeD; typedef thrust::host_vector HostScalingVectorType; + typedef IncoherentBeamformer::BeamsetWeightsVectorTypeD BeamsetWeightsVectorTypeD; + typedef thrust::host_vector + HostBeamsetWeightsVectorType; protected: void SetUp() override; @@ -51,7 +54,7 @@ class IncoherentBeamformerTester: public ::testing::Test int nantennas, HostScalingVectorType const& scale, HostScalingVectorType const& offset, - HostScalingVectorType const& beamset_weights, + HostBeamsetWeightsVectorType const& beamset_weights, int nbeamsets); void compare_against_host(VoltageVectorTypeD const& ftpa_voltages_gpu, @@ -59,7 +62,7 @@ class IncoherentBeamformerTester: public ::testing::Test DevicePowerVectorType& tf_powers_gpu, ScalingVectorTypeD const& scaling_vector, ScalingVectorTypeD const& offset_vector, - ScalingVectorTypeD const& beamset_weights, + BeamsetWeightsVectorTypeD const& beamset_weights, int ntimestamps, int nbeamsets); diff --git a/cpp/skyweaver/test/src/CoherentBeamformerTester.cu b/cpp/skyweaver/test/src/CoherentBeamformerTester.cu index a369015..9a98df5 100644 --- a/cpp/skyweaver/test/src/CoherentBeamformerTester.cu +++ b/cpp/skyweaver/test/src/CoherentBeamformerTester.cu @@ -51,8 +51,8 @@ void CoherentBeamformerTester::beamformer_c_reference( int nbeams, int nantennas, int npol, - float const* scales, - float const* offsets, + float4 const* scales, + float4 const* offsets, float const* antenna_weights, int const* beamset_mapping) { @@ -157,7 +157,7 @@ void CoherentBeamformerTester::compare_against_host( WeightsVectorTypeD const& fbpa_weights_gpu, ScalingVectorTypeD const& scales_gpu, ScalingVectorTypeD const& offsets_gpu, - ScalingVectorTypeD const& antenna_weights, + BeamsetWeightsVectorTypeD const& antenna_weights, MappingVectorTypeD const& beamset_mapping, PowerVectorTypeD& btf_powers_gpu, int nsamples) @@ -170,7 +170,7 @@ void CoherentBeamformerTester::compare_against_host( HostScalingVectorType scales = scales_gpu; HostScalingVectorType offsets = offsets_gpu; - HostScalingVectorType antenna_weights_host = antenna_weights; + thrust::host_vector antenna_weights_host = antenna_weights; MappingVectorTypeH beamset_mapping_host = beamset_mapping; beamformer_c_reference( @@ -227,29 +227,36 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) const float input_level = 32.0f; const double pi = std::acos(-1); config.output_level(input_level); - float scale = - std::pow(127.0f * input_level * - std::sqrt(static_cast(config.nantennas())), - 2); - float dof = 2 * config.cb_tscrunch() * config.cb_fscrunch() * config.npol(); - float offset_val = (scale * dof); - float scale_val = (scale * std::sqrt(2 * dof) / config.output_level()); + + float4 cb_offset_val{ + static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(127.0f * input_level, 2)) / config.output_level()), + 0.0f, + 0.0f, + 0.0f + }; + float cb_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * std::pow(config.nantennas() * std::pow(127.0f * input_level, 2), 2) ) / config.output_level()); + float4 cb_scale_val{ + cb_scaling, + cb_scaling, + cb_scaling, + cb_scaling + }; // Set constant scales and offsets for all channels and beamsets typename CBT::ScalingVectorTypeD cb_scales( config.nchans() / config.cb_fscrunch() * nbeamsets, - scale_val); + cb_scale_val); typename CBT::ScalingVectorTypeD cb_offsets( config.nchans() / config.cb_fscrunch() * nbeamsets, - offset_val); + cb_offset_val); // Map all beams to the first beamset by default typename CBT::MappingVectorTypeD beamset_mapping(config.nbeams(), 0); // Enable all antennas in all beamsets - typename CBT::ScalingVectorTypeD beamset_weights(config.nantennas() * + typename thrust::device_vector beamset_weights(config.nantennas() * nbeamsets, - 1.0f); + 1.0f); /** This currently causes tests to fail even though the weights are the same for @@ -260,9 +267,6 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) */ beamset_mapping[0] = 1; - BOOST_LOG_TRIVIAL(info) << "CB scaling: " << scale_val; - BOOST_LOG_TRIVIAL(info) << "CB offset: " << offset_val; - std::default_random_engine generator; std::normal_distribution normal_dist(0.0, input_level); std::uniform_real_distribution uniform_dist(0.0, 2 * pi); @@ -280,9 +284,6 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) std::size_t input_size = (ntimestamps * config.nantennas() * config.nchans() * config.nsamples_per_heap() * config.npol()); - BOOST_LOG_TRIVIAL(info) << "FTPA input dims: " << config.nchans() << ", " - << ntimestamps * config.nsamples_per_heap() << ", " - << config.npol() << ", " << config.nantennas(); int nsamples = config.nsamples_per_heap() * ntimestamps; std::size_t weights_size = @@ -314,20 +315,28 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) typename CBT::WeightsVectorTypeD fbpa_weights_gpu = fbpa_weights_host; typename CBT::PowerVectorTypeD tfb_powers_gpu; - // Note that below even though this is for the IB we have to use the - // CB scrunching parameters to make sure we get the right data out. - float ib_scale = std::pow(input_level, 2); - float ib_dof = 2 * config.cb_tscrunch() * config.cb_fscrunch() * - config.nantennas() * config.npol(); - float ib_power_offset = ib_scale * ib_dof; - float ib_power_scaling = - ib_scale * std::sqrt(2 * ib_dof) / config.output_level(); + float4 ib_offset_val{ + static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 2)) / config.output_level()), + 0.0f, + 0.0f, + 0.0f + }; + + float ib_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 4)) / config.output_level()); + float4 ib_scale_val{ + ib_scaling, + ib_scaling, + ib_scaling, + ib_scaling + }; + typename CBT::ScalingVectorTypeD ib_scales( config.nchans() / config.cb_fscrunch() * nbeamsets, - ib_power_scaling); + ib_scale_val); typename CBT::ScalingVectorTypeD ib_offset( config.nchans() / config.cb_fscrunch() * nbeamsets, - ib_power_offset); + ib_offset_val); + typename CBT::IBPowerVectorTypeD tf_powers_gpu; typename CBT::RawIBPowerVectorTypeD tf_powers_raw_gpu; diff --git a/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu b/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu index a52f675..d7af1c5 100644 --- a/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu +++ b/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu @@ -49,7 +49,7 @@ void IncoherentBeamformerTester::beamformer_c_reference( int nantennas, HostScalingVectorType const& scale, HostScalingVectorType const& offset, - HostScalingVectorType const& beamset_weights, + HostBeamsetWeightsVectorType const& beamset_weights, int nbeamsets) { static_assert(SKYWEAVER_NPOL == 2, "Tests only work for dual poln data."); @@ -109,7 +109,7 @@ void IncoherentBeamformerTester::compare_against_host( DevicePowerVectorType& tf_powers_gpu, ScalingVectorTypeD const& scaling_vector, ScalingVectorTypeD const& offset_vector, - ScalingVectorTypeD const& beamset_weights, + BeamsetWeightsVectorTypeD const& beamset_weights, int ntimestamps, int nbeamsets) { @@ -118,7 +118,7 @@ void IncoherentBeamformerTester::compare_against_host( HostRawPowerVectorType tf_powers_raw_cuda = tf_powers_raw_gpu; HostScalingVectorType h_scaling_vector = scaling_vector; HostScalingVectorType h_offset_vector = offset_vector; - HostScalingVectorType h_beamset_weights = beamset_weights; + HostBeamsetWeightsVectorType h_beamset_weights = beamset_weights; HostRawPowerVectorType tf_powers_raw_host; tf_powers_raw_host.like(tf_powers_raw_gpu); HostPowerVectorType tf_powers_host; @@ -175,23 +175,31 @@ TYPED_TEST(IncoherentBeamformerTester, ib_representative_noise_test) static_cast(std::lround(normal_dist(generator))); } - float ib_scale = std::pow(input_level, 2); - float ib_dof = 2 * config.ib_tscrunch() * config.ib_fscrunch() * - config.nantennas() * config.npol(); - float ib_power_offset = ib_scale * ib_dof; - float ib_power_scaling = - ib_scale * std::sqrt(2 * ib_dof) / config.output_level(); + float4 ib_offset_val{ + static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 2)) / config.output_level()), + 0.0f, + 0.0f, + 0.0f + }; + + float ib_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 4)) / config.output_level()); + float4 ib_scale_val{ + ib_scaling, + ib_scaling, + ib_scaling, + ib_scaling + }; for(int nbeamsets = 1; nbeamsets < 5; ++nbeamsets) { typename IBT::ScalingVectorTypeD scales( config.nchans() / config.ib_fscrunch() * nbeamsets, - ib_power_scaling); + ib_scale_val); typename IBT::ScalingVectorTypeD offset( config.nchans() / config.ib_fscrunch() * nbeamsets, - ib_power_offset); - typename IBT::ScalingVectorTypeD beamset_weights(config.nantennas() * + ib_offset_val); + typename IBT::BeamsetWeightsVectorTypeD beamset_weights(config.nantennas() * nbeamsets, - 1.0f); + 1.0f); typename IBT::VoltageVectorTypeD ftpa_voltages_gpu = ftpa_voltages_host; typename IBT::DevicePowerVectorType tf_powers_gpu; From 49c966a6a1d606162409632a9d7e0a4469bf8008 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 12 Dec 2024 17:26:08 +0100 Subject: [PATCH 39/65] linear pol output and actual beam positions in skycleaver --- cpp/skyweaver/SkyCleaver.hpp | 8 + cpp/skyweaver/SkyCleaverConfig.hpp | 20 +- cpp/skyweaver/detail/SkyCleaver.cpp | 202 +++++++++++++++--- .../detail/file_writer_callbacks.cpp | 6 +- cpp/skyweaver/src/skycleaver_cli.cpp | 57 +---- 5 files changed, 203 insertions(+), 90 deletions(-) diff --git a/cpp/skyweaver/SkyCleaver.hpp b/cpp/skyweaver/SkyCleaver.hpp index 6ff993f..fad34cc 100644 --- a/cpp/skyweaver/SkyCleaver.hpp +++ b/cpp/skyweaver/SkyCleaver.hpp @@ -29,6 +29,13 @@ struct BridgeReader { std::string freq; }; // BridgeReader + +struct BeamInfo { + std::string beam_name; + std::string beam_ra; + std::string beam_dec; +}; + template class SkyCleaver { @@ -47,6 +54,7 @@ class SkyCleaver std::vector _available_freqs; std::size_t _nsamples_to_read; ObservationHeader _header; + std::vector _beam_infos; std::vector _beam_filenames; std::map< diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index 6e4f76f..924d0d2 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -24,19 +24,20 @@ class SkyCleaverConfig std::size_t _nsamples_to_read; std::vector _required_beams; std::vector _required_dms; + std::string _targets; std::string _out_stokes; - std::vector _stokes_positions; + std::vector> _stokes_positions; public: SkyCleaverConfig() : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), - _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), + _nthreads(0), _nsamples_per_block(32768), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096), _start_sample(0), _nsamples_to_read(0), _required_beams({}), _required_dms({}), - _out_stokes("I"), _stokes_positions({}) + _targets(""), _out_stokes("I"), _stokes_positions({}) { } SkyCleaverConfig(SkyCleaverConfig const&) = delete; @@ -73,7 +74,11 @@ class SkyCleaverConfig void required_beams(std::vector required_beams) { _required_beams = required_beams; } void required_dms(std::vector required_dms) { _required_dms = required_dms; } void out_stokes(std::string out_stokes) { _out_stokes = out_stokes; } - void stokes_positions(std::vector stokes_positions) { _stokes_positions = stokes_positions; } + void stokes_positions(std::vector> stokes_positions) + { + _stokes_positions = stokes_positions; + } + void targets_file(std::string targets) { _targets = targets; } @@ -98,8 +103,11 @@ class SkyCleaverConfig std::vector required_beams() const { return _required_beams; } std::vector required_dms() const { return _required_dms; } std::string out_stokes() const { return _out_stokes; } - std::vector stokes_positions() const { return _stokes_positions; } - + std::vector> stokes_positions() const + { + return _stokes_positions; + } + std::string targets_file() const { return _targets; } }; diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index a17b669..b84cff7 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -3,6 +3,8 @@ #include "skyweaver/beamformer_utils.cuh" #include "skyweaver/types.cuh" +#include "skyweaver/skycleaver_utils.hpp" + #include #include @@ -16,9 +18,20 @@ namespace fs = std::filesystem; using BridgeReader = skyweaver::BridgeReader; using MultiFileReader = skyweaver::MultiFileReader; +using BeamInfo = skyweaver::BeamInfo; namespace { + +std::string trim(const std::string& str) { + auto start = str.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) { + return ""; // String is all whitespace + } + auto end = str.find_last_not_of(" \t\n\r"); + return str.substr(start, end - start + 1); +} + template std::string to_string_with_padding(T num, int width, int precision = -1) { @@ -71,6 +84,10 @@ std::vector get_files(std::string directory_path, if(fs::is_regular_file(entry.status())) { std::string file_name = entry.path().string(); if(file_name.find(extension) != std::string::npos) { + //check if .tmp not in filename + if(file_name.find(".tmp") != std::string::npos) { + continue; + } files.push_back(file_name); } } @@ -88,8 +105,62 @@ std::vector get_files(std::string directory_path, return files; } + } // namespace +void parse_target_file(std::string file_name, std::vector& beam_infos){ + std::ifstream targets_file(file_name); + if(!targets_file.is_open()) { + std::runtime_error("Error opening target file: " + file_name); + } + // the file is in csv format. First read the header to know the positions of name, ra and dec + std::string header; + do{ + std::getline(targets_file, header); + } while(header.empty() || header.find("#") != std::string::npos); + + std::vector header_tokens; + std::stringstream header_stream(header); + std::string token; + while(std::getline(header_stream, token, ',')) { + header_tokens.push_back(token); + } + std::size_t name_pos = std::distance(header_tokens.begin(), std::find(header_tokens.begin(), header_tokens.end(), "name")); + std::size_t ra_pos = std::distance(header_tokens.begin(), std::find(header_tokens.begin(), header_tokens.end(), "ra")); + std::size_t dec_pos = std::distance(header_tokens.begin(), std::find(header_tokens.begin(), header_tokens.end(), "dec")); + + if(name_pos == header_tokens.size() || ra_pos == header_tokens.size() || dec_pos == header_tokens.size()) { + std::runtime_error("Invalid header in target file: " + file_name); + } + + std::string line; + while(std::getline(targets_file, line)) { + + line = trim(line); + + //if empty line or # anywhere in line, continue + if(line.empty() || line.find("#") != std::string::npos) { + BOOST_LOG_TRIVIAL(debug) << "Skipping line: " << line; + continue; + } + + std::vector tokens; + std::stringstream line_stream(line); + std::string token; + while(std::getline(line_stream, token, ',')) { + tokens.push_back(token); + } + if(tokens.size() != header_tokens.size()) { + std::runtime_error("Invalid number of columns in target file: " + file_name); + } + BeamInfo beam_info; + beam_info.beam_name = tokens[name_pos]; + beam_info.beam_ra = tokens[ra_pos]; + beam_info.beam_dec = tokens[dec_pos]; + beam_infos.push_back(beam_info); + } +} + void compare_bridge_headers(const skyweaver::ObservationHeader& first, const skyweaver::ObservationHeader& second) { @@ -202,19 +273,30 @@ void skyweaver::SkyCleaver::init_readers() << " Stokes mode: " << _config.stokes_mode() << " Number of channels: " << _config.nchans(); - std::vector stokes_positions; - // make sure that every character in out_stokes is a valid stokes mode - // if present add the relevant position to the stokes_positions vector - for(auto stokes: _config.out_stokes()) { + std::vector> stokes_positions; + for(const auto stokes: _config.out_stokes()) { std::size_t pos = _config.stokes_mode().find(stokes); if(pos == std::string::npos) { - throw std::runtime_error("Invalid Stokes mode: " + stokes); + + if(stokes == 'L') { + std::size_t pos1 = _config.stokes_mode().find("Q"); + std::size_t pos2 = _config.stokes_mode().find("U"); + if(pos1 == std::string::npos || pos2 == std::string::npos) { + throw std::runtime_error("Asked for L, but beamformed data does not have Q and/or U"); + } + stokes_positions.push_back({pos1, pos2}); + continue; + } + else { + throw std::runtime_error("Requested stokes not found in beamformed data: " + stokes); + } } - stokes_positions.push_back(pos); + stokes_positions.push_back({pos}); } _config.stokes_positions(stokes_positions); + long double obs_centre_freq = _header.obs_frequency; long double obs_bandwidth = _header.obs_bandwidth; @@ -350,15 +432,16 @@ template void skyweaver::SkyCleaver::init_writers() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_writers") + BOOST_LOG_TRIVIAL(debug) << "_config.output_dir(); " << _config.output_dir(); + std::string output_dir = _config.output_dir(); - if(!fs::exists(_config.output_dir())) { - fs::create_directories(_config.output_dir()); + if(!fs::exists(output_dir)) { + fs::create_directories(output_dir); } - std::string output_dir = _config.output_dir(); - for(std::size_t istokes = 0; istokes < _config.stokes_positions().size(); + for(std::size_t istokes = 0; istokes < _config.out_stokes().size(); istokes++) { for(int idm = 0; idm < _config.ndms(); idm++) { if(!_config.required_dms().empty()) { @@ -385,19 +468,18 @@ void skyweaver::SkyCleaver::init_writers() continue; } } + BeamInfo beam_info = _beam_infos[ibeam]; MultiFileWriterConfig writer_config; writer_config.header_size = _config.dada_header_size(); writer_config.max_file_size = _config.max_output_filesize(); writer_config.stokes_mode = _config.out_stokes().at(istokes); writer_config.base_output_dir = output_dir; writer_config.prefix = _config.out_prefix(); - std::string suffix = - _config.ndms() > 1 - ? "_idm_" + - to_string_with_padding(_header.dms[idm], 9, 3) - : ""; - writer_config.suffix = suffix + "_cb_" + - to_string_with_padding(ibeam, 5) + "_" + + std::string suffix = "idm_" + + to_string_with_padding(_header.dms[idm], 9, 3); + + writer_config.suffix = suffix + "_" + + beam_info.beam_name + "_" + _config.out_stokes().at(istokes); writer_config.extension = ".fil"; @@ -414,6 +496,9 @@ void skyweaver::SkyCleaver::init_writers() "", create_stream_callback_sigproc); _header.ibeam = ibeam; + + _header.ra = beam_info.beam_ra; + _header.dec = beam_info.beam_dec; _beam_writers[istokes][idm][ibeam]->init(_header); _beam_data[istokes][idm][ibeam] = @@ -436,9 +521,37 @@ skyweaver::SkyCleaver::SkyCleaver( SkyCleaverConfig& config) : _config(config) { + BOOST_LOG_TRIVIAL(info) << "Reading and initialising beam details from file: " + << _config.targets_file(); + + parse_target_file(_config.targets_file(), _beam_infos); + + BOOST_LOG_TRIVIAL(info) << "Number of beams in target file: " << _beam_infos.size(); + _timer.start("skycleaver::init_readers"); init_readers(); _timer.stop("skycleaver::init_readers"); + + + if(_beam_infos.size() < _config.nbeams()){ // there are some null beams with zeros, do not create filterbanks for them. + std::string required_beams = "0:" + std::to_string(_beam_infos.size()-1); + std::vector required_beam_numbers = skyweaver::get_list_from_string(required_beams); + + if(_config.required_beams().empty()) { // if nothing given, set the valid beams to required beams + BOOST_LOG_TRIVIAL(warning) << "Number of beams in target file is less than the number of beams in the header. " + << "Setting required beams to: " << required_beams; + _config.required_beams(skyweaver::get_list_from_string(required_beams)); + } + else{ + for(auto beam_num: _config.required_beams()){ // if given, check if all requested beams are valid beams + if(std::find(required_beam_numbers.begin(), required_beam_numbers.end(), beam_num) == required_beam_numbers.end()){ + std::runtime_error("Beam number " + std::to_string(beam_num) + " not found in target file."); + } + } + } + } + + _timer.start("skycleaver::init_writers"); init_writers(); _timer.stop("skycleaver::init_writers"); @@ -524,7 +637,7 @@ void skyweaver::SkyCleaver::cleave() std::size_t nbridges = _config.nbridges(); std::size_t ndms = _config.ndms(); std::size_t nbeams = _config.nbeams(); - std::size_t nstokes_out = _config.stokes_positions().size(); + std::size_t nstokes_out = _config.out_stokes().size(); std::size_t nstokes_in = _config.stokes_mode().size(); #pragma omp parallel for schedule(static) collapse(3) @@ -532,9 +645,7 @@ void skyweaver::SkyCleaver::cleave() for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { for(std::size_t idm = 0; idm < ndms; idm++) { // cannot separate loops, so do checks later - BOOST_LOG_TRIVIAL(debug) - << "Processing data for stokes: " << istokes - << " beam: " << ibeam << " dm: " << idm; + if(_beam_data.find(istokes) == _beam_data.end()) { continue; } @@ -546,29 +657,55 @@ void skyweaver::SkyCleaver::cleave() _beam_data[istokes][idm].end()) { continue; } - const int stokes_position = - _config.stokes_positions()[istokes]; + + const std::vector stokes_positions = + _config.stokes_positions()[istokes]; + #pragma omp simd for(std::size_t isample = 0; isample < gulp_samples; isample++) { - // This is the input, so use nstokes_in - const std::size_t base_index = + const std::size_t out_offset = isample * nbridges; + + // This is stupid but preferred over a more elegant solution that is not fast, this can be easily vectorised + if(stokes_positions.size() == 1) { + const std::size_t base_index = isample * ndms * nbeams * nstokes_in + idm * nbeams * nstokes_in + ibeam * nstokes_in + - stokes_position; - - std::size_t ifreq = 0; - const std::size_t out_offset = isample * nbridges; - for(const auto& [freq, ifreq_data]: + stokes_positions[0]; + + std::size_t ifreq = 0; + for(const auto& [freq, ifreq_data]: _bridge_data) { // for each frequency _beam_data[istokes][idm][ibeam]->at( out_offset + nbridges - 1 - ifreq) = clamp(127 + ifreq_data->at(base_index)); ++ifreq; + } + } + else{ + std::size_t ifreq = 0; + for(const auto& [freq, ifreq_data]: _bridge_data) { + float value = 0; + for(int stokes_position=0; stokes_positionat(base_index) * ifreq_data->at(base_index)); + } + // for each frequency + _beam_data[istokes][idm][ibeam]->at( + out_offset + nbridges - 1 - ifreq) = + clamp(127 + + sqrt(value)); + ++ifreq; + } } + } + } } } @@ -578,15 +715,16 @@ void skyweaver::SkyCleaver::cleave() write(); _timer.stop("skyweaver::write_data"); } - _timer.show_all_timings(); + } + template void skyweaver::SkyCleaver::write() { omp_set_num_threads(_config.nthreads()); #pragma omp parallel for schedule(static) collapse(3) - for(std::size_t istokes = 0; istokes < _config.stokes_positions().size(); + for(std::size_t istokes = 0; istokes < _config.out_stokes().size(); istokes++) { for(std::size_t idm = 0; idm < _config.ndms(); idm++) { for(std::size_t ibeam = 0; ibeam < _config.nbeams(); ibeam++) { diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 882d91b..e1c59dc 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -38,13 +38,10 @@ HEADER DADA HDR_VERSION 1.0 HDR_SIZE 4096 DADA_VERSION 1.0 - FILE_SIZE 100000000000 FILE_NUMBER 0 - UTC_START 1708082229.000020336 MJD_START 60356.47024305579093 - SOURCE J1644-4559 RA 16:44:49.27 DEC -45:59:09.7 @@ -54,7 +51,6 @@ RECEIVER L-band FREQ 1284000000.000000 BW 856000000.000000 TSAMP 4.7850467290 - NBIT 8 NDIM 1 NPOL 1 @@ -62,7 +58,7 @@ NCHAN 64 NBEAM 800 ORDER TFB CHAN0_IDX 2688 - )"; +)"; #define MJD_UNIX_EPOCH 40587.0 } // namespace diff --git a/cpp/skyweaver/src/skycleaver_cli.cpp b/cpp/skyweaver/src/skycleaver_cli.cpp index 9fec4af..88b4e55 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cpp +++ b/cpp/skyweaver/src/skycleaver_cli.cpp @@ -2,6 +2,7 @@ #include "skyweaver/SkyCleaver.hpp" #include "skyweaver/SkyCleaverConfig.hpp" #include "skyweaver/logging.hpp" +#include "skyweaver/skycleaver_utils.hpp" #include #include @@ -43,51 +44,7 @@ const size_t ERROR_UNHANDLED_EXCEPTION = 2; const char* build_time = __DATE__ " " __TIME__; -template -std::vector -get_list_from_string(const std::string& value, - T epsilon = std::numeric_limits::epsilon()) -{ - std::vector output; - std::vector comma_chunks; - - // Split the input string by commas - std::stringstream ss(value); - std::string token; - while(std::getline(ss, token, ',')) { comma_chunks.push_back(token); } - - for(const auto& comma_chunk: comma_chunks) { - // Check if the chunk contains a colon (indicating a range) - if(comma_chunk.find(':') == std::string::npos) { - output.push_back(static_cast(std::atof(comma_chunk.c_str()))); - continue; - } - - // Split the range chunk by colons - std::stringstream ss_chunk(comma_chunk); - std::vector colon_chunks; - std::string colon_token; - while(std::getline(ss_chunk, colon_token, ':')) { - colon_chunks.push_back( - static_cast(std::atof(colon_token.c_str()))); - } - // Determine the step size - T step = colon_chunks.size() == 3 ? colon_chunks[2] : static_cast(1); - T start = colon_chunks[0]; - T stop = colon_chunks[1]; - - // Loop and add values to the output vector - if constexpr(std::is_floating_point::value) { - for(T k = start; k <= stop + epsilon; k += step) { - output.push_back(k); - } - } else { - for(T k = start; k <= stop; k += step) { output.push_back(k); } - } - } - return output; -} } // namespace @@ -192,17 +149,23 @@ int main(int argc, char** argv) ->notifier( [&config](std::size_t key) { config.nsamples_to_read(key); }), "total number of samples to read from start_sample")( + "targets_file", + po::value()->default_value("")->notifier( + [&config](std::string key) { + config.targets_file(key); + }), "update beam names and positions from this file" + )( "required_beams", po::value()->default_value("")->notifier( [&config](std::string key) { - config.required_beams(get_list_from_string(key)); + config.required_beams(skyweaver::get_list_from_string(key)); }), "Comma separated list of beams to process. Syntax - beam1, beam2, " "beam3:beam4:step, beam5 etc..")( "required_dms", po::value()->default_value("")->notifier( [&config](std::string key) { - config.required_dms(get_list_from_string(key)); + config.required_dms(skyweaver::get_list_from_string(key)); }), "Comma separated list of DMs to process. Syntax - dm1, dm2, " "dm1:dm2:step, etc..")( @@ -210,7 +173,7 @@ int main(int argc, char** argv) po::value() ->default_value(config.out_stokes()) ->notifier([&config](std::string key) { - if(key.find_first_not_of("IQUV") != std::string::npos) { + if(key.find_first_not_of("IQUVL") != std::string::npos) { throw std::runtime_error("Invalid Stokes mode: " + key); } config.out_stokes(key); From ff71497be6db64406ffc0b95a5fa5255c43b768a Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 12 Dec 2024 17:58:03 +0100 Subject: [PATCH 40/65] fix filename convention --- cpp/skyweaver/detail/MultiFileWriter.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 16b6c05..439f1ab 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -121,7 +121,7 @@ MultiFileWriter::get_basefilename(VectorType const& stream_data, base_filename << _config.prefix << "_"; } base_filename << get_formatted_time(_header.utc_start) << "_" << stream_idx - << "_" << std::fixed << std::setprecision(3) + << "_cdm_" << std::fixed << std::setprecision(3) << std::setfill('0') << std::setw(9) << stream_data.reference_dm(); From 45964ca84d7fc2bae5f461237938902b9820abf0 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Tue, 17 Dec 2024 13:31:26 +0100 Subject: [PATCH 41/65] draft documentation --- README.md | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/README.md b/README.md index ffc3e94..038f3bd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,177 @@ # skyweaver Implementation of an offline FBFUSE beamformer for the MeerKAT telescope + +# Installation + +It is easiest to use the software inside a Docker container. Two dockerfiles is included part of this repository: To compile the c++ and python parts respectively. + +# Usage + +## Step 1: Get delays for the beamformer + +### Start with the skyweaver python CLI + +```bash +alias sw="python /path/to/python/skyweaver/cli.py" +sw --help +``` +This will print: +```console +usage: skyweaver [-h] {metadata,delays} ... + +positional arguments: + {metadata,delays} sub-command help + metadata Tools for observation metadata files + delays Tools for delay files + +optional arguments: + -h, --help show this help message and exit +``` + +### Get the metadata for the corresponding observation + +This is done outside this repository + +### Obtain metadata information + +```bash +sw metadata show +``` + +This will produce an output like the following: +```console +sw metadata show bvrmetadata_2024-02-16T10\:50\:46_72275.hdf5 +---------------Array configuration---------------- +Nantennas: 57 +Subarray: m000,m002,m003,m004,m005,m007,m008,m009,m010,m011,m012,m014,m015,m016, + m017,m018,m019,m020,m021,m022,m023,m024,m025,m026,m027,m029,m030,m031, + m032,m033,m034,m035,m036,m037,m038,m039,m040,m041,m042,m043,m044,m045, + m046,m048,m049,m050,m051,m053,m054,m056,m057,m058,m059,m060,m061,m062, + m063 +Centre frequency: 1284000000.0 Hz +Bandwidth: 856000000.0 Hz +Nchannels: 4096 +Sync epoch (UNIX): 1708039531.0 +Project ID: - +Schedule block ID: - +CBF version: cbf_dev +--------------------Pointings--------------------- +#0 J1644-4559 2024-02-16T11:16:08.957000000 until 2024-02-16T11:21:04.865000000 (UTC) + 1708082168.957 until 1708082464.865 (UNIX) + 72996182384029 until 73502776880016 (SAMPLE CLOCK) +#1 J1644-4559_Offset1 2024-02-16T11:21:22.092000000 until 2024-02-16T11:26:15.682000000 (UTC) + 1708082482.092 until 1708082775.682 (UNIX) + 73532269504013 until 74034895583866 (SAMPLE CLOCK) +#2 J1644-4559_Offset2 2024-02-16T11:26:34.536000000 until 2024-02-16T11:31:25.723000000 (UTC) + 1708082794.536 until 1708083085.723 (UNIX) + 74067173632022 until 74565685776084 (SAMPLE CLOCK) +#3 M28 2024-02-16T11:31:52.503000000 until 2024-02-16T12:01:51.651000000 (UTC) + 1708083112.503 until 1708084911.651 (UNIX) + 74611533136035 until 77691674512039 (SAMPLE CLOCK) +#4 J0437-4715 2024-02-16T12:03:06.117000000 until 2024-02-16T12:08:04.364000000 (UTC) + 1708084986.117 until 1708085284.364 (UNIX) + 77819160304176 until 78329759168140 (SAMPLE CLOCK) +``` + +### Create a config file in .yml format +Here is an example - there are comments to explain each parameter. + +```.yml +created_by: Vivek +beamformer_config: + # The total number of beams to be produced (must be a multiple of 32). This needs to be <= the number that SKYWEAVER is compiled for. + total_nbeams: 800 + # The number of time samples that will be accumulated after detection, inside the beamformer + tscrunch: 4 + # The number of frequency channels that will be accumulated after detection, inside the beamformer + # Will be coerced to 1 if coherent dedispersion is specified. + fscrunch: 1 + # The Stokes product to be calculated in the beamformer (I=0, Q=1, U=2, V=3) + stokes_mode: 0 + # Enable CB-IB subtraction in the beamformer + subtract_ib: True + + # Dispersion measure for coherent / incoherent dedispersion in pc cm^-3 + # A dispersion plan definition string " + # "(::::) or " + # "(:) " + # "or ()") +# Each DD plan is a "Stream" with zero indexed stream-ids + +ddplan: + - "478.6:478.6:478.6:1:1" #stream-id=0 + - "0.00:478.6:478.6:1:1" #stream-id=1 + +# every beamset can contain arbitrary set of antennas, corresponding targeted beams, and tiled beams +# total number of beams across all beamsets should be <= the number of beams that SKYWEAVER is compiled for. +beam_sets: + + - antenna_set: ['m000','m002','m003','m004','m005','m007','m008','m009','m010','m011', + 'm012','m014','m015','m016','m017','m018','m019','m020','m021','m022', + 'm023','m024','m025','m026','m027','m029','m030','m031','m032','m033', + 'm034','m035','m036','m037','m038','m039','m040','m041','m042','m043', + 'm044','m045','m046','m048','m049','m050','m051','m053','m054','m056', + 'm057','m058','m059','m060','m061','m062','m063'] + beams: [] + tilings: + - nbeams: 32 + reference_frequency: null + target: "J1644-4559,radec,16:44:49.273,-45:59:09.71" + overlap: 0.9 +``` + + + +### Create delay file for the corresponding pointing + +```bash +sw delays create --pointing-idx 0 --outfile J1644-4559_pointing_0.delays --step 4 bvrmetadata_2024-02-16T10\:50\:46_72275.hdf5 J1644-4559_boresight.yaml +``` + +This produces a `.delays` file used for beamforming, and a `.targets` file that contains beam metadata. There are also other files produced here for reproducibility and for visualisation. + +## Step 2: Initialise input and compile skyweaver + +### Create a list of dada files that correspond to the pointing + +```console +ls /b/u/vivek/00_DADA_FILES/J1644-4559/2024-02-16-11\:16\:08/L/48/*dada -1 > /bscratch/vivek/skyweaver_tests/J1644-4559_boresight_dadafiles.list +``` + +### Compile skyweaver + +This is done inside the dockerfile too. Either edit that to produce a docker image that has the software precompiled, or compile separately. + +```bash + cmake -S . -B $cmake_tmp_dir -DENABLE_TESTING=0 -DCMAKE_INSTALL_PREFIX=$install_dir -DARCH=native -DPSRDADA_INCLUDE_DIR=/usr/local/include/psrdada -DPSRDADACPP_INCLUDE_DIR=/usr/local/include/psrdada_cpp -DSKYWEAVER_NANTENNAS=64 -DSKYWEAVER_NBEAMS=${nbeams} -DSKYWEAVER_NCHANS=64 -DSKYWEAVER_IB_SUBTRACTION=1 -DCMAKE_BUILD_TYPE=RELEASE -DSKYWEAVER_CB_TSCRUNCH=${tscrunch} -DSKYWEAVER_IB_TSCRUNCH=${tscrunch}; + cd $cmake_tmp_dir + make -j 16 +``` +This compilation produces two binaries: `skyweavercpp` and `skycleaver` +## Step 3: Run the beamformer + +```bash + +/path/to/skyweavercpp --input-file J1644-4559_boresight_dadafiles.list --delay-file J1644-4559_pointing_0.delays --output-dir=/bscratch/vivek/skyweaver_out --gulp-size=32768 --log-level=warning --output-level=12 --stokes-mode I +``` + +Change the output level to 7 for bright pulsars like J1644-4559. + +This will produce `.tdb` files for the corresponding bridge. Run Step 3 for ALL 64 bridges with their corresponding dada file lists. These are DADA format files with the dimensions of TIME, INCOHERENT DM and BEAM as the order. For stokes I mode, The datatype is `int8_t`. For IQUV it is `char4`. + +## Steo 4: Cleave all bridges to form Filterbanks + +Here we cleave the 64 TDB[I/Q/U/V/IV/QU/IQUV] files to produce `NDM*NBEAMS*NSTOKES` number of T(F=64) files. + +to run this, do + +```bash + +/path/to/skycleaver -r /bscratch/vivek/skyweaver_out --output-dir /bscratch/vivek/skycleaver_out --nsamples-per-block 65536 --nthreads 32 --stream-id 0 --targets_file/bscratch/vivek/skyweaver_out/swdlays_J1644-4559.targets --out-stokes I --required_beams 0 + +``` + +This will produce a standard sigproc format `.fil` file that can be used for traditional processing. + + + From 8610cab48b4eb0eefefcf144d1daa7c81e8dfc2e Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Sun, 22 Dec 2024 14:35:08 +0100 Subject: [PATCH 42/65] Revert "compiling now and beamformer tests running" This reverts commit 820efb994e341feea8119be5ec12b90143905b90. --- cpp/skyweaver/StatisticsCalculator.cuh | 2 +- cpp/skyweaver/beamformer_utils.cuh | 16 ++--- cpp/skyweaver/benchmark/beamformer_bench.cu | 2 +- cpp/skyweaver/src/StatisticsCalculator.cu | 10 +-- .../test/CoherentBeamformerTester.cuh | 17 +---- .../test/IncoherentBeamformerTester.cuh | 7 +- .../test/src/CoherentBeamformerTester.cu | 71 ++++++++----------- .../test/src/IncoherentBeamformerTester.cu | 34 ++++----- 8 files changed, 64 insertions(+), 95 deletions(-) diff --git a/cpp/skyweaver/StatisticsCalculator.cuh b/cpp/skyweaver/StatisticsCalculator.cuh index 1b9dee3..5767a78 100644 --- a/cpp/skyweaver/StatisticsCalculator.cuh +++ b/cpp/skyweaver/StatisticsCalculator.cuh @@ -104,7 +104,7 @@ class StatisticsCalculator void dump_scalings(std::string const& timestamp, std::string const& tag, std::string const& path, - ScalingVectorTypeH const& ar) const; + thrust::host_vector const& ar) const; private: PipelineConfig const& _config; diff --git a/cpp/skyweaver/beamformer_utils.cuh b/cpp/skyweaver/beamformer_utils.cuh index 15ec773..ae165e8 100644 --- a/cpp/skyweaver/beamformer_utils.cuh +++ b/cpp/skyweaver/beamformer_utils.cuh @@ -275,11 +275,11 @@ struct IncoherentBeamSubtract { { if constexpr(S == StokesParameter::I) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.x); - } else if constexpr(S == StokesParameter::Q) { + } constexpr(S == StokesParameter::Q) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.y); - } else if constexpr(S == StokesParameter::U) { + } constexpr(S == StokesParameter::U) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.z); - } else if constexpr(S == StokesParameter::V) { + } constexpr(S == StokesParameter::V) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.w); } else { static_no_match(); @@ -290,17 +290,17 @@ struct IncoherentBeamSubtract { struct Rescale { template static inline __host__ __device__ void apply(T const& power, - float4 const& offset, - float4 const& scale_factor, + float const& offset, + float const& scale_factor, T& result) { if constexpr(S == StokesParameter::I) { AT(result, I) = rintf((AT(power, I) - offset.x) / scale_factor.x); - } else if constexpr(S == StokesParameter::Q) { + } constexpr(S == StokesParameter::Q) { AT(result, I) = rintf((AT(power, I) - offset.y) / scale_factor.y); - } else if constexpr(S == StokesParameter::U) { + } constexpr(S == StokesParameter::U) { AT(result, I) = rintf((AT(power, I) - offset.z) / scale_factor.z); - } else if constexpr(S == StokesParameter::V) { + } constexpr(S == StokesParameter::V) { AT(result, I) = rintf((AT(power, I) - offset.w) / scale_factor.w); } else { static_no_match(); diff --git a/cpp/skyweaver/benchmark/beamformer_bench.cu b/cpp/skyweaver/benchmark/beamformer_bench.cu index 663cb3b..a179741 100644 --- a/cpp/skyweaver/benchmark/beamformer_bench.cu +++ b/cpp/skyweaver/benchmark/beamformer_bench.cu @@ -70,7 +70,7 @@ class BeamformerBencher: public benchmark::Fixture typename CoherentBeamformer::PowerVectorTypeD btf_powers_gpu; typename CoherentBeamformer::ScalingVectorTypeD scales; typename CoherentBeamformer::ScalingVectorTypeD offsets; - typename IncoherentBeamformer::BeamsetWeightsVectorTypeD _antenna_weights; + typename CoherentBeamformer::ScalingVectorTypeD _antenna_weights; typename IncoherentBeamformer::PowerVectorTypeD tf_powers_gpu; typename IncoherentBeamformer::RawPowerVectorTypeD tf_powers_raw_gpu; typename CoherentBeamformer::MappingVectorTypeD _beamset_mapping; diff --git a/cpp/skyweaver/src/StatisticsCalculator.cu b/cpp/skyweaver/src/StatisticsCalculator.cu index a8dcb33..a2cf68c 100644 --- a/cpp/skyweaver/src/StatisticsCalculator.cu +++ b/cpp/skyweaver/src/StatisticsCalculator.cu @@ -191,15 +191,15 @@ void StatisticsCalculator::update_scalings( float var_p1_sum = 0.0f; // sum(sigma_p1^2) float quad_sum = 0.0f; // sum(sigma_p0^4 + sigma_p1^4) float sum_mul = 0.0f; // sum(sigma_p0^2 * sigma_p1^2) - for (int a_idx = 0; a_idx < _config.npol(); ++a_idx) { + for (int a_idx = 0; a_idx < _cofig.npol(); ++a_idx) { // stats are in FPA order // weights are in FPBA order const int input_idx = f_idx * pa + a_idx; Statistics p0 = _stats_h[input_idx]; - Statistics p1 = _stats_h[input_idx + a]; + Statistics p1 = _stats_h[input_idx + nantennas]; const float weight = beamset_weights[beamset_idx * a + a_idx]; - const float std_p0 = p0.std * weight * weights_amp; - const float std_p1 = p1.std * weight * weights_amp; + const float std_p0 = p0.std * weight; + const float std_p1 = p1.std * weight; const float var_p0 = std_p0 * std_p0; const float var_p1 = std_p1 * std_p1; meanI += var_p0 + var_p1; @@ -279,7 +279,7 @@ void StatisticsCalculator::dump_scalings( BOOST_LOG_TRIVIAL(error) << error_message.str(); return; } - writer.write(reinterpret_cast(thrust::raw_pointer_cast(ar.data())), ar.size() * sizeof(ScalingType)); + writer.write(static_cast(ar.data()), ar.size() * sizeof(ScalingType)); writer.close(); } diff --git a/cpp/skyweaver/test/CoherentBeamformerTester.cuh b/cpp/skyweaver/test/CoherentBeamformerTester.cuh index e956de1..fd60262 100644 --- a/cpp/skyweaver/test/CoherentBeamformerTester.cuh +++ b/cpp/skyweaver/test/CoherentBeamformerTester.cuh @@ -21,37 +21,26 @@ class CoherentBeamformerTester: public ::testing::Test typedef CoherentBeamformer CoherentBeamformer; typedef IncoherentBeamformer IncoherentBeamformer; typedef CoherentBeamformer::VoltageVectorTypeD VoltageVectorTypeD; - typedef FTPAVoltagesH VoltageVectorTypeH; - typedef CoherentBeamformer::PowerVectorTypeD PowerVectorTypeD; typedef TFBPowersH HostPowerVectorType; - typedef IncoherentBeamformer::PowerVectorTypeD IBPowerVectorTypeD; typedef BTFPowersH HostIBPowerVectorType; - typedef IncoherentBeamformer::RawPowerVectorTypeD RawIBPowerVectorTypeD; typedef BTFPowersH HostRawIBPowerVectorType; - typedef CoherentBeamformer::WeightsVectorTypeD WeightsVectorTypeD; typedef thrust::host_vector WeightsVectorTypeH; - typedef CoherentBeamformer::ScalingVectorTypeD ScalingVectorTypeD; typedef thrust::host_vector HostScalingVectorType; - typedef CoherentBeamformer::MappingVectorTypeD MappingVectorTypeD; typedef thrust::host_vector MappingVectorTypeH; - - typedef IncoherentBeamformer::BeamsetWeightsVectorTypeD BeamsetWeightsVectorTypeD; - typedef thrust::host_vector - BeamsetWeightsVectorTypeH; protected: void SetUp() override; @@ -72,8 +61,8 @@ class CoherentBeamformerTester: public ::testing::Test int nbeams, int nantennas, int npol, - float4 const* scales, - float4 const* offsets, + float const* scales, + float const* offsets, float const* antenna_weights, int const* beamset_mapping); @@ -81,7 +70,7 @@ class CoherentBeamformerTester: public ::testing::Test WeightsVectorTypeD const& fbpa_weights_gpu, ScalingVectorTypeD const& scales_gpu, ScalingVectorTypeD const& offsets_gpu, - BeamsetWeightsVectorTypeD const& antenna_weights, + ScalingVectorTypeD const& antenna_weights, MappingVectorTypeD const& beamset_mapping, PowerVectorTypeD& btf_powers_gpu, int nsamples); diff --git a/cpp/skyweaver/test/IncoherentBeamformerTester.cuh b/cpp/skyweaver/test/IncoherentBeamformerTester.cuh index 5983321..37da649 100644 --- a/cpp/skyweaver/test/IncoherentBeamformerTester.cuh +++ b/cpp/skyweaver/test/IncoherentBeamformerTester.cuh @@ -31,9 +31,6 @@ class IncoherentBeamformerTester: public ::testing::Test typedef IncoherentBeamformer::ScalingVectorTypeD ScalingVectorTypeD; typedef thrust::host_vector HostScalingVectorType; - typedef IncoherentBeamformer::BeamsetWeightsVectorTypeD BeamsetWeightsVectorTypeD; - typedef thrust::host_vector - HostBeamsetWeightsVectorType; protected: void SetUp() override; @@ -54,7 +51,7 @@ class IncoherentBeamformerTester: public ::testing::Test int nantennas, HostScalingVectorType const& scale, HostScalingVectorType const& offset, - HostBeamsetWeightsVectorType const& beamset_weights, + HostScalingVectorType const& beamset_weights, int nbeamsets); void compare_against_host(VoltageVectorTypeD const& ftpa_voltages_gpu, @@ -62,7 +59,7 @@ class IncoherentBeamformerTester: public ::testing::Test DevicePowerVectorType& tf_powers_gpu, ScalingVectorTypeD const& scaling_vector, ScalingVectorTypeD const& offset_vector, - BeamsetWeightsVectorTypeD const& beamset_weights, + ScalingVectorTypeD const& beamset_weights, int ntimestamps, int nbeamsets); diff --git a/cpp/skyweaver/test/src/CoherentBeamformerTester.cu b/cpp/skyweaver/test/src/CoherentBeamformerTester.cu index 9a98df5..a369015 100644 --- a/cpp/skyweaver/test/src/CoherentBeamformerTester.cu +++ b/cpp/skyweaver/test/src/CoherentBeamformerTester.cu @@ -51,8 +51,8 @@ void CoherentBeamformerTester::beamformer_c_reference( int nbeams, int nantennas, int npol, - float4 const* scales, - float4 const* offsets, + float const* scales, + float const* offsets, float const* antenna_weights, int const* beamset_mapping) { @@ -157,7 +157,7 @@ void CoherentBeamformerTester::compare_against_host( WeightsVectorTypeD const& fbpa_weights_gpu, ScalingVectorTypeD const& scales_gpu, ScalingVectorTypeD const& offsets_gpu, - BeamsetWeightsVectorTypeD const& antenna_weights, + ScalingVectorTypeD const& antenna_weights, MappingVectorTypeD const& beamset_mapping, PowerVectorTypeD& btf_powers_gpu, int nsamples) @@ -170,7 +170,7 @@ void CoherentBeamformerTester::compare_against_host( HostScalingVectorType scales = scales_gpu; HostScalingVectorType offsets = offsets_gpu; - thrust::host_vector antenna_weights_host = antenna_weights; + HostScalingVectorType antenna_weights_host = antenna_weights; MappingVectorTypeH beamset_mapping_host = beamset_mapping; beamformer_c_reference( @@ -227,36 +227,29 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) const float input_level = 32.0f; const double pi = std::acos(-1); config.output_level(input_level); - - float4 cb_offset_val{ - static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(127.0f * input_level, 2)) / config.output_level()), - 0.0f, - 0.0f, - 0.0f - }; - float cb_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * std::pow(config.nantennas() * std::pow(127.0f * input_level, 2), 2) ) / config.output_level()); - float4 cb_scale_val{ - cb_scaling, - cb_scaling, - cb_scaling, - cb_scaling - }; + float scale = + std::pow(127.0f * input_level * + std::sqrt(static_cast(config.nantennas())), + 2); + float dof = 2 * config.cb_tscrunch() * config.cb_fscrunch() * config.npol(); + float offset_val = (scale * dof); + float scale_val = (scale * std::sqrt(2 * dof) / config.output_level()); // Set constant scales and offsets for all channels and beamsets typename CBT::ScalingVectorTypeD cb_scales( config.nchans() / config.cb_fscrunch() * nbeamsets, - cb_scale_val); + scale_val); typename CBT::ScalingVectorTypeD cb_offsets( config.nchans() / config.cb_fscrunch() * nbeamsets, - cb_offset_val); + offset_val); // Map all beams to the first beamset by default typename CBT::MappingVectorTypeD beamset_mapping(config.nbeams(), 0); // Enable all antennas in all beamsets - typename thrust::device_vector beamset_weights(config.nantennas() * + typename CBT::ScalingVectorTypeD beamset_weights(config.nantennas() * nbeamsets, - 1.0f); + 1.0f); /** This currently causes tests to fail even though the weights are the same for @@ -267,6 +260,9 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) */ beamset_mapping[0] = 1; + BOOST_LOG_TRIVIAL(info) << "CB scaling: " << scale_val; + BOOST_LOG_TRIVIAL(info) << "CB offset: " << offset_val; + std::default_random_engine generator; std::normal_distribution normal_dist(0.0, input_level); std::uniform_real_distribution uniform_dist(0.0, 2 * pi); @@ -284,6 +280,9 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) std::size_t input_size = (ntimestamps * config.nantennas() * config.nchans() * config.nsamples_per_heap() * config.npol()); + BOOST_LOG_TRIVIAL(info) << "FTPA input dims: " << config.nchans() << ", " + << ntimestamps * config.nsamples_per_heap() << ", " + << config.npol() << ", " << config.nantennas(); int nsamples = config.nsamples_per_heap() * ntimestamps; std::size_t weights_size = @@ -315,28 +314,20 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) typename CBT::WeightsVectorTypeD fbpa_weights_gpu = fbpa_weights_host; typename CBT::PowerVectorTypeD tfb_powers_gpu; - float4 ib_offset_val{ - static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 2)) / config.output_level()), - 0.0f, - 0.0f, - 0.0f - }; - - float ib_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 4)) / config.output_level()); - float4 ib_scale_val{ - ib_scaling, - ib_scaling, - ib_scaling, - ib_scaling - }; - + // Note that below even though this is for the IB we have to use the + // CB scrunching parameters to make sure we get the right data out. + float ib_scale = std::pow(input_level, 2); + float ib_dof = 2 * config.cb_tscrunch() * config.cb_fscrunch() * + config.nantennas() * config.npol(); + float ib_power_offset = ib_scale * ib_dof; + float ib_power_scaling = + ib_scale * std::sqrt(2 * ib_dof) / config.output_level(); typename CBT::ScalingVectorTypeD ib_scales( config.nchans() / config.cb_fscrunch() * nbeamsets, - ib_scale_val); + ib_power_scaling); typename CBT::ScalingVectorTypeD ib_offset( config.nchans() / config.cb_fscrunch() * nbeamsets, - ib_offset_val); - + ib_power_offset); typename CBT::IBPowerVectorTypeD tf_powers_gpu; typename CBT::RawIBPowerVectorTypeD tf_powers_raw_gpu; diff --git a/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu b/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu index d7af1c5..a52f675 100644 --- a/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu +++ b/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu @@ -49,7 +49,7 @@ void IncoherentBeamformerTester::beamformer_c_reference( int nantennas, HostScalingVectorType const& scale, HostScalingVectorType const& offset, - HostBeamsetWeightsVectorType const& beamset_weights, + HostScalingVectorType const& beamset_weights, int nbeamsets) { static_assert(SKYWEAVER_NPOL == 2, "Tests only work for dual poln data."); @@ -109,7 +109,7 @@ void IncoherentBeamformerTester::compare_against_host( DevicePowerVectorType& tf_powers_gpu, ScalingVectorTypeD const& scaling_vector, ScalingVectorTypeD const& offset_vector, - BeamsetWeightsVectorTypeD const& beamset_weights, + ScalingVectorTypeD const& beamset_weights, int ntimestamps, int nbeamsets) { @@ -118,7 +118,7 @@ void IncoherentBeamformerTester::compare_against_host( HostRawPowerVectorType tf_powers_raw_cuda = tf_powers_raw_gpu; HostScalingVectorType h_scaling_vector = scaling_vector; HostScalingVectorType h_offset_vector = offset_vector; - HostBeamsetWeightsVectorType h_beamset_weights = beamset_weights; + HostScalingVectorType h_beamset_weights = beamset_weights; HostRawPowerVectorType tf_powers_raw_host; tf_powers_raw_host.like(tf_powers_raw_gpu); HostPowerVectorType tf_powers_host; @@ -175,31 +175,23 @@ TYPED_TEST(IncoherentBeamformerTester, ib_representative_noise_test) static_cast(std::lround(normal_dist(generator))); } - float4 ib_offset_val{ - static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 2)) / config.output_level()), - 0.0f, - 0.0f, - 0.0f - }; - - float ib_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 4)) / config.output_level()); - float4 ib_scale_val{ - ib_scaling, - ib_scaling, - ib_scaling, - ib_scaling - }; + float ib_scale = std::pow(input_level, 2); + float ib_dof = 2 * config.ib_tscrunch() * config.ib_fscrunch() * + config.nantennas() * config.npol(); + float ib_power_offset = ib_scale * ib_dof; + float ib_power_scaling = + ib_scale * std::sqrt(2 * ib_dof) / config.output_level(); for(int nbeamsets = 1; nbeamsets < 5; ++nbeamsets) { typename IBT::ScalingVectorTypeD scales( config.nchans() / config.ib_fscrunch() * nbeamsets, - ib_scale_val); + ib_power_scaling); typename IBT::ScalingVectorTypeD offset( config.nchans() / config.ib_fscrunch() * nbeamsets, - ib_offset_val); - typename IBT::BeamsetWeightsVectorTypeD beamset_weights(config.nantennas() * + ib_power_offset); + typename IBT::ScalingVectorTypeD beamset_weights(config.nantennas() * nbeamsets, - 1.0f); + 1.0f); typename IBT::VoltageVectorTypeD ftpa_voltages_gpu = ftpa_voltages_host; typename IBT::DevicePowerVectorType tf_powers_gpu; From 112a857d41062342d1aff003e416b2f180e98172 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Sun, 22 Dec 2024 14:35:32 +0100 Subject: [PATCH 43/65] Revert "dirty pass on scaling change" This reverts commit cbaf03bca758f85ba14e07600d1a3ac12e036f41. --- cpp/skyweaver/CoherentBeamformer.cuh | 6 +- cpp/skyweaver/IncoherentBeamformer.cuh | 10 +- cpp/skyweaver/StatisticsCalculator.cuh | 6 +- cpp/skyweaver/beamformer_utils.cuh | 34 ++--- cpp/skyweaver/src/CoherentBeamformer.cu | 10 +- cpp/skyweaver/src/IncoherentBeamformer.cu | 14 +- cpp/skyweaver/src/StatisticsCalculator.cu | 166 ++++++++++------------ 7 files changed, 109 insertions(+), 137 deletions(-) diff --git a/cpp/skyweaver/CoherentBeamformer.cuh b/cpp/skyweaver/CoherentBeamformer.cuh index d10cc35..dc47b9c 100644 --- a/cpp/skyweaver/CoherentBeamformer.cuh +++ b/cpp/skyweaver/CoherentBeamformer.cuh @@ -33,8 +33,8 @@ __global__ void bf_ftpa_general_k( int2 const* __restrict__ ftpa_voltages, int2 const* __restrict__ fbpa_weights, typename BfTraits::QuantisedPowerType* __restrict__ btf_powers, - float4 const* __restrict__ output_scale, - float4 const* __restrict__ output_offset, + float const* __restrict__ output_scale, + float const* __restrict__ output_offset, int const* __restrict__ beamset_mapping, typename BfTraits::RawPowerType const* __restrict__ ib_powers, int nsamples); @@ -55,7 +55,7 @@ class CoherentBeamformer typedef BTFPowersD RawPowerVectorTypeD; // FBA order (assuming equal weight per polarisation) typedef thrust::device_vector WeightsVectorTypeD; - typedef thrust::device_vector ScalingVectorTypeD; + typedef thrust::device_vector ScalingVectorTypeD; typedef thrust::device_vector MappingVectorTypeD; public: diff --git a/cpp/skyweaver/IncoherentBeamformer.cuh b/cpp/skyweaver/IncoherentBeamformer.cuh index ef18b6d..e40d73b 100644 --- a/cpp/skyweaver/IncoherentBeamformer.cuh +++ b/cpp/skyweaver/IncoherentBeamformer.cuh @@ -34,8 +34,8 @@ __global__ void icbf_ftpa_general_k( char2 const* __restrict__ ftpa_voltages, typename BfTraits::RawPowerType* __restrict__ tf_powers_raw, typename BfTraits::QuantisedPowerType* __restrict__ tf_powers, - float4 const* __restrict__ output_scale, - float4 const* __restrict__ output_offset, + float const* __restrict__ output_scale, + float const* __restrict__ output_offset, float const* __restrict__ antenna_weights, int nsamples, int nbeamsets); @@ -52,9 +52,7 @@ class IncoherentBeamformer // TF order typedef BTFPowersD RawPowerVectorTypeD; // TF order - typedef thrust::device_vector ScalingVectorTypeD; // Always IQUV scalings - // BsA order - typedef thrust::device_vector BeamsetWeightsVectorTypeD; + typedef thrust::device_vector ScalingVectorTypeD; public: /** @@ -88,7 +86,7 @@ class IncoherentBeamformer PowerVectorTypeD& output, ScalingVectorTypeD const& output_scale, ScalingVectorTypeD const& output_offset, - BeamsetWeightsVectorTypeD const& antenna_weights, + ScalingVectorTypeD const& antenna_weights, int nbeamsets, cudaStream_t stream); diff --git a/cpp/skyweaver/StatisticsCalculator.cuh b/cpp/skyweaver/StatisticsCalculator.cuh index 5767a78..204f82b 100644 --- a/cpp/skyweaver/StatisticsCalculator.cuh +++ b/cpp/skyweaver/StatisticsCalculator.cuh @@ -39,11 +39,9 @@ struct StatisticsFileHeader { class StatisticsCalculator { public: - typedef float4 ScalingType; // Always caclulate for full stokes + typedef float ScalingType; typedef thrust::device_vector ScalingVectorTypeD; typedef thrust::host_vector ScalingVectorTypeH; - typedef thrust::device_vector BeamsetWeightsVectorTypeD; - typedef thrust::host_vector BeamsetWeightsVectorTypeH; typedef FPAStatsD StatisticsVectorTypeD; typedef FPAStatsH StatisticsVectorTypeH; @@ -95,7 +93,7 @@ class StatisticsCalculator * @brief Update the scaling arrays based on the last statistics * calculation. */ - void update_scalings(BeamsetWeightsVectorTypeH const& beamset_weights, + void update_scalings(ScalingVectorTypeH const& beamset_weights, int nbeamsets); private: diff --git a/cpp/skyweaver/beamformer_utils.cuh b/cpp/skyweaver/beamformer_utils.cuh index ae165e8..74d4873 100644 --- a/cpp/skyweaver/beamformer_utils.cuh +++ b/cpp/skyweaver/beamformer_utils.cuh @@ -270,20 +270,10 @@ struct IncoherentBeamSubtract { static inline __host__ __device__ void apply(T const& power, T const& ib_power, float const& ib_mutliplier, // 127^2 as default - float4 const& scale_factor, + float const& scale_factor, T& result) { - if constexpr(S == StokesParameter::I) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.x); - } constexpr(S == StokesParameter::Q) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.y); - } constexpr(S == StokesParameter::U) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.z); - } constexpr(S == StokesParameter::V) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.w); - } else { - static_no_match(); - } + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor); } }; @@ -295,16 +285,10 @@ struct Rescale { T& result) { if constexpr(S == StokesParameter::I) { - AT(result, I) = rintf((AT(power, I) - offset.x) / scale_factor.x); - } constexpr(S == StokesParameter::Q) { - AT(result, I) = rintf((AT(power, I) - offset.y) / scale_factor.y); - } constexpr(S == StokesParameter::U) { - AT(result, I) = rintf((AT(power, I) - offset.z) / scale_factor.z); - } constexpr(S == StokesParameter::V) { - AT(result, I) = rintf((AT(power, I) - offset.w) / scale_factor.w); + AT(result, I) = rintf((AT(power, I) - offset) / scale_factor); } else { - static_no_match(); - } + AT(result, I) = rintf(AT(power, I) / scale_factor); + } } }; @@ -351,16 +335,16 @@ struct StokesTraits ib_subtract(RawPowerType const& power, RawPowerType const& ib_power, float const& ib_mutliplier, - float4 const& scale_factor) { + float const& scale_factor) { RawPowerType result{}; - Iterate<0, Stokes...>::template apply(power, ib_power, ib_mutliplier, scale_factor, result); + Iterate<0, Stokes...>::template apply(power, ib_power, ib_mutliplier, scale_factor, result); return result; } static inline __host__ __device__ RawPowerType rescale(RawPowerType const& power, - float4 const& offset, - float4 const& scale_factor) { + float const& offset, + float const& scale_factor) { RawPowerType result{}; Iterate<0, Stokes...>::template apply(power, offset, scale_factor, result); return result; diff --git a/cpp/skyweaver/src/CoherentBeamformer.cu b/cpp/skyweaver/src/CoherentBeamformer.cu index ff1e563..43a8688 100644 --- a/cpp/skyweaver/src/CoherentBeamformer.cu +++ b/cpp/skyweaver/src/CoherentBeamformer.cu @@ -14,8 +14,8 @@ __global__ void bf_ftpa_general_k( int2 const* __restrict__ ftpa_voltages, int2 const* __restrict__ fbpa_weights, typename BfTraits::QuantisedPowerType* __restrict__ tfb_powers, - float4 const* __restrict__ output_scale, - float4 const* __restrict__ output_offset, + float const* __restrict__ output_scale, + float const* __restrict__ output_offset, int const* __restrict__ beamset_mapping, typename BfTraits::RawPowerType const* __restrict__ ib_powers, int nsamples) @@ -156,7 +156,7 @@ __global__ void bf_ftpa_general_k( int const ib_power_idx = beamset_idx * nsamps_out * gridDim.y + output_sample_idx * gridDim.y + blockIdx.y; int const scloff_idx = beamset_idx * gridDim.y + blockIdx.y; - float4 scale = output_scale[scloff_idx]; + float scale = output_scale[scloff_idx]; typename BfTraits::RawPowerType ib_power = ib_powers[ib_power_idx]; #if SKYWEAVER_IB_SUBTRACTION /* @@ -246,9 +246,9 @@ void CoherentBeamformer::beamform( char2 const* fbpa_weights_ptr = thrust::raw_pointer_cast(weights.data()); typename BfTraits::QuantisedPowerType* tfb_powers_ptr = thrust::raw_pointer_cast(output.data()); - float4 const* power_scaling_ptr = + float const* power_scaling_ptr = thrust::raw_pointer_cast(output_scale.data()); - float4 const* power_offset_ptr = + float const* power_offset_ptr = thrust::raw_pointer_cast(output_offset.data()); typename BfTraits::RawPowerType const* ib_powers_ptr = thrust::raw_pointer_cast(ib_powers.data()); diff --git a/cpp/skyweaver/src/IncoherentBeamformer.cu b/cpp/skyweaver/src/IncoherentBeamformer.cu index ee4a80e..c91618e 100644 --- a/cpp/skyweaver/src/IncoherentBeamformer.cu +++ b/cpp/skyweaver/src/IncoherentBeamformer.cu @@ -17,8 +17,8 @@ __global__ void icbf_ftpa_general_k( char2 const* __restrict__ ftpa_voltages, typename BfTraits::RawPowerType* __restrict__ tf_powers_raw, typename BfTraits::QuantisedPowerType* __restrict__ tf_powers, - float4 const* __restrict__ output_scale, - float4 const* __restrict__ output_offset, + float const* __restrict__ output_scale, + float const* __restrict__ output_offset, float const* __restrict__ antenna_weights, int nsamples, int nbeamsets) @@ -79,9 +79,9 @@ __global__ void icbf_ftpa_general_k( const typename BfTraits::RawPowerType power_f32 = acc_buffer[0]; const int output_idx = beamset_idx * gridDim.x * gridDim.y + output_t_idx * gridDim.y + output_f_idx; - const float4 scale = + const float scale = output_scale[beamset_idx * gridDim.y + output_f_idx]; - const float4 offset = + const float offset = output_offset[beamset_idx * gridDim.y + output_f_idx]; tf_powers_raw[output_idx] = power_f32; tf_powers[output_idx] = @@ -112,7 +112,7 @@ void IncoherentBeamformer::beamform( PowerVectorTypeD& output, ScalingVectorTypeD const& output_scale, ScalingVectorTypeD const& output_offset, - BeamsetWeightsVectorTypeD const& antenna_weights, + ScalingVectorTypeD const& antenna_weights, int nbeamsets, cudaStream_t stream) { @@ -156,9 +156,9 @@ void IncoherentBeamformer::beamform( dim3 grid(nsamples / _config.ib_tscrunch(), _config.nchans() / _config.ib_fscrunch()); char2 const* ftpa_voltages_ptr = thrust::raw_pointer_cast(input.data()); - float4 const* output_scale_ptr = + float const* output_scale_ptr = thrust::raw_pointer_cast(output_scale.data()); - float4 const* output_offset_ptr = + float const* output_offset_ptr = thrust::raw_pointer_cast(output_offset.data()); float const* antenna_weights_ptr = thrust::raw_pointer_cast(antenna_weights.data()); diff --git a/cpp/skyweaver/src/StatisticsCalculator.cu b/cpp/skyweaver/src/StatisticsCalculator.cu index a2cf68c..4de451a 100644 --- a/cpp/skyweaver/src/StatisticsCalculator.cu +++ b/cpp/skyweaver/src/StatisticsCalculator.cu @@ -130,7 +130,7 @@ void StatisticsCalculator::calculate_statistics( } void StatisticsCalculator::update_scalings( - BeamsetWeightsVectorTypeH const& beamset_weights, + ScalingVectorTypeH const& beamset_weights, int nbeamsets) { // At this stage we have the standard deviations of each channel @@ -139,107 +139,99 @@ void StatisticsCalculator::update_scalings( // disk. const float weights_amp = 127.0f; - if ((_config.ib_fscrunch() != _config.cb_fscrunch()) || - (_config.ib_tscrunch() != _config.cb_tscrunch())) { - throw std::invalid_argument("IB and CB must share same F and T scrunch"); - } - const std::size_t output_nchans = _config.nchans() / _config.cb_fscrunch(); - const std::size_t fscrunch = _config.cb_fscrunch(); - const std::size_t tscrunch = _config.cb_tscrunch(); + std::size_t reduced_nchans_ib = _config.nchans() / _config.ib_fscrunch(); + std::size_t reduced_nchans_cb = _config.nchans() / _config.cb_fscrunch(); // Offsets for the coherent beams - _cb_offsets_d.resize(output_nchans * nbeamsets); - _cb_offsets_h.resize(output_nchans * nbeamsets); + _cb_offsets_d.resize(reduced_nchans_cb * nbeamsets); + _cb_offsets_h.resize(reduced_nchans_cb * nbeamsets); // Scalings for the coherent beams - _cb_scaling_d.resize(output_nchans * nbeamsets); - _cb_scaling_h.resize(output_nchans * nbeamsets); + _cb_scaling_d.resize(reduced_nchans_cb * nbeamsets); + _cb_scaling_h.resize(reduced_nchans_cb * nbeamsets); // Offsets for the incoherent beam - _ib_offsets_d.resize(output_nchans * nbeamsets); - _ib_offsets_h.resize(output_nchans * nbeamsets); + _ib_offsets_d.resize(reduced_nchans_ib * nbeamsets); + _ib_offsets_h.resize(reduced_nchans_ib * nbeamsets); // Scalings for the incoherent beam - _ib_scaling_d.resize(output_nchans * nbeamsets); - _ib_scaling_h.resize(output_nchans * nbeamsets); + _ib_scaling_d.resize(reduced_nchans_ib * nbeamsets); + _ib_scaling_h.resize(reduced_nchans_ib * nbeamsets); const std::uint32_t pa = _config.npol() * _config.nantennas(); const std::uint32_t a = _config.nantennas(); for(std::uint32_t beamset_idx = 0; beamset_idx < nbeamsets; ++beamset_idx) { - - // Here we compute the offsets and scaling factors for I, Q, U and V for each beamset - // Statistics are in FPA order - - for(int fo_idx = 0; fo_idx < output_nchans; ++fo_idx) { - - const int output_idx = beamset_idx * nbeamsets + fo_idx; - - // reset main accumulators - float meanI = 0.0f; - float meanQ = 0.0f; - float varIQc = 0.0f; - float varUVc = 0.0f; - float varIQi = 0.0f; - float varUVi = 0.0f; - - for(int fs_idx = 0; fs_idx < fscrunch; ++fs_idx) { - const int f_idx = fo_idx * fscrunch + fs_idx; - - // Per antenna/pol accumulators - float var_p0_sum = 0.0f; // sum(sigma_p0^2) - float var_p1_sum = 0.0f; // sum(sigma_p1^2) - float quad_sum = 0.0f; // sum(sigma_p0^4 + sigma_p1^4) - float sum_mul = 0.0f; // sum(sigma_p0^2 * sigma_p1^2) - for (int a_idx = 0; a_idx < _cofig.npol(); ++a_idx) { - // stats are in FPA order - // weights are in FPBA order - const int input_idx = f_idx * pa + a_idx; - Statistics p0 = _stats_h[input_idx]; - Statistics p1 = _stats_h[input_idx + nantennas]; - const float weight = beamset_weights[beamset_idx * a + a_idx]; - const float std_p0 = p0.std * weight; - const float std_p1 = p1.std * weight; - const float var_p0 = std_p0 * std_p0; - const float var_p1 = std_p1 * std_p1; - meanI += var_p0 + var_p1; - meanQ += var_p0 - var_p1; - var_p0_sum += var_p0; - var_p1_sum += var_p1; - quad_sum += (var_p0 * var_p0) + (var_p1 * var_p1); - sum_mul += var_p0 * var_p1; + // For each frequency channel we average the std estimates then + // calculate the offsets and scalings. + for(int f_idx = 0; f_idx < _config.nchans(); ++f_idx) { + float sum = 0.0f; + float count = 0.0f; + for(int p_idx = 0; p_idx < _config.npol(); ++p_idx) { + for(int a_idx = 0; a_idx < _config.nantennas(); ++a_idx) { + const float weight = + beamset_weights[_config.nantennas() * beamset_idx + + a_idx]; + sum += + weight * _stats_h[f_idx * pa + p_idx * a + a_idx].std; + count += weight; } - varIQc += (var_p0_sum * var_p0_sum) + (var_p1_sum * var_p1_sum); - varIQi += quad_sum; - varUVc += var_p0_sum * var_p1_sum; - varUVi += sum_mul; + } + const float avg_std = sum / count; + BOOST_LOG_TRIVIAL(debug) << "Channel " << f_idx; + BOOST_LOG_TRIVIAL(debug) + << "Averaged standard deviation = " << avg_std; + float const effective_nantennas = count / _config.npol(); + + // CB OFFSET + { + float scale = std::pow(weights_amp * avg_std * + std::sqrt(effective_nantennas), + 2); + float dof = 2 * _config.cb_tscrunch() * _config.npol(); + _cb_offsets_h[f_idx] = scale * dof; + BOOST_LOG_TRIVIAL(debug) + << "CB offset = " << _cb_offsets_h[f_idx]; + } + + // CB SCALE + { + float scale = std::pow(weights_amp * avg_std * + std::sqrt(effective_nantennas), + 2); + float dof = 2 * _config.cb_tscrunch() * _config.npol(); + _cb_scaling_h[f_idx] = + scale * std::sqrt(2 * dof) / _config.output_level(); + BOOST_LOG_TRIVIAL(debug) + << "CB scaling = " << _cb_scaling_h[f_idx]; } - // Coherent offsets - _cb_offsets_h[output_idx].x = 2 * tscrunch * meanI; // I - _cb_offsets_h[output_idx].y = 2 * tscrunch * meanQ; // Q - _cb_offsets_h[output_idx].z = 0.0f; // U - _cb_offsets_h[output_idx].w = 0.0f; // V - - // Coherent scales - _cb_scaling_h[output_idx].x = 4 * tscrunch * varIQc / _config.output_level(); // I - _cb_scaling_h[output_idx].y = 4 * tscrunch * varIQc / _config.output_level(); // Q - _cb_scaling_h[output_idx].z = 8 * tscrunch * varUVc / _config.output_level(); // U - _cb_scaling_h[output_idx].w = 8 * tscrunch * varUVc / _config.output_level(); // V - - // Incoherent offsets - _ib_offsets_h[output_idx].x = 2 * tscrunch * meanI; // I - _ib_offsets_h[output_idx].y = 2 * tscrunch * meanQ; // Q - _ib_offsets_h[output_idx].z = 0.0f; // U - _ib_offsets_h[output_idx].w = 0.0f; // V - - // Incoherent scales - _ib_scaling_h[output_idx].x = 4 * tscrunch * varIQi / _config.output_level(); // I - _ib_scaling_h[output_idx].y = 4 * tscrunch * varIQi / _config.output_level(); // Q - _ib_scaling_h[output_idx].z = 8 * tscrunch * varUVi / _config.output_level(); // U - _ib_scaling_h[output_idx].w = 8 * tscrunch * varUVi / _config.output_level(); // V + // IB OFFSET + { + float scale = std::pow(avg_std, 2); + float dof = 2 * _config.ib_tscrunch() * effective_nantennas * + _config.npol(); + _ib_offsets_h[f_idx] = scale * dof; + BOOST_LOG_TRIVIAL(debug) + << "IB offset = " << _ib_offsets_h[f_idx]; + } + + // IB SCALE + { + float scale = std::pow(avg_std, 2); + float dof = 2 * _config.ib_tscrunch() * effective_nantennas * + _config.npol(); + _ib_scaling_h[f_idx] = + scale * std::sqrt(2 * dof) / _config.output_level(); + BOOST_LOG_TRIVIAL(debug) + << "IB scaling = " << _ib_scaling_h[f_idx]; + } } } + // At this stage, all scaling vectors are available on the host and + // could be written to disk. + + // Copying these back to the device _cb_offsets_d = _cb_offsets_h; _cb_scaling_d = _cb_scaling_h; _ib_offsets_d = _ib_offsets_h; @@ -264,7 +256,7 @@ void StatisticsCalculator::dump_scalings( std::string const& timestamp, std::string const& tag, std::string const& path, - ScalingVectorTypeH const& ar) const + thrust::host_vector const& ar) const { std::ofstream writer; std::string filename = path + "/" + timestamp + "_" + tag + "_" + @@ -279,7 +271,7 @@ void StatisticsCalculator::dump_scalings( BOOST_LOG_TRIVIAL(error) << error_message.str(); return; } - writer.write(static_cast(ar.data()), ar.size() * sizeof(ScalingType)); + writer.write((char*)ar.data(), ar.size() * sizeof(float)); writer.close(); } @@ -313,4 +305,4 @@ StatisticsCalculator::ib_scaling() const return _ib_scaling_d; } -} // namespace skyweaver \ No newline at end of file +} // namespace skyweaver From 13047bb4b22a998b2227562d3c4e38e3688b8be1 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 2 Jan 2025 13:18:54 +0100 Subject: [PATCH 44/65] segregate filterbanks in beam directories --- cpp/skyweaver/MultiFileWriter.cuh | 6 ++- cpp/skyweaver/detail/MultiFileWriter.cu | 5 +++ cpp/skyweaver/detail/SkyCleaver.cpp | 1 + cpp/skyweaver/skycleaver_utils.hpp | 53 +++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 cpp/skyweaver/skycleaver_utils.hpp diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index d107713..3d5fb45 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -19,6 +19,7 @@ struct MultiFileWriterConfig { std::string stokes_mode; std::string output_dir; std::string base_output_dir; + std::string inner_dir; std::string prefix; std::string extension; std::string output_basename; @@ -26,7 +27,7 @@ struct MultiFileWriterConfig { MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), - output_dir("default/"), prefix(""), extension(""), output_basename(""), suffix("") {}; + output_dir("default/"), base_output_dir("default_base/"), inner_dir(""), prefix(""), extension(""), output_basename(""), suffix("") {}; MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, @@ -47,7 +48,8 @@ struct MultiFileWriterConfig { ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + ", prefix: " + prefix + ", extension: " + extension + ", output_basename: " + output_basename + - ", base_output_dir: " + base_output_dir; + ", base_output_dir: " + base_output_dir + + ", inner_dir: " + inner_dir + ", suffix: " + suffix; } }; /** diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 439f1ab..5c2dbd8 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -47,6 +47,7 @@ MultiFileWriter::MultiFileWriter( _config.max_file_size = config.max_output_filesize(); _config.stokes_mode = config.stokes_mode(); _config.base_output_dir = config.output_dir(); + _config.inner_dir = ""; } @@ -106,6 +107,10 @@ MultiFileWriter::get_output_dir(VectorType const& stream_data, output_dir << _config.base_output_dir << "/" << get_formatted_time(_header.utc_start) << "/" << stream_idx; + if(!_config.inner_dir.empty()) { + output_dir << "/" << _config.inner_dir; + } + return output_dir.str(); } diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index b84cff7..8505c94 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -474,6 +474,7 @@ void skyweaver::SkyCleaver::init_writers() writer_config.max_file_size = _config.max_output_filesize(); writer_config.stokes_mode = _config.out_stokes().at(istokes); writer_config.base_output_dir = output_dir; + writer_config.inner_dir = beam_info.beam_name; writer_config.prefix = _config.out_prefix(); std::string suffix = "idm_" + to_string_with_padding(_header.dms[idm], 9, 3); diff --git a/cpp/skyweaver/skycleaver_utils.hpp b/cpp/skyweaver/skycleaver_utils.hpp new file mode 100644 index 0000000..d0ad9e9 --- /dev/null +++ b/cpp/skyweaver/skycleaver_utils.hpp @@ -0,0 +1,53 @@ +#ifndef SKYWEAVER_SKYCLEAVER_UTILS_HPP +#define SKYWEAVER_SKYCLEAVER_UTILS_HPP +namespace skyweaver +{ + +template +std::vector +get_list_from_string(const std::string& value, + T epsilon = std::numeric_limits::epsilon()) +{ + std::vector output; + std::vector comma_chunks; + + // Split the input string by commas + std::stringstream ss(value); + std::string token; + while(std::getline(ss, token, ',')) { comma_chunks.push_back(token); } + + for(const auto& comma_chunk: comma_chunks) { + // Check if the chunk contains a colon (indicating a range) + if(comma_chunk.find(':') == std::string::npos) { + output.push_back(static_cast(std::atof(comma_chunk.c_str()))); + continue; + } + + // Split the range chunk by colons + std::stringstream ss_chunk(comma_chunk); + std::vector colon_chunks; + std::string colon_token; + while(std::getline(ss_chunk, colon_token, ':')) { + colon_chunks.push_back( + static_cast(std::atof(colon_token.c_str()))); + } + + // Determine the step size + T step = colon_chunks.size() == 3 ? colon_chunks[2] : static_cast(1); + T start = colon_chunks[0]; + T stop = colon_chunks[1]; + + // Loop and add values to the output vector + if constexpr(std::is_floating_point::value) { + for(T k = start; k <= stop + epsilon; k += step) { + output.push_back(k); + } + } else { + for(T k = start; k <= stop; k += step) { output.push_back(k); } + } + } + return output; +} + +} +#endif // SKYWEAVER_SKYCLEAVER_UTILS_HPP \ No newline at end of file From 7ef868ca4a97e7842dfc1f277dbbb0b348266e1b Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Mon, 27 Jan 2025 10:26:21 +0000 Subject: [PATCH 45/65] Fix to incoherent de-dispersion pipeline header --- cpp/skyweaver/IncoherentDedispersionPipeline.cuh | 4 +++- cpp/skyweaver/PipelineConfig.hpp | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/IncoherentDedispersionPipeline.cuh b/cpp/skyweaver/IncoherentDedispersionPipeline.cuh index 9dc3934..8326ee1 100644 --- a/cpp/skyweaver/IncoherentDedispersionPipeline.cuh +++ b/cpp/skyweaver/IncoherentDedispersionPipeline.cuh @@ -50,10 +50,12 @@ class IncoherentDedispersionPipeline DedisperserVector _dedispersers; std::vector _output_buffers; Timer _timer; + OutputVectorType _beamsplit_buffer; + std::size_t _n_tdb_files; }; } // namespace skyweaver #include "skyweaver/detail/IncoherentDedispersionPipeline.cu" -#endif // SKYWEAVER_INCOHERENTDEDISPERSIONPIPELINE_CUH \ No newline at end of file +#endif // SKYWEAVER_INCOHERENTDEDISPERSIONPIPELINE_CUH diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index c751b9c..517491c 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -271,6 +271,7 @@ class PipelineConfig /** * @brief Return the total number of antennas that will be beamformed */ + std::size_t nantennas() const { return SKYWEAVER_NANTENNAS; } /** From 29021ad6d1dcf0389708632c5068b7c527626ca3 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Mon, 27 Jan 2025 16:10:23 +0000 Subject: [PATCH 46/65] Add beam index to output dada headers --- cpp/skyweaver/DescribedVector.hpp | 26 ++++++++++++++++--- cpp/skyweaver/ObservationHeader.hpp | 3 ++- .../detail/IncoherentDedispersionPipeline.cu | 1 + .../detail/file_writer_callbacks.cpp | 4 ++- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/cpp/skyweaver/DescribedVector.hpp b/cpp/skyweaver/DescribedVector.hpp index 9b11578..33f2348 100644 --- a/cpp/skyweaver/DescribedVector.hpp +++ b/cpp/skyweaver/DescribedVector.hpp @@ -106,7 +106,7 @@ struct DescribedVector { */ DescribedVector() : _dms_stale(true), _frequencies_stale(true), _dims{dims...}, - _tsamp(0.0) + _tsamp(0.0), _beam0_idx(0) { } @@ -118,7 +118,7 @@ struct DescribedVector { */ DescribedVector(std::initializer_list sizes) : _dms_stale(true), _frequencies_stale(true), _sizes(sizes), - _dims{dims...}, _tsamp(0.0) + _dims{dims...}, _tsamp(0.0), _beam0_idx(0) { if(_sizes.size() != sizeof...(dims)) { throw std::invalid_argument( @@ -135,7 +135,7 @@ struct DescribedVector { */ DescribedVector(std::initializer_list sizes, value_type default_value) : _dms_stale(true), _frequencies_stale(true), _sizes(sizes), - _dims{dims...}, _tsamp(0.0) + _dims{dims...}, _tsamp(0.0), _beam0_idx(0) { if(_sizes.size() != sizeof...(dims)) { throw std::invalid_argument( @@ -390,6 +390,23 @@ struct DescribedVector { _frequencies[0] = freq; } + /** + * @brief Set the index of the first beam in the file + * + * @param index + */ + void beam0_idx(std::size_t idx) + { + _beam0_idx = idx; + } + + /** + * @brief Return the index of the first beam in the file + * + * @return std::size_t + */ + std::size_t beam0_idx() const { return _beam0_idx; } + /** * @brief Return the number of frequency channels * @@ -594,6 +611,7 @@ struct DescribedVector { double _tsamp = 0.0; double _utc_offset = 0.0; float _reference_dm = 0.0; + std::size_t _beam0_idx; VectorType _vector; }; @@ -707,4 +725,4 @@ using TFPowersStdH = DescribedVector, } // namespace skyweaver -#endif // SKYWEAVER_DESCRIBEDVECTORS_HPP \ No newline at end of file +#endif // SKYWEAVER_DESCRIBEDVECTORS_HPP diff --git a/cpp/skyweaver/ObservationHeader.hpp b/cpp/skyweaver/ObservationHeader.hpp index 6e2fbb9..ff5d456 100644 --- a/cpp/skyweaver/ObservationHeader.hpp +++ b/cpp/skyweaver/ObservationHeader.hpp @@ -17,6 +17,7 @@ struct ObservationHeader { std::size_t nantennas = 0; // Number of antennas std::size_t sample_clock_start = 0; // The start epoch in sampler ticks std::size_t chan0_idx = 0; // The index of the first channel + std::size_t beam0_idx = 0; // The index of the first beam std::size_t obs_nchans = 0; // The total number of channels in the observation long double bandwidth = 0.0; // Bandwidth in Hz of the subband @@ -92,4 +93,4 @@ bool are_headers_similar(ObservationHeader const& header1, } // namespace skyweaver -#endif // SKYWEAVER_OBSERVATIONHEADER_HPP \ No newline at end of file +#endif // SKYWEAVER_OBSERVATIONHEADER_HPP diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index f033d19..6f339f3 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -101,6 +101,7 @@ void IncoherentDedispersionPipeline:: } } + _beamsplit_buffer.beam0_idx(tdb_file_idx * _config.nbeams_per_file()); _handler(_beamsplit_buffer, ref_dm_idx * _n_tdb_files + tdb_file_idx); } } diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index e1c59dc..704e020 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -58,6 +58,7 @@ NCHAN 64 NBEAM 800 ORDER TFB CHAN0_IDX 2688 +BEAM0_IDX 0 )"; #define MJD_UNIX_EPOCH 40587.0 @@ -160,6 +161,7 @@ create_dada_file_stream(MultiFileWriterConfig const& config, header_writer.set("ORDER", stream_data.dims_as_string()); header_writer.set("CHAN0_IDX", header.chan0_idx); + header_writer.set("BEAM0_IDX", stream_data.beam0_idx()); header_writer.set("FILE_SIZE", filesize); header_writer.set("FILE_NUMBER", file_idx); header_writer.set("OBS_OFFSET", bytes_written); @@ -292,4 +294,4 @@ create_sigproc_file_stream(MultiFileWriterConfig const& config, } } // namespace detail -} // namespace skyweaver \ No newline at end of file +} // namespace skyweaver From 4937a65fb05430ee0c090bc1b0e09c951703ba23 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Tue, 28 Jan 2025 09:53:06 +0000 Subject: [PATCH 47/65] Fixing incorrect indices when splitting tdb stream into multiple files --- cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index 6f339f3..64f07c0 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -97,7 +97,7 @@ void IncoherentDedispersionPipeline:: { for (std::size_t bidx = 0; bidx < nbeams_per_file ; ++bidx) { - _beamsplit_buffer[tdidx + bidx] = _output_buffers[ref_dm_idx][tdidx + bidx + tdb_file_idx * nbeams_per_file]; + _beamsplit_buffer[tdidx * nbeams_per_file + bidx] = _output_buffers[ref_dm_idx][tdidx * nbeams + tdb_file_idx * nbeams_per_file + bidx]; } } From 6572ec188be6d22de204c691686909102d570dd9 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Tue, 28 Jan 2025 10:41:22 +0000 Subject: [PATCH 48/65] Slightly clearer indexing when splitting tdb files by beam --- .../detail/IncoherentDedispersionPipeline.cu | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index 64f07c0..e71fc0b 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -91,13 +91,22 @@ void IncoherentDedispersionPipeline:: _beamsplit_buffer.metalike(_output_buffers[ref_dm_idx]); _beamsplit_buffer.resize({nsamples, ndms, nbeams_per_file}); + std::size_t td_input_offset; + std::size_t td_output_offset; + std::size_t b_offset; + for (std::size_t tdb_file_idx = 0; tdb_file_idx < _n_tdb_files; ++tdb_file_idx) { + b_offset = tdb_file_idx * nbeams_per_file; + for (std::size_t tdidx = 0; tdidx < nsamples * ndms; ++tdidx) { + td_input_offset = tdidx * nbeams; + td_output_offset = tdidx * nbeams_per_file; + for (std::size_t bidx = 0; bidx < nbeams_per_file ; ++bidx) { - _beamsplit_buffer[tdidx * nbeams_per_file + bidx] = _output_buffers[ref_dm_idx][tdidx * nbeams + tdb_file_idx * nbeams_per_file + bidx]; + _beamsplit_buffer[td_output_offset + bidx] = _output_buffers[ref_dm_idx][td_input_offset + b_offset + bidx]; } } From 4f15b64e74bc7867a658789a2e1682b66dd1febd Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Wed, 29 Jan 2025 15:29:49 +0000 Subject: [PATCH 49/65] Fix for incorrect S-band frequencies being reported in filenames due to int overflow --- cpp/skyweaver/detail/file_writer_callbacks.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index e1c59dc..ffb2c42 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -87,12 +87,12 @@ create_dada_file_stream(MultiFileWriterConfig const& config, std::stringstream output_dir; output_dir << config.output_dir << "/" << std::fixed << std::setfill('0') - << std::setw(9) << static_cast(header.frequency); + << std::setw(9) << static_cast(header.frequency); std::stringstream output_basename; output_basename << config.output_basename << "_" << std::fixed << std::setfill('0') << std::setw(9) - << static_cast(header.frequency); + << static_cast(header.frequency); std::unique_ptr file_stream = std::make_unique( From cac388314aeb69ab199a41c3960641c33204edbb Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Wed, 5 Feb 2025 12:30:54 +0000 Subject: [PATCH 50/65] Use std::copy when splitting output buffers by beam --- cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index e71fc0b..a3f86c5 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -104,11 +104,9 @@ void IncoherentDedispersionPipeline:: td_input_offset = tdidx * nbeams; td_output_offset = tdidx * nbeams_per_file; - for (std::size_t bidx = 0; bidx < nbeams_per_file ; ++bidx) - { - _beamsplit_buffer[td_output_offset + bidx] = _output_buffers[ref_dm_idx][td_input_offset + b_offset + bidx]; - } - + std::copy(&_output_buffers[ref_dm_idx][td_input_offset + b_offset], + &_output_buffers[ref_dm_idx][td_input_offset + b_offset + nbeams_per_file], + &_beamsplit_buffer[td_output_offset]); } _beamsplit_buffer.beam0_idx(tdb_file_idx * _config.nbeams_per_file()); _handler(_beamsplit_buffer, ref_dm_idx * _n_tdb_files + tdb_file_idx); From f2759e65f0036f608775508b8fc3aa8e1ac37d0a Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Wed, 5 Feb 2025 12:41:08 +0000 Subject: [PATCH 51/65] Fix bug with beam0_idx being uninitialised for IB --- cpp/skyweaver/DescribedVector.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/skyweaver/DescribedVector.hpp b/cpp/skyweaver/DescribedVector.hpp index 33f2348..024fd43 100644 --- a/cpp/skyweaver/DescribedVector.hpp +++ b/cpp/skyweaver/DescribedVector.hpp @@ -171,7 +171,8 @@ struct DescribedVector { _frequencies(other._frequencies), _dms_stale(other._dms_stale), _tsamp(other._tsamp), _utc_offset(other._utc_offset), _reference_dm(other._reference_dm), - _frequencies_stale(other._frequencies_stale) + _frequencies_stale(other._frequencies_stale), + _beam0_idx(other._beam0_idx) { } @@ -206,6 +207,7 @@ struct DescribedVector { _tsamp = other._tsamp; _utc_offset = other._utc_offset; _reference_dm = other._reference_dm; + _beam0_idx = other._beam0_idx; } /** @@ -611,7 +613,7 @@ struct DescribedVector { double _tsamp = 0.0; double _utc_offset = 0.0; float _reference_dm = 0.0; - std::size_t _beam0_idx; + std::size_t _beam0_idx = 0; VectorType _vector; }; From d57409c0a3f3652db632efb2eeace4c9cb2058ea Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Wed, 5 Feb 2025 13:19:07 +0000 Subject: [PATCH 52/65] TDB output splitting now works if beams do not divide evenly between files --- .../detail/IncoherentDedispersionPipeline.cu | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index a3f86c5..38fca20 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -47,7 +47,9 @@ IncoherentDedispersionPipeline:: // TODO: give these sensible numbers // TODO: work out how to get the max delay into the handler } - _n_tdb_files = _config.nbeams() / _config.nbeams_per_file(); + + // ceil division + _n_tdb_files = (_config.nbeams() + _config.nbeams_per_file() - 1) / _config.nbeams_per_file(); } template @@ -86,18 +88,22 @@ void IncoherentDedispersionPipeline:: const std::size_t ndms = _output_buffers[ref_dm_idx].ndms(); const std::size_t nbeams = _output_buffers[ref_dm_idx].nbeams(); const std::size_t nsamples = _output_buffers[ref_dm_idx].nsamples(); - const std::size_t nbeams_per_file = _config.nbeams_per_file(); - - _beamsplit_buffer.metalike(_output_buffers[ref_dm_idx]); - _beamsplit_buffer.resize({nsamples, ndms, nbeams_per_file}); - std::size_t td_input_offset; - std::size_t td_output_offset; - std::size_t b_offset; + std::size_t nbeams_per_file = _config.nbeams_per_file(); + std::size_t td_input_offset = 0; + std::size_t td_output_offset = 0; + std::size_t b_offset = 0; + std::size_t tdb_file_idx = 0; - for (std::size_t tdb_file_idx = 0; tdb_file_idx < _n_tdb_files; ++tdb_file_idx) + while (b_offset < nbeams) { - b_offset = tdb_file_idx * nbeams_per_file; + if (b_offset + nbeams_per_file > nbeams) + { + nbeams_per_file = nbeams - b_offset; + } + + _beamsplit_buffer.metalike(_output_buffers[ref_dm_idx]); + _beamsplit_buffer.resize({nsamples, ndms, nbeams_per_file}); for (std::size_t tdidx = 0; tdidx < nsamples * ndms; ++tdidx) { @@ -108,8 +114,12 @@ void IncoherentDedispersionPipeline:: &_output_buffers[ref_dm_idx][td_input_offset + b_offset + nbeams_per_file], &_beamsplit_buffer[td_output_offset]); } - _beamsplit_buffer.beam0_idx(tdb_file_idx * _config.nbeams_per_file()); + + _beamsplit_buffer.beam0_idx(b_offset); _handler(_beamsplit_buffer, ref_dm_idx * _n_tdb_files + tdb_file_idx); + + b_offset += nbeams_per_file; + tdb_file_idx++; } } else From 61644a4fab614274b402dda461cdf06c11f0f7eb Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Wed, 5 Feb 2025 13:55:40 +0000 Subject: [PATCH 53/65] dashes instead of underscores in nbeams-per-file cmdline argument, for consistency with the others --- cpp/skyweaver/src/skyweaver_cli.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 4bce8f2..8061e4f 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -479,7 +479,7 @@ int main(int argc, char** argv) "The logging level to use (debug, info, warning, error)") - ("nbeams_per_file", + ("nbeams-per-file", po::value()->default_value(SKYWEAVER_NBEAMS)->notifier( [&config](std::size_t key) { config.nbeams_per_file(key); }), "The number of beams per output file. Default = all beams."); From 03b92c5e814e997d9402ecc8a3e77471b783298f Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 13 Feb 2025 11:15:24 +0000 Subject: [PATCH 54/65] Fix for integer overflow in SkyCleaver while computing expected bridge frequencies --- cpp/skyweaver/detail/SkyCleaver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index 8505c94..9f86c58 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -303,7 +303,7 @@ void skyweaver::SkyCleaver::init_readers() long double start_freq = obs_centre_freq - obs_bandwidth / 2; for(int i = 0; i < _config.nbridges(); i++) { - int ifreq = std::lround(std::floor( + std::size_t ifreq = std::lround(std::floor( start_freq + (i + 0.5) * obs_bandwidth / _config.nbridges())); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) From 4242cf90f49c72239e77a4bff146c75986987e24 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 20 Feb 2025 09:51:40 +0000 Subject: [PATCH 55/65] Added command-line option for computing/outputing voltage statistics --- cpp/skyweaver/PipelineConfig.hpp | 12 +++++++ cpp/skyweaver/detail/BeamformerPipeline.cu | 39 +++++++++++++--------- cpp/skyweaver/src/PipelineConfig.cpp | 12 ++++++- cpp/skyweaver/src/skyweaver_cli.cu | 11 ++++-- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 1656b0c..807d1bb 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -165,6 +165,17 @@ class PipelineConfig */ bool enable_incoherent_dedispersion() const; + + /** + * @brief Enable/disable calculation and writing of voltage statistics + */ + void output_statistics(bool enable); + + /** + * @brief Check if calculation of voltage statistics is enabled + */ + bool output_statistics() const; + /** * @brief Return the number of time samples to be integrated * in the coherent beamformer. @@ -339,6 +350,7 @@ class PipelineConfig std::size_t _max_output_filesize; std::string _output_file_prefix; bool _enable_incoherent_dedispersion; + bool _output_statistics; double _cfreq; double _bw; mutable bool _channel_frequencies_stale; diff --git a/cpp/skyweaver/detail/BeamformerPipeline.cu b/cpp/skyweaver/detail/BeamformerPipeline.cu index 7c44422..82f7b41 100644 --- a/cpp/skyweaver/detail/BeamformerPipeline.cu +++ b/cpp/skyweaver/detail/BeamformerPipeline.cu @@ -115,7 +115,10 @@ void BeamformerPipeline:: _utc_offset = utc_offset; _cb_handler.init(_header); _ib_handler.init(_header); - _stats_handler.init(_header); + if (_config.output_statistics()) + { + _stats_handler.init(_header); + } NVTX_RANGE_POP(); } @@ -161,20 +164,23 @@ void BeamformerPipeline:: _timer.stop("transpose TAFTP to FTPA"); NVTX_RANGE_POP(); - NVTX_RANGE_PUSH("Calculate statistics"); - BOOST_LOG_TRIVIAL(debug) << "Checking if channel statistics update request"; - _timer.start("calculate statistics"); - _stats_manager->calculate_statistics(_ftpa_post_transpose); - _timer.stop("calculate statistics"); - NVTX_RANGE_POP(); - NVTX_RANGE_PUSH("Update scalings"); - if(_call_count == 0) { + if ((_call_count == 0) || _config.output_statistics()) + { + NVTX_RANGE_PUSH("Calculate statistics"); + BOOST_LOG_TRIVIAL(debug) << "Checking if channel statistics update request"; + _timer.start("calculate statistics"); + _stats_manager->calculate_statistics(_ftpa_post_transpose); + _timer.stop("calculate statistics"); + NVTX_RANGE_POP(); + if(_call_count == 0) { + NVTX_RANGE_PUSH("Update scalings"); _timer.start("update scalings"); _stats_manager->update_scalings(_delay_manager->beamset_weights(), _delay_manager->nbeamsets()); _timer.stop("update scalings"); + NVTX_RANGE_POP(); + } } - NVTX_RANGE_POP(); // BOOST_LOG_TRIVIAL(debug) << "Peeking the statistics"; // peek(_stats_manager->statistics(), 64); @@ -251,11 +257,14 @@ void BeamformerPipeline:: NVTX_RANGE_POP(); } NVTX_RANGE_POP(); - NVTX_RANGE_PUSH("Stats handler"); - _timer.start("statistics handler"); - _stats_handler(_stats_manager->statistics()); - _timer.stop("statistics handler"); - NVTX_RANGE_POP(); + if (_config.output_statistics()) + { + NVTX_RANGE_PUSH("Stats handler"); + _timer.start("statistics handler"); + _stats_handler(_stats_manager->statistics()); + _timer.stop("statistics handler"); + NVTX_RANGE_POP(); + } NVTX_RANGE_POP(); } diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index c2cdf3b..ca99c8d 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -15,7 +15,7 @@ PipelineConfig::PipelineConfig() _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f) + _stokes_mode("I"), _output_level(24.0f), _output_statistics(true) { } @@ -169,6 +169,16 @@ bool PipelineConfig::enable_incoherent_dedispersion() const return _enable_incoherent_dedispersion; } +void PipelineConfig::output_statistics(bool enable) +{ + _output_statistics = enable; +} + +bool PipelineConfig::output_statistics() const +{ + return _output_statistics; +} + std::vector const& PipelineConfig::channel_frequencies() const { if(_channel_frequencies_stale) { diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 8204ca3..342dcaa 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -476,7 +476,14 @@ int main(int argc, char** argv) ("log-level", po::value()->default_value("info")->notifier( [](std::string level) { skyweaver::set_log_level(level); }), - "The logging level to use (debug, info, warning, error)"); + "The logging level to use (debug, info, warning, error)") + + ("statistics", + po::value()->default_value(true)->notifier( + [&config](bool const& enable) { + config.output_statistics(enable); }), + "Turn on/off calculation and output of voltage statistics"); + // set options allowed on command line po::options_description cmdline_options; @@ -617,4 +624,4 @@ int main(int argc, char** argv) return ERROR_UNHANDLED_EXCEPTION; } return SUCCESS; -} \ No newline at end of file +} From 6675ae6e080109c92105d88ae34649dfa12c458b Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 20 Feb 2025 10:11:47 +0000 Subject: [PATCH 56/65] Added command-line option to disable writing the incoherent beam --- cpp/skyweaver/PipelineConfig.hpp | 12 +++++++++++- cpp/skyweaver/detail/BeamformerPipeline.cu | 18 ++++++++++++------ cpp/skyweaver/src/PipelineConfig.cpp | 12 +++++++++++- cpp/skyweaver/src/skyweaver_cli.cu | 8 +++++++- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 807d1bb..65fa664 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -165,7 +165,6 @@ class PipelineConfig */ bool enable_incoherent_dedispersion() const; - /** * @brief Enable/disable calculation and writing of voltage statistics */ @@ -176,6 +175,16 @@ class PipelineConfig */ bool output_statistics() const; + /** + * @brief Enable/disable writing out the incoherent beam + */ + void output_incoherent_beam(bool enable); + + /** + * @brief Check if outputting incoherent beam is enabled + */ + bool output_incoherent_beam() const; + /** * @brief Return the number of time samples to be integrated * in the coherent beamformer. @@ -351,6 +360,7 @@ class PipelineConfig std::string _output_file_prefix; bool _enable_incoherent_dedispersion; bool _output_statistics; + bool _output_incoherent_beam; double _cfreq; double _bw; mutable bool _channel_frequencies_stale; diff --git a/cpp/skyweaver/detail/BeamformerPipeline.cu b/cpp/skyweaver/detail/BeamformerPipeline.cu index 82f7b41..18617b3 100644 --- a/cpp/skyweaver/detail/BeamformerPipeline.cu +++ b/cpp/skyweaver/detail/BeamformerPipeline.cu @@ -114,7 +114,10 @@ void BeamformerPipeline:: _header = header; _utc_offset = utc_offset; _cb_handler.init(_header); - _ib_handler.init(_header); + if (_config.output_incoherent_beam()) + { + _ib_handler.init(_header); + } if (_config.output_statistics()) { _stats_handler.init(_header); @@ -250,11 +253,14 @@ void BeamformerPipeline:: _timer.stop("coherent beam handler"); NVTX_RANGE_POP(); - NVTX_RANGE_PUSH("Incoherent beamformer handler"); - _timer.start("incoherent beam handler"); - _ib_handler(_tf_ib, dm_idx); - _timer.stop("incoherent beam handler"); - NVTX_RANGE_POP(); + if (_config.output_incoherent_beam()) + { + NVTX_RANGE_PUSH("Incoherent beamformer handler"); + _timer.start("incoherent beam handler"); + _ib_handler(_tf_ib, dm_idx); + _timer.stop("incoherent beam handler"); + NVTX_RANGE_POP(); + } } NVTX_RANGE_POP(); if (_config.output_statistics()) diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index ca99c8d..15605c2 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -15,7 +15,7 @@ PipelineConfig::PipelineConfig() _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f), _output_statistics(true) + _stokes_mode("I"), _output_level(24.0f), _output_statistics(true), _output_incoherent_beam(true) { } @@ -179,6 +179,16 @@ bool PipelineConfig::output_statistics() const return _output_statistics; } +void PipelineConfig::output_incoherent_beam(bool enable) +{ + _output_incoherent_beam = enable; +} + +bool PipelineConfig::output_incoherent_beam() const +{ + return _output_incoherent_beam; +} + std::vector const& PipelineConfig::channel_frequencies() const { if(_channel_frequencies_stale) { diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 342dcaa..7c656b9 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -482,8 +482,14 @@ int main(int argc, char** argv) po::value()->default_value(true)->notifier( [&config](bool const& enable) { config.output_statistics(enable); }), - "Turn on/off calculation and output of voltage statistics"); + "Turn on/off calculation and output of voltage statistics") + ("write-incoherent-beam", + po::value()->default_value(true)->notifier( + [&config](bool const& enable) { + config.output_incoherent_beam(enable); }), + "Turn on/off output of incoherent beam" + "Turning off does not disable incoherent beam subtraction"); // set options allowed on command line po::options_description cmdline_options; From 3b6530b1ef6b17db207929e53d12acb2f1e7a6c7 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 27 Feb 2025 20:08:34 +0000 Subject: [PATCH 57/65] Fix issue with SkyCleaver where tdb files were not being read in time order --- cpp/skyweaver/detail/SkyCleaver.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index 9f86c58..6c0c503 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -102,6 +102,8 @@ std::vector get_files(std::string directory_path, directory_path); } + std::sort(files.begin(), files.end()); + return files; } From 77d5c6ec834ebc3e11d1ad889c242adbb0b96a73 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Wed, 5 Mar 2025 10:12:19 +0000 Subject: [PATCH 58/65] Switch order of TDB output streams (loop first over coherent DMs, then beam subsets) --- cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index 38fca20..7d3403d 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -116,7 +116,7 @@ void IncoherentDedispersionPipeline:: } _beamsplit_buffer.beam0_idx(b_offset); - _handler(_beamsplit_buffer, ref_dm_idx * _n_tdb_files + tdb_file_idx); + _handler(_beamsplit_buffer, tdb_file_idx * _config.coherent_dms().size() + ref_dm_idx); b_offset += nbeams_per_file; tdb_file_idx++; From da9cea3c1fab56045732c864c12c9f154978cf36 Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 10 Apr 2025 22:13:18 +0200 Subject: [PATCH 59/65] Prototype fix for threading of CB handler (UNTESTED) --- cpp/skyweaver/BeamformerPipeline.cuh | 32 ++++++++ cpp/skyweaver/DescribedVector.hpp | 62 +++++++++++----- cpp/skyweaver/detail/BeamformerPipeline.cu | 86 +++++++++++----------- 3 files changed, 120 insertions(+), 60 deletions(-) diff --git a/cpp/skyweaver/BeamformerPipeline.cuh b/cpp/skyweaver/BeamformerPipeline.cuh index ead7433..7b4cf25 100644 --- a/cpp/skyweaver/BeamformerPipeline.cuh +++ b/cpp/skyweaver/BeamformerPipeline.cuh @@ -17,10 +17,41 @@ #include "skyweaver/WeightsManager.cuh" #include +#include namespace skyweaver { +class ThreadWrapper +{ + public: + // Construct and start the thread + template + explicit ThreadWrapper(Callable&& func) + : _thread(std::forward(func)) + { + } + + // Non-copyable + ThreadWrapper(const ThreadWrapper&) = delete; + ThreadWrapper& operator=(const ThreadWrapper&) = delete; + + // Movable + ThreadWrapper(ThreadWrapper&&) = default; + ThreadWrapper& operator=(ThreadWrapper&&) = default; + + // Join on destruction + ~ThreadWrapper() + { + if(_thread.joinable()) { + _thread.join(); + } + } + + private: + std::thread _thread; +}; + template _cb_handler_wrapper; IBHandler& _ib_handler; StatsHandler& _stats_handler; diff --git a/cpp/skyweaver/DescribedVector.hpp b/cpp/skyweaver/DescribedVector.hpp index 9b11578..47ce672 100644 --- a/cpp/skyweaver/DescribedVector.hpp +++ b/cpp/skyweaver/DescribedVector.hpp @@ -75,8 +75,7 @@ struct is_dv_copyable: std::false_type { }; template