From 28b50f399d29fda21db94b39be94306c61f87dab Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:06:27 +0000 Subject: [PATCH 01/69] define the pre_write struct, containing possible configuration important for pre write actions --- cpp/skyweaver/PipelineConfig.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 1656b0c..3902ea1 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -9,6 +9,19 @@ namespace skyweaver { + struct WaitConfig + { + bool is_enabled; + int iterations; + int sleep_time; + std::size_t min_free_space; + }; + + struct PreWriteConfig + { + WaitConfig wait; + }; + /** * @brief Class for wrapping the skyweaver pipeline configuration. From 3b5d2d64d41631ec284630e31d13e92ae77a3fba Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:07:11 +0000 Subject: [PATCH 02/69] create pre write config infrastructure --- cpp/skyweaver/PipelineConfig.hpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 3902ea1..d582ff0 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -166,6 +166,11 @@ class PipelineConfig */ DedispersionPlan const& ddplan() const; + /** + * @brief configures wait for filesystem space + */ + void configure_wait(std::string argument); + /** * @brief Enable/disable incoherent dedispersion based fscrunch after * beamforming @@ -222,6 +227,11 @@ class PipelineConfig return SKYWEAVER_CB_NSAMPLES_PER_BLOCK; } + PreWriteConfig pre_write_config() const + { + return _pre_write_config; + } + /** * @brief Return the total number of samples to read from file in each gulp. * @@ -363,6 +373,7 @@ class PipelineConfig float _output_level; DedispersionPlan _ddplan; mutable std::vector _channel_frequencies; + PreWriteConfig _pre_write_config; }; } // namespace skyweaver From 5f694348188a3620b5e6b04f8c639d6ca1c34223 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:07:42 +0000 Subject: [PATCH 03/69] needed to compute bytes from units like G, M or k --- cpp/skyweaver/PipelineConfig.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index d582ff0..ef1920b 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -346,6 +346,7 @@ class PipelineConfig } private: + std::size_t convertMemorySize(const std::string& str) const; void calculate_channel_frequencies() const; void update_power_offsets_and_scalings(); From 76785467ea488973d2b83379b0c4f748c1897cc8 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:08:27 +0000 Subject: [PATCH 04/69] add the pre write config to the MultiWriterConfig struct --- cpp/skyweaver/MultiFileWriter.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index ceeb2d8..77b41e1 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -24,7 +24,7 @@ struct MultiFileWriterConfig{ std::string output_dir; std::string prefix; std::string extension; - + PreWriteConfig pre_write; MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("test"), prefix("test"), extension("default"){}; MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, std::string extension) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension){}; From bd50c5a3575c4f9febdbb3db8b25cf928bb55c8f Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:08:58 +0000 Subject: [PATCH 05/69] define a type for a pre write callback function --- cpp/skyweaver/MultiFileWriter.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 77b41e1..cf8e863 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -51,13 +51,13 @@ template class MultiFileWriter { public: - using CreateStreamCallBackType = std::function(MultiFileWriterConfig const&, ObservationHeader const&, VectorType const&, std::size_t)>; public: + using PreWriteCallback = std::function; /** * @brief Construct a new Multi File Writer object * From 168de0a58495a22845fb715d2f8707d3fe515ef7 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:09:33 +0000 Subject: [PATCH 06/69] overload the constructors for the MultiFileWriter class --- cpp/skyweaver/MultiFileWriter.cuh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index cf8e863..21aa212 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -67,7 +67,9 @@ public: */ // MultiFileWriter(PipelineConfig const& config, std::string tag = ""); MultiFileWriter(PipelineConfig const& config, std::string tag, CreateStreamCallBackType create_stream_callback); + MultiFileWriter(PipelineConfig const& config, std::string tag, CreateStreamCallBackType create_stream_callback, PreWriteCallback pre_write_callback); MultiFileWriter(MultiFileWriterConfig config, std::string tag, CreateStreamCallBackType create_stream_callback); + MultiFileWriter(MultiFileWriterConfig config, std::string tag, CreateStreamCallBackType create_stream_callback, PreWriteCallback pre_write_callback); MultiFileWriter(MultiFileWriter const&) = delete; /** From 065b8f539283beef10524b445e8bae86a1fb9bbd Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:09:51 +0000 Subject: [PATCH 07/69] define a private call back function --- cpp/skyweaver/MultiFileWriter.cuh | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 21aa212..643647c 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -118,6 +118,7 @@ public: std::string get_extension(VectorType const& stream_data); CreateStreamCallBackType _create_stream_callback; MultiFileWriterConfig _config; + PreWriteCallback _pre_write_callback; std::string _tag; ObservationHeader _header; std::map> _file_streams; From 00e86a3306a06ef7a305f9c7e30c172e758205d4 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:11:19 +0000 Subject: [PATCH 08/69] define the overladed constructors of the MultiFileWriter class --- cpp/skyweaver/detail/MultiFileWriter.cu | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 0fdead7..9667ec7 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -50,6 +50,24 @@ MultiFileWriter::MultiFileWriter(PipelineConfig const& config, writer_config.max_file_size = config.max_output_filesize(); writer_config.stokes_mode = config.stokes_mode(); _config = writer_config; + _pre_write_callback = nullptr; +} + + template +MultiFileWriter::MultiFileWriter(PipelineConfig const& config, + std::string tag, + CreateStreamCallBackType create_stream_callback, + PreWriteCallback pre_write_callback) + : _tag(tag), _create_stream_callback(create_stream_callback), _pre_write_callback(pre_write_callback) +{ + MultiFileWriterConfig writer_config; + writer_config.header_size = config.dada_header_size(); + writer_config.max_file_size = config.max_output_filesize(); + writer_config.stokes_mode = config.stokes_mode(); + writer_config.output_dir = config.output_dir(); + writer_config.pre_write = config.pre_write_config(); + _config = writer_config; + _config.pre_write = writer_config.pre_write; } template @@ -57,11 +75,22 @@ MultiFileWriter::MultiFileWriter(MultiFileWriterConfig config, std::string tag, CreateStreamCallBackType create_stream_callback) : _config(config), _tag(tag), _create_stream_callback(create_stream_callback) +{ + _pre_write_callback = nullptr; +} + +template +MultiFileWriter::MultiFileWriter(MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback, + PreWriteCallback pre_write_callback) + : _config(config), _tag(tag), _create_stream_callback(create_stream_callback), _pre_write_callback(pre_write_callback) { } + template MultiFileWriter::~MultiFileWriter(){}; From 1d2566880f98dc0a29bd8cef929d2dbae0acc73e Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:11:53 +0000 Subject: [PATCH 09/69] implement the logic for the pre write callback --- cpp/skyweaver/detail/MultiFileWriter.cu | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 9667ec7..b28d991 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -178,6 +178,11 @@ template bool MultiFileWriter::operator()(VectorType const& stream_data, std::size_t stream_idx) { + std::size_t const data_size = stream_data.size() * sizeof(typename VectorType::value_type); + if (_pre_write_callback != nullptr && _config.pre_write.wait.is_enabled) + { + _pre_write_callback(data_size, _config); + } if(!has_stream(stream_idx)) { create_stream(stream_data, stream_idx); } @@ -192,8 +197,7 @@ bool MultiFileWriter::operator()(VectorType const& stream_data, _file_streams.at(stream_idx) ->write(reinterpret_cast( thrust::raw_pointer_cast(stream_data.data())), - stream_data.size() * - sizeof(typename VectorType::value_type)); + data_size); } return false; } From 044a939285a14ae38d07280a9317302eceb9bf4f Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:12:58 +0000 Subject: [PATCH 10/69] method to read the wait arguments --- cpp/skyweaver/src/PipelineConfig.cpp | 58 ++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index c2cdf3b..b5121e9 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -157,6 +157,64 @@ DedispersionPlan const& PipelineConfig::ddplan() const return _ddplan; } +std::size_t PipelineConfig::convertMemorySize(const std::string& str) const { + std::size_t lastCharPos = str.find_last_not_of("0123456789"); + std::string numberPart = str.substr(0, lastCharPos); + std::string unitPart = str.substr(lastCharPos); + + std::size_t number = std::stoull(numberPart); + + if (unitPart.empty()) + return number; + else if (unitPart == "K" || unitPart == "k") + return number * 1024; + else if (unitPart == "M" || unitPart == "m") + return number * 1024 * 1024; + else if (unitPart == "G" || unitPart == "g") + return number * 1024 * 1024 * 1024; + else + throw std::runtime_error("Invalid memory unit!"); +} + +void PipelineConfig::configure_wait(std::string argument) +{ + std::vector tokens; + std::string token; + std::istringstream tokenStream(argument); + int indx = 0; + _pre_write_config.wait.is_enabled = true; + while (std::getline(tokenStream, token, ':')) { + if(indx == 0) + { + errno = 0; + _pre_write_config.wait.iterations = std::stoi(token); + if (errno == ERANGE) { + throw std::runtime_error("Wait iteration number out of range!"); + } + if (_pre_write_config.wait.iterations < 0) _pre_write_config.wait.iterations = 0; + } else if(indx == 1) { + errno = 0; + _pre_write_config.wait.sleep_time = std::stoi(token); + if (errno == ERANGE) { + throw std::runtime_error("Sleep time out of range!"); + } + if (_pre_write_config.wait.sleep_time < 1) _pre_write_config.wait.sleep_time = 1; + } else if(indx == 2) { + if (!token.empty() && std::all_of(token.begin(), token.end(), ::isdigit)) + { + _pre_write_config.wait.min_free_space = std::stoull(token); + } else { + try { + _pre_write_config.wait.min_free_space = convertMemorySize(token); + } catch (std::runtime_error& e) { + std::cout << "Memory conversion error: " << e.what() << std::endl; + throw; + } + } + } + indx++; + } +} void PipelineConfig::enable_incoherent_dedispersion(bool enable) From a8a966b8387435b047715b7ffd224a6c339882b6 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:14:09 +0000 Subject: [PATCH 11/69] the pre write call back function --- cpp/skyweaver/src/skyweaver_cli.cu | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index e168f63..d6d1763 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -239,6 +240,34 @@ void run_pipeline(Pipeline& pipeline, stopwatch.show_all_timings(); } +auto pre_write_callback = [] (std::size_t const size, skyweaver::MultiFileWriterConfig const& config) +{ + if (!config.pre_write.wait.is_enabled) return; + std::filesystem::space_info space = std::filesystem::space(config.output_dir); + size_t limit = std::min(config.pre_write.wait.min_free_space, size); + if(space.available >= limit) + return; + + BOOST_LOG_TRIVIAL(info) + << space.available + << " bytes available space is not enough. Need at least " + << limit + << " bytes in " << config.output_dir << "."; + BOOST_LOG_TRIVIAL(warning) << "Start pausing."; + int max_iterations = (config.pre_write.wait.iterations == 0) ? INT_MAX : config.pre_write.wait.iterations; + for (int i = 0; i < max_iterations; i++) + { + sleep(config.pre_write.wait.sleep_time); + space = std::filesystem::space(config.output_dir); + if (space.available >= limit) + { + BOOST_LOG_TRIVIAL(info) << "Space has been freed up. Will proceed."; + return; + } + } + throw std::runtime_error("Space for writing hasn't been freed up in time."); +}; + template void setup_pipeline(skyweaver::PipelineConfig& config) { From f7766a615077ce950c813491a8e6a7d9820e1ebd Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:14:40 +0000 Subject: [PATCH 12/69] use the pre write call back function --- cpp/skyweaver/src/skyweaver_cli.cu | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index d6d1763..24c2f9e 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -289,7 +289,8 @@ void setup_pipeline(skyweaver::PipelineConfig& config) typename IBWriterType::CreateStreamCallBackType create_stream_callback_ib = skyweaver::detail::create_dada_file_stream< skyweaver::BTFPowersH>; - IBWriterType ib_handler(config, "ib", create_stream_callback_ib); + + IBWriterType ib_handler(config, "ib", create_stream_callback_ib, pre_write_callback); using StatsWriterType = skyweaver::MultiFileWriter>; @@ -297,8 +298,8 @@ void setup_pipeline(skyweaver::PipelineConfig& config) create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; - StatsWriterType stats_handler(config, "stats", create_stream_callback_stats); + StatsWriterType stats_handler(config, "stats", create_stream_callback_stats, pre_write_callback); if constexpr(enable_incoherent_dedispersion) { using CBWriterType = skyweaver::MultiFileWriter>; @@ -306,7 +307,7 @@ void setup_pipeline(skyweaver::PipelineConfig& config) create_stream_callback_cb = skyweaver::detail::create_dada_file_stream>; skyweaver::MultiFileWriter> - cb_file_writer(config, "cb", create_stream_callback_cb); + cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); skyweaver::IncoherentDedispersionPipeline @@ -325,7 +326,7 @@ void setup_pipeline(skyweaver::PipelineConfig& config) typename CBWriterType::CreateStreamCallBackType create_stream_callback_cb = skyweaver::detail::create_dada_file_stream>; - CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb); + CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); skyweaver::BeamformerPipeline Date: Fri, 20 Sep 2024 13:15:02 +0000 Subject: [PATCH 13/69] read possible wait arguments --- cpp/skyweaver/src/skyweaver_cli.cu | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 24c2f9e..d1b9842 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -497,7 +497,14 @@ int main(int argc, char** argv) [](std::size_t nthreads) { omp_set_num_threads(nthreads); }), "The number of threads to use for incoherent dedispersion") - // Logging options + ("wait-for-space", + po::value() + ->notifier( + [&config](std::string key) { config.configure_wait(key); }), + "Wait for enough disk space for the output. " + "::") + + // Logging options ("log-level", po::value()->default_value("info")->notifier( [](std::string level) { skyweaver::set_log_level(level); }), From 4059a6345d247672b247ab872b76bc83a4360917 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:21:37 +0000 Subject: [PATCH 14/69] That shouldn't be removed --- cpp/skyweaver/MultiFileWriter.cuh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 643647c..d9e0a01 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -24,6 +24,7 @@ struct MultiFileWriterConfig{ std::string output_dir; std::string prefix; std::string extension; + std::string output_basename; PreWriteConfig pre_write; MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("test"), prefix("test"), extension("default"){}; @@ -131,4 +132,4 @@ public: #include "skyweaver/detail/MultiFileWriter.cu" #include "skyweaver/detail/file_writer_callbacks.cpp" -#endif // SKYWEAVER_MULTIFILEWRITER_CUH \ No newline at end of file +#endif // SKYWEAVER_MULTIFILEWRITER_CUH From 86e5e4d5ee85924f64ea11254796aab3b75b5509 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:30:05 +0000 Subject: [PATCH 15/69] set default values --- cpp/skyweaver/src/PipelineConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index b5121e9..a7446a5 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -15,7 +15,7 @@ PipelineConfig::PipelineConfig() _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f) + _stokes_mode("I"), _output_level(24.0f), _pre_write_config({{false, 0, 0, 0}}) { } From 620a52357f40b5c9f2f3d532922e5ef5f97acbda Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Mon, 23 Sep 2024 06:54:56 +0000 Subject: [PATCH 16/69] PreWriting could be used more broadly. So is_enabled shouldn't belong to the wait config --- cpp/skyweaver/PipelineConfig.hpp | 4 ++-- cpp/skyweaver/detail/MultiFileWriter.cu | 2 +- cpp/skyweaver/src/PipelineConfig.cpp | 4 ++-- cpp/skyweaver/src/skyweaver_cli.cu | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index ef1920b..e54dca1 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -11,7 +11,6 @@ namespace skyweaver { struct WaitConfig { - bool is_enabled; int iterations; int sleep_time; std::size_t min_free_space; @@ -19,7 +18,8 @@ namespace skyweaver struct PreWriteConfig { - WaitConfig wait; + bool is_enabled; + WaitConfig wait; }; diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index b28d991..b963a56 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -179,7 +179,7 @@ bool MultiFileWriter::operator()(VectorType const& stream_data, std::size_t stream_idx) { std::size_t const data_size = stream_data.size() * sizeof(typename VectorType::value_type); - if (_pre_write_callback != nullptr && _config.pre_write.wait.is_enabled) + if (_pre_write_callback != nullptr && _config.pre_write.is_enabled) { _pre_write_callback(data_size, _config); } diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index a7446a5..9a4b3eb 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -15,7 +15,7 @@ PipelineConfig::PipelineConfig() _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f), _pre_write_config({{false, 0, 0, 0}}) + _stokes_mode("I"), _output_level(24.0f), _pre_write_config({0, {false, 0, 0}}) { } @@ -182,7 +182,7 @@ void PipelineConfig::configure_wait(std::string argument) std::string token; std::istringstream tokenStream(argument); int indx = 0; - _pre_write_config.wait.is_enabled = true; + _pre_write_config.is_enabled = true; while (std::getline(tokenStream, token, ':')) { if(indx == 0) { diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index d1b9842..04aea39 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -242,7 +242,7 @@ void run_pipeline(Pipeline& pipeline, auto pre_write_callback = [] (std::size_t const size, skyweaver::MultiFileWriterConfig const& config) { - if (!config.pre_write.wait.is_enabled) return; + if (!config.pre_write.is_enabled) return; std::filesystem::space_info space = std::filesystem::space(config.output_dir); size_t limit = std::min(config.pre_write.wait.min_free_space, size); if(space.available >= limit) From c89902c1726bb48171ce13fa5e57b832877e5d85 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Wed, 25 Sep 2024 07:46:53 +0000 Subject: [PATCH 17/69] introduce unique_ptr --- cpp/skyweaver/src/skyweaver_cli.cu | 46 +++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 04aea39..165d272 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -293,45 +293,59 @@ void setup_pipeline(skyweaver::PipelineConfig& config) IBWriterType ib_handler(config, "ib", create_stream_callback_ib, pre_write_callback); using StatsWriterType = - skyweaver::MultiFileWriter>; + skyweaver::MultiFileWriter>; + std::unique_ptr stats_handler; + typename StatsWriterType::CreateStreamCallBackType create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; - - StatsWriterType stats_handler(config, "stats", create_stream_callback_stats, pre_write_callback); - if constexpr(enable_incoherent_dedispersion) { using CBWriterType = skyweaver::MultiFileWriter>; typename CBWriterType::CreateStreamCallBackType create_stream_callback_cb = - skyweaver::detail::create_dada_file_stream>; - skyweaver::MultiFileWriter> - cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); + skyweaver::detail::create_dada_file_stream>; + std::unique_ptr cb_file_writer; + if (config.pre_write_config().is_enabled) + { + stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats, pre_write_callback)); + cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb, pre_write_callback)); + }else{ + stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats)); + cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb)); + } + skyweaver::IncoherentDedispersionPipeline - incoherent_dispersion_pipeline(config, cb_file_writer); + decltype(* cb_file_writer.get())> + incoherent_dispersion_pipeline(config, * cb_file_writer.get()); skyweaver::BeamformerPipeline pipeline(config, incoherent_dispersion_pipeline, ib_handler, - stats_handler); + * stats_handler.get()); run_pipeline(pipeline, config, file_reader, header); } else { using CBWriterType = skyweaver::MultiFileWriter>; + std::unique_ptr cb_file_writer; typename CBWriterType::CreateStreamCallBackType create_stream_callback_cb = skyweaver::detail::create_dada_file_stream>; - CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); - skyweaver::BeamformerPipeline - pipeline(config, cb_file_writer, ib_handler, stats_handler); + pipeline(config, * cb_file_writer.get(), ib_handler, * stats_handler.get()); run_pipeline(pipeline, config, file_reader, header); } } @@ -645,4 +659,4 @@ int main(int argc, char** argv) return ERROR_UNHANDLED_EXCEPTION; } return SUCCESS; -} \ No newline at end of file +} From 96adc08af53abd694e1082d40320c2f7f40f2471 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Wed, 25 Sep 2024 08:38:03 +0000 Subject: [PATCH 18/69] also IBWriter is a unique_ptr now --- cpp/skyweaver/src/skyweaver_cli.cu | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 165d272..45e9ad3 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -290,7 +290,7 @@ void setup_pipeline(skyweaver::PipelineConfig& config) skyweaver::detail::create_dada_file_stream< skyweaver::BTFPowersH>; - IBWriterType ib_handler(config, "ib", create_stream_callback_ib, pre_write_callback); + std::unique_ptr ib_handler; using StatsWriterType = skyweaver::MultiFileWriter>; @@ -308,9 +308,11 @@ void setup_pipeline(skyweaver::PipelineConfig& config) std::unique_ptr cb_file_writer; if (config.pre_write_config().is_enabled) { + ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib, pre_write_callback)); stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats, pre_write_callback)); cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb, pre_write_callback)); }else{ + ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib)); stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats)); cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb)); } @@ -320,12 +322,12 @@ void setup_pipeline(skyweaver::PipelineConfig& config) decltype(* cb_file_writer.get())> incoherent_dispersion_pipeline(config, * cb_file_writer.get()); skyweaver::BeamformerPipeline pipeline(config, incoherent_dispersion_pipeline, - ib_handler, + * ib_handler.get(), * stats_handler.get()); run_pipeline(pipeline, config, file_reader, header); } else { @@ -336,16 +338,20 @@ void setup_pipeline(skyweaver::PipelineConfig& config) skyweaver::detail::create_dada_file_stream>; if (config.pre_write_config().is_enabled) { + ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib, pre_write_callback)); cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb, pre_write_callback)); + stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats, pre_write_callback)); }else{ + ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib)); cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb)); + stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats)); } skyweaver::BeamformerPipeline - pipeline(config, * cb_file_writer.get(), ib_handler, * stats_handler.get()); + pipeline(config, * cb_file_writer.get(), * ib_handler.get(), * stats_handler.get()); run_pipeline(pipeline, config, file_reader, header); } } From 510fc01a7c36a71d2e4a6961354d94e4e9f5d6eb Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 25 Sep 2024 12:18:00 +0200 Subject: [PATCH 19/69] use long double for times, incorporate downstream changes for half channel cfreq change --- .../detail/file_writer_callbacks.cpp | 2 +- cpp/skyweaver/src/ObservationHeader.cpp | 4 +-- cpp/skyweaver/src/SkyCleaver.cu | 28 ++++++++++++------- cpp/skyweaver/src/skyweaver_cli.cu | 2 +- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index e560aac..71f0b42 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -163,7 +163,7 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig header_writer.set("OBS_OFFSET", bytes_written); header_writer.set("OBS_OVERLAP", 0); - double tstart = header.utc_start + stream_data.utc_offset(); + long double tstart = header.utc_start + stream_data.utc_offset(); header_writer.set("UTC_START", tstart); header_writer.set("MJD_START", MJD_UNIX_EPOCH + tstart / 86400.0); diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index f554a59..13255aa 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -58,8 +58,8 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.chan0_idx = parser.get("CHAN0_IDX"); header.obs_offset = parser.get("OBS_OFFSET"); - header.obs_bandwidth = parser.get_or_default("OBS_BW", header.bandwidth); - header.obs_nchans = parser.get_or_default("OBS_NCHAN", header.nchans); + header.obs_bandwidth = parser.get_or_default("OBS_BW", 856e6); + header.obs_nchans = parser.get_or_default("OBS_NCHAN", 4096); header.obs_frequency = parser.get_or_default("OBS_FREQUENCY", parser.get_or_default("OBS_FREQ", header.frequency)); diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index 9df09e3..bf195cd 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -149,11 +149,17 @@ void SkyCleaver::init_readers() BOOST_LOG_TRIVIAL(info) << "Read header from first file: " << header.to_string(); - float obs_centre_freq = header.obs_frequency; - float obs_bandwidth = header.obs_bandwidth; + long double obs_centre_freq = header.obs_frequency; + long double obs_bandwidth = header.obs_bandwidth; + std::size_t obs_nchans = header.obs_nchans; + + long double start_freq = obs_centre_freq - obs_bandwidth / 2; + + + for(int i = 0; i < nbridges; i++) { - int ifreq = std::lround(obs_centre_freq - obs_bandwidth / 2 + - (i + 0.5) * obs_bandwidth / nbridges); + + int ifreq = std::lround( std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) << "Expected frequency [" << i << "]: " << ifreq; @@ -170,6 +176,8 @@ void SkyCleaver::init_readers() 0); } + // print _expected_freqs + std::size_t smallest_data_size = std::numeric_limits::max(); for(const auto& [freq, reader]: _bridge_readers) { @@ -190,13 +198,13 @@ void SkyCleaver::init_readers() 1e-6; // Header has it in microseconds, converting to seconds std::size_t nsamples = std::floor(time_diff / tsamp); - BOOST_LOG_TRIVIAL(info) + BOOST_LOG_TRIVIAL(debug) << "Frequency: " << freq << " Timestamp: " << timestamp << "tsamp: " << tsamp << " Latest timestamp: " << latest_timestamp << " Time difference: " << time_diff << " Number of samples to skip: " << nsamples; - BOOST_LOG_TRIVIAL(info) + BOOST_LOG_TRIVIAL(debug) << "Seeking " << nsamples * _config.ndms() * _config.nbeams() << " bytes in bridge reader for frequency: " << freq; @@ -242,8 +250,7 @@ void SkyCleaver::init_readers() << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; _header = _bridge_readers[_available_freqs[0]]->get_header(); - BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver"; - BOOST_LOG_TRIVIAL(info) << "Header: " << _header.to_string(); + BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver: " << _header.to_string(); _header.nchans = _header.nchans * _config.nbridges(); _header.nbeams = _config.nbeams(); } @@ -260,11 +267,12 @@ void SkyCleaver::init_writers() ? "" : _config.out_prefix() + "_"; std::string output_dir = _config.output_dir(); + for(int idm = 0; idm < _config.ndms(); idm++) { std::string prefix = _config.ndms() > 1 ? out_prefix + "idm_" + - to_string_with_padding(idm, 9, 3) + "_": out_prefix; + to_string_with_padding(_header.dms[idm], 9, 3) + "_": out_prefix; for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { @@ -359,7 +367,7 @@ void SkyCleaver::cleave() reader->read(reinterpret_cast(thrust::raw_pointer_cast( _bridge_data[freq]->data())), gulp_size); // read a big chunk of data - BOOST_LOG_TRIVIAL(info) + BOOST_LOG_TRIVIAL(debug) << "Read " << read_size << " bytes from bridge" << freq; if(read_size < gulp_size * sizeof(InputVectorType::value_type)) { BOOST_LOG_TRIVIAL(warning) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index e168f63..3092164 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -268,7 +268,7 @@ void setup_pipeline(skyweaver::PipelineConfig& config) create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; - StatsWriterType stats_handler(config, "stats", create_stream_callback_stats); + StatsWriterType stats_handler(config, "stats", create_stream_callback_stats); if constexpr(enable_incoherent_dedispersion) { From 4ae82abc1aa6ee824cfee5877f3f3a3c3b99b4ea Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 26 Sep 2024 11:37:29 +0200 Subject: [PATCH 20/69] fixed formatting on MultiFileWriter.cuh --- cpp/skyweaver/MultiFileWriter.cuh | 112 +++++++++++++++++------------- 1 file changed, 64 insertions(+), 48 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 15825c6..c0e8c9d 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -13,43 +13,55 @@ namespace skyweaver { - - - -struct MultiFileWriterConfig{ - - std::size_t header_size; - std::size_t max_file_size; - std::string stokes_mode; - std::string output_dir; - std::string base_output_dir; - std::string prefix; - std::string extension; - std::string output_basename; - - - - MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("default/"), prefix(""), extension(""){}; - MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, std::string extension) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), output_basename(""){ }; - MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), - stokes_mode(other.stokes_mode), output_dir(other.output_dir), base_output_dir(other.base_output_dir), prefix(other.prefix), extension(other.extension), output_basename(other.output_basename){}; - MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other){ - header_size = other.header_size; - max_file_size = other.max_file_size; - stokes_mode = other.stokes_mode; - output_dir = other.output_dir; - prefix = other.prefix; - extension = other.extension; - output_basename = other.output_basename; - base_output_dir = other.base_output_dir; - return *this; - } - - std::string to_string(){ - return "header_size: " + std::to_string(header_size) + ", max_file_size: " + std::to_string(max_file_size) - + ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + ", prefix: " + prefix + ", extension: " + extension - + ", output_basename: " + output_basename + ", base_output_dir: " + base_output_dir; - } +struct MultiFileWriterConfig { + std::size_t header_size; + std::size_t max_file_size; + std::string stokes_mode; + std::string output_dir; + std::string base_output_dir; + std::string prefix; + std::string extension; + std::string output_basename; + + MultiFileWriterConfig() + : header_size(4096), max_file_size(2147483647), stokes_mode("I"), + output_dir("default/"), prefix(""), extension("") {}; + MultiFileWriterConfig(std::size_t header_size, + std::size_t max_file_size, + std::string stokes_mode, + std::string output_dir, + std::string prefix, + std::string extension) + : header_size(header_size), max_file_size(max_file_size), + stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), + extension(extension), output_basename("") {}; + MultiFileWriterConfig(MultiFileWriterConfig const& other) + : header_size(other.header_size), max_file_size(other.max_file_size), + stokes_mode(other.stokes_mode), output_dir(other.output_dir), + base_output_dir(other.base_output_dir), prefix(other.prefix), + extension(other.extension), output_basename(other.output_basename) {}; + MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other) + { + header_size = other.header_size; + max_file_size = other.max_file_size; + stokes_mode = other.stokes_mode; + output_dir = other.output_dir; + prefix = other.prefix; + extension = other.extension; + output_basename = other.output_basename; + base_output_dir = other.base_output_dir; + return *this; + } + + std::string to_string() + { + return "header_size: " + std::to_string(header_size) + + ", max_file_size: " + std::to_string(max_file_size) + + ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + + ", prefix: " + prefix + ", extension: " + extension + + ", output_basename: " + output_basename + + ", base_output_dir: " + base_output_dir; + } }; /** * @brief A class for handling writing of DescribedVectors @@ -58,12 +70,13 @@ struct MultiFileWriterConfig{ template class MultiFileWriter { -public: - - using CreateStreamCallBackType = std::function(MultiFileWriterConfig const&, - ObservationHeader const&, - VectorType const&, - std::size_t)>; + public: + using CreateStreamCallBackType = + std::function( + MultiFileWriterConfig const&, + ObservationHeader const&, + VectorType const&, + std::size_t)>; public: /** @@ -74,8 +87,12 @@ public: * (used to avoid clashing file names). */ // MultiFileWriter(PipelineConfig const& config, std::string tag = ""); - MultiFileWriter(PipelineConfig const& config, std::string tag, CreateStreamCallBackType create_stream_callback); - MultiFileWriter(MultiFileWriterConfig config, std::string tag, CreateStreamCallBackType create_stream_callback); + MultiFileWriter(PipelineConfig const& config, + std::string tag, + CreateStreamCallBackType create_stream_callback); + MultiFileWriter(MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback); MultiFileWriter(MultiFileWriter const&) = delete; /** @@ -110,13 +127,12 @@ public: */ bool operator()(VectorType const& stream_data, std::size_t stream_idx = 0); - bool write(VectorType const& stream_data, - std::size_t stream_idx = 0); + bool write(VectorType const& stream_data, std::size_t stream_idx = 0); private: bool has_stream(std::size_t stream_idx); FileOutputStream& create_stream(VectorType const& stream_data, - std::size_t stream_idx); + std::size_t stream_idx); std::string get_output_dir(VectorType const& stream_data, std::size_t stream_idx); std::string get_basefilename(VectorType const& stream_data, From 2787d007eeef1f0a5a54586c75cd8c7f9d577a55 Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 26 Sep 2024 12:04:05 +0200 Subject: [PATCH 21/69] formatting --- cpp/skyweaver/MultiFileWriter.cuh | 3 + cpp/skyweaver/ObservationHeader.hpp | 70 ++-- cpp/skyweaver/SkyCleaver.cuh | 53 +-- cpp/skyweaver/SkyCleaverConfig.hpp | 95 ++--- cpp/skyweaver/detail/MultiFileWriter.cu | 56 ++- .../detail/file_writer_callbacks.cpp | 344 +++++++++--------- cpp/skyweaver/src/CoherentDedisperser.cu | 2 +- cpp/skyweaver/src/ObservationHeader.cpp | 44 ++- cpp/skyweaver/src/skycleaver_cli.cu | 315 ++++++++-------- cpp/skyweaver/src/skyweaver_cli.cu | 55 +-- 10 files changed, 512 insertions(+), 525 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index c0e8c9d..352b2f5 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -26,6 +26,7 @@ struct MultiFileWriterConfig { MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("default/"), prefix(""), extension("") {}; + MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, @@ -35,11 +36,13 @@ struct MultiFileWriterConfig { : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), output_basename("") {}; + MultiFileWriterConfig(MultiFileWriterConfig const& other) : header_size(other.header_size), max_file_size(other.max_file_size), stokes_mode(other.stokes_mode), output_dir(other.output_dir), base_output_dir(other.base_output_dir), prefix(other.prefix), extension(other.extension), output_basename(other.output_basename) {}; + MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other) { header_size = other.header_size; diff --git a/cpp/skyweaver/ObservationHeader.hpp b/cpp/skyweaver/ObservationHeader.hpp index 356d71e..d09564d 100644 --- a/cpp/skyweaver/ObservationHeader.hpp +++ b/cpp/skyweaver/ObservationHeader.hpp @@ -4,12 +4,12 @@ #include "psrdada_cpp/raw_bytes.hpp" #include "skyweaver/Header.hpp" #include "skyweaver/PipelineConfig.hpp" + #include namespace skyweaver { - struct ObservationHeader { std::size_t nchans = 0; // Number of frequency channels in the subband std::size_t npol = 0; // Number of polarisations @@ -30,37 +30,37 @@ struct ObservationHeader { long double sync_time = 0.0; // The UNIX epoch of the sampler zero long double utc_start = 0.0; // The UTC start time of the data long double mjd_start = 0.0; // The MJD start time of the data - std::size_t obs_offset = 0; // The offset of the current file from UTC_START in bytesß - long double refdm = 0.0; // Reference DM - std::size_t ibeam = 0.0; // Beam number - std::size_t nbeams = 0; // Number of beams - std::string source_name; // Name of observation target - std::string ra; // Right ascension - std::string dec; // Declination - std::string telescope; // Telescope name - std::string instrument; // Name of the recording instrument - std::string order; // Order of the dimensions in the data - std::string ndms; // Number of DMs - std::vector dms; // DMs - - std::string to_string() const; // Convert the header to a string - - long double az; // Azimuth - long double za; // Zenith angle - std::size_t machineid = 0; // Machine ID - std::size_t nifs = 0; // Number of IFs - std::size_t telescopeid = 0; // Telescope ID - std::size_t datatype = 0; // Data type - std::size_t barycentric = 0; // Barycentric correction - std::string rawfile; // Raw file name - double fch1 = 0.0; // Centre frequency of the first channel - double foff = 0.0; // Frequency offset between channels - - bool sigproc_params = false; // Whether to include sigproc parameters + std::size_t obs_offset = + 0; // The offset of the current file from UTC_START in bytesß + long double refdm = 0.0; // Reference DM + std::size_t ibeam = 0.0; // Beam number + std::size_t nbeams = 0; // Number of beams + std::string source_name; // Name of observation target + std::string ra; // Right ascension + std::string dec; // Declination + std::string telescope; // Telescope name + std::string instrument; // Name of the recording instrument + std::string order; // Order of the dimensions in the data + std::string ndms; // Number of DMs + std::vector dms; // DMs + + std::string to_string() const; // Convert the header to a string + + long double az; // Azimuth + long double za; // Zenith angle + std::size_t machineid = 0; // Machine ID + std::size_t nifs = 0; // Number of IFs + std::size_t telescopeid = 0; // Telescope ID + std::size_t datatype = 0; // Data type + std::size_t barycentric = 0; // Barycentric correction + std::string rawfile; // Raw file name + double fch1 = 0.0; // Centre frequency of the first channel + double foff = 0.0; // Frequency offset between channels + + bool sigproc_params = false; // Whether to include sigproc parameters ObservationHeader() = default; - ObservationHeader(ObservationHeader const&) = default; + ObservationHeader(ObservationHeader const&) = default; ObservationHeader& operator=(ObservationHeader const&) = default; - }; // template for comparing two floating point objects @@ -72,7 +72,6 @@ is_close(T a, T b, T tolerance = 1e-12) return std::fabs(a - b) < tolerance; } - /** * @brief Parse header information for a DADA header block * @@ -90,15 +89,6 @@ void update_config(PipelineConfig& config, ObservationHeader const& header); bool are_headers_similar(ObservationHeader const& header1, ObservationHeader const& header2); - - - - - - } // namespace skyweaver - - - #endif // SKYWEAVER_OBSERVATIONHEADER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaver.cuh b/cpp/skyweaver/SkyCleaver.cuh index 8c91290..4c7be58 100644 --- a/cpp/skyweaver/SkyCleaver.cuh +++ b/cpp/skyweaver/SkyCleaver.cuh @@ -1,78 +1,81 @@ #ifndef SKYWEAVER_SKYCLEAVER_HPP #define SKYWEAVER_SKYCLEAVER_HPP -#include "skyweaver/DescribedVector.hpp" #include "boost/log/trivial.hpp" #include "psrdada_cpp/psrdadaheader.hpp" #include "psrdada_cpp/raw_bytes.hpp" +#include "skyweaver/DescribedVector.hpp" +#include "skyweaver/MultiFileReader.cuh" +#include "skyweaver/MultiFileWriter.cuh" #include "skyweaver/ObservationHeader.hpp" +#include "skyweaver/SkyCleaverConfig.hpp" +#include "skyweaver/Timer.hpp" #include #include #include #include +#include #include #include #include -#include -#include "skyweaver/MultiFileReader.cuh" -#include "skyweaver/SkyCleaverConfig.hpp" -#include "skyweaver/MultiFileWriter.cuh" -#include "skyweaver/ObservationHeader.hpp" -#include "skyweaver/Timer.hpp" namespace skyweaver { -struct BridgeReader -{ +struct BridgeReader { public: std::vector _tdb_filenames; std::unique_ptr _tdb_reader; std::string freq; }; // BridgeReader + + class SkyCleaver { public: - using InputType = int8_t; - using OutputType = uint8_t; - using InputVectorType = TDBPowersH; + using InputType = int8_t; + using OutputType = uint8_t; + using InputVectorType = TDBPowersH; using OutputVectorType = TFPowersH; - using FreqType = std::size_t; // up to the nearest Hz - using BeamNumberType = std::size_t; - using DMNumberType = std::size_t; + using FreqType = std::size_t; // up to the nearest Hz + using BeamNumberType = std::size_t; + using DMNumberType = std::size_t; - - private: + private: SkyCleaverConfig const& _config; std::map> _bridge_readers; - std::map> _bridge_data; - + std::map> _bridge_data; + std::vector _expected_freqs; std::vector _available_freqs; std::size_t _nsamples_to_read; ObservationHeader _header; std::vector _beam_filenames; - std::map>>> _beam_writers; - std::map>> _beam_data; + std::map>>> + _beam_writers; + std::map>> + _beam_data; std::size_t _total_beam_writers; - void init_readers(); void init_writers(); Timer _timer; - public: SkyCleaver(SkyCleaverConfig const& config); - SkyCleaver(SkyCleaver const&) = delete; + SkyCleaver(SkyCleaver const&) = delete; void operator=(SkyCleaver const&) = delete; void cleave(); - + }; // class SkyCleaver + } // namespace skyweaver #endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index a1ae8c7..feadd9b 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -1,6 +1,7 @@ #ifndef SKYCLEAVERCONFIG_HPP #define SKYCLEAVERCONFIG_HPP -namespace skyweaver { +namespace skyweaver +{ class SkyCleaverConfig { @@ -20,52 +21,56 @@ class SkyCleaverConfig std::string _stokes_mode; std::size_t _dada_header_size; - - - - - public: - - SkyCleaverConfig() : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096) {} - SkyCleaverConfig(SkyCleaverConfig const&) = delete; - - void output_dir(std::string output_dir) { _output_dir = output_dir; } - void root_dir(std::string root_dir) { _root_dir = root_dir; } - void root_prefix(std::string root_prefix) { _root_prefix = root_prefix; } - void out_prefix(std::string out_prefix) { _out_prefix = out_prefix; } - void nthreads(std::size_t nthreads) { _nthreads = nthreads; } - void nsamples_per_block(std::size_t nsamples_per_block) { _nsamples_per_block = nsamples_per_block; } - void nchans(std::size_t nchans) { _nchans = nchans; } - void nbeams(std::size_t nbeams) { _nbeams = nbeams; } - void max_ram_gb(std::size_t max_ram_gb) { _max_ram_gb = max_ram_gb; } - void max_output_filesize(std::size_t max_output_filesize) { _max_output_filesize = max_output_filesize; } - void stream_id(std::size_t stream_id) { _stream_id = stream_id; } - void nbridges(std::size_t nbridges) { _nbridges = nbridges; } - void ndms(std::size_t ndms) { _ndms = ndms; } - void stokes_mode(std::string stokes_mode) { _stokes_mode = stokes_mode; } - void dada_header_size(std::size_t dada_header_size) { _dada_header_size = dada_header_size; } - - - - std::string output_dir() const { return _output_dir; } - std::string root_dir() const { return _root_dir; } - std::string root_prefix() const { return _root_prefix; } - std::string out_prefix() const { return _out_prefix; } - std::size_t nthreads() const { return _nthreads; } - std::size_t nsamples_per_block() const { return _nsamples_per_block; } - std::size_t nchans() const { return _nchans; } - std::size_t nbeams() const { return _nbeams; } - std::size_t max_ram_gb() const { return _max_ram_gb; } - std::size_t max_output_filesize() const { return _max_output_filesize; } - std::size_t stream_id() const { return _stream_id; } - std::size_t nbridges() const { return _nbridges; } - std::size_t ndms() const { return _ndms; } - std::string stokes_mode() const { return _stokes_mode; } - std::size_t dada_header_size() const { return _dada_header_size; } - - + public: + SkyCleaverConfig() + : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), + _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), + _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), + _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096) + { + } + SkyCleaverConfig(SkyCleaverConfig const&) = delete; + void output_dir(std::string output_dir) { _output_dir = output_dir; } + void root_dir(std::string root_dir) { _root_dir = root_dir; } + void root_prefix(std::string root_prefix) { _root_prefix = root_prefix; } + void out_prefix(std::string out_prefix) { _out_prefix = out_prefix; } + void nthreads(std::size_t nthreads) { _nthreads = nthreads; } + void nsamples_per_block(std::size_t nsamples_per_block) + { + _nsamples_per_block = nsamples_per_block; + } + void nchans(std::size_t nchans) { _nchans = nchans; } + void nbeams(std::size_t nbeams) { _nbeams = nbeams; } + void max_ram_gb(std::size_t max_ram_gb) { _max_ram_gb = max_ram_gb; } + void max_output_filesize(std::size_t max_output_filesize) + { + _max_output_filesize = max_output_filesize; + } + void stream_id(std::size_t stream_id) { _stream_id = stream_id; } + void nbridges(std::size_t nbridges) { _nbridges = nbridges; } + void ndms(std::size_t ndms) { _ndms = ndms; } + void stokes_mode(std::string stokes_mode) { _stokes_mode = stokes_mode; } + void dada_header_size(std::size_t dada_header_size) + { + _dada_header_size = dada_header_size; + } + std::string output_dir() const { return _output_dir; } + std::string root_dir() const { return _root_dir; } + std::string root_prefix() const { return _root_prefix; } + std::string out_prefix() const { return _out_prefix; } + std::size_t nthreads() const { return _nthreads; } + std::size_t nsamples_per_block() const { return _nsamples_per_block; } + std::size_t nchans() const { return _nchans; } + std::size_t nbeams() const { return _nbeams; } + std::size_t max_ram_gb() const { return _max_ram_gb; } + std::size_t max_output_filesize() const { return _max_output_filesize; } + std::size_t stream_id() const { return _stream_id; } + std::size_t nbridges() const { return _nbridges; } + std::size_t ndms() const { return _ndms; } + std::string stokes_mode() const { return _stokes_mode; } + std::size_t dada_header_size() const { return _dada_header_size; } }; } // namespace skyweaver #endif // SKYCLEAVERCONFIG_HPP \ No newline at end of file diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 7529621..cfe8a9e 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -7,8 +7,6 @@ #include #include - - /** * Now write a DADA file per DM * with optional time splitting @@ -20,7 +18,6 @@ namespace skyweaver namespace { - std::string get_formatted_time(long double unix_timestamp) { char formatted_time[80]; @@ -40,30 +37,31 @@ std::string get_formatted_time(long double unix_timestamp) // } template -MultiFileWriter::MultiFileWriter(PipelineConfig const& config, - std::string tag, - CreateStreamCallBackType create_stream_callback) +MultiFileWriter::MultiFileWriter( + PipelineConfig const& config, + std::string tag, + CreateStreamCallBackType create_stream_callback) : _tag(tag), _create_stream_callback(create_stream_callback) { MultiFileWriterConfig writer_config; - writer_config.header_size = config.dada_header_size(); - writer_config.max_file_size = config.max_output_filesize(); - writer_config.stokes_mode = config.stokes_mode(); + writer_config.header_size = config.dada_header_size(); + writer_config.max_file_size = config.max_output_filesize(); + writer_config.stokes_mode = config.stokes_mode(); writer_config.base_output_dir = config.output_dir(); _config = writer_config; } template -MultiFileWriter::MultiFileWriter(MultiFileWriterConfig config, - std::string tag, - CreateStreamCallBackType create_stream_callback) - : _config(config), _tag(tag), _create_stream_callback(create_stream_callback) +MultiFileWriter::MultiFileWriter( + MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback) + : _config(config), _tag(tag), + _create_stream_callback(create_stream_callback) { } - - template MultiFileWriter::~MultiFileWriter(){}; @@ -81,14 +79,10 @@ bool MultiFileWriter::has_stream(std::size_t stream_idx) } template -FileOutputStream& +FileOutputStream& MultiFileWriter::create_stream(VectorType const& stream_data, std::size_t stream_idx) { - - - - _config.output_dir = get_output_dir(stream_data, stream_idx); if(_config.extension.empty()) { @@ -97,10 +91,8 @@ MultiFileWriter::create_stream(VectorType const& stream_data, _config.output_basename = get_basefilename(stream_data, stream_idx); - - _file_streams[stream_idx] = _create_stream_callback(_config, _header, stream_data, stream_idx); - - + _file_streams[stream_idx] = + _create_stream_callback(_config, _header, stream_data, stream_idx); return *_file_streams[stream_idx]; } @@ -114,9 +106,8 @@ MultiFileWriter::get_output_dir(VectorType const& stream_data, // // std::stringstream output_dir; output_dir << _config.base_output_dir << "/" - << get_formatted_time(_header.utc_start) << "/" - << stream_idx; - + << get_formatted_time(_header.utc_start) << "/" << stream_idx; + return output_dir.str(); } @@ -147,10 +138,9 @@ MultiFileWriter::get_extension(VectorType const& stream_data) { std::string dims = stream_data.dims_as_string(); for(auto& c: dims) { c = std::tolower(static_cast(c)); } - if(dims =="t") { + if(dims == "t") { return ".dat"; - } - else if(dims == "tf") { + } else if(dims == "tf") { return ".fil"; } return "." + dims; @@ -182,11 +172,9 @@ bool MultiFileWriter::operator()(VectorType const& stream_data, template bool MultiFileWriter::write(VectorType const& stream_data, - std::size_t stream_idx) + std::size_t stream_idx) { - - return this->operator()(stream_data, stream_idx); - + return this->operator()(stream_data, stream_idx); } } // namespace skyweaver \ No newline at end of file diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 71f0b42..10d9929 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -4,22 +4,23 @@ #include "skyweaver/MultiFileWriter.cuh" #include "skyweaver/ObservationHeader.hpp" #include "skyweaver/SigprocHeader.hpp" + #include -#include -#include -#include -#include #include +#include +#include +#include +#include #include +#include #include -#include -#include #include -#include - +#include +#include -namespace { - /** +namespace +{ +/** * The expectation is that each file contains data in * TBTF order. Need to explicitly update: * INNER_T --> the number of timesamples per block that was processed @@ -66,16 +67,17 @@ CHAN0_IDX 2688 )"; #define MJD_UNIX_EPOCH 40587.0 -} +} // namespace namespace skyweaver { -namespace detail +namespace detail { -template inline -std::unique_ptr create_dada_file_stream(MultiFileWriterConfig const& config, - ObservationHeader const& header, - VectorType const& stream_data, - std::size_t stream_idx) +template +inline std::unique_ptr +create_dada_file_stream(MultiFileWriterConfig const& config, + ObservationHeader const& header, + VectorType const& stream_data, + std::size_t stream_idx) { BOOST_LOG_TRIVIAL(debug) << "Creating stream based on stream prototype: " << stream_data.describe(); @@ -85,113 +87,120 @@ std::unique_ptr create_dada_file_stream(MultiFileWriterConfig config.max_file_size / stream_data.size() / sizeof(typename VectorType::value_type)) * stream_data.size(); - + BOOST_LOG_TRIVIAL(debug) << "Maximum allowed file size = " << filesize << " bytes (+header)"; std::stringstream output_dir; - output_dir << config.output_dir << "/" - << std::fixed << std::setfill('0') << std::setw(9) - << static_cast(header.frequency); - - std::stringstream output_basename; - output_basename << config.output_basename << "_" - << std::fixed << std::setfill('0') << std::setw(9) - << static_cast(header.frequency); - - std::unique_ptr file_stream = std::make_unique( - output_dir.str(), - output_basename.str(), - config.extension, - filesize, - [&, header, stream_data, stream_idx, filesize]( - std::size_t& header_size, - std::size_t bytes_written, - std::size_t file_idx) -> std::shared_ptr { - header_size = config.header_size; - char* temp_header = new char[header_size]; - std::fill(temp_header, temp_header + header_size, 0); - std::memcpy(temp_header, - default_dada_header.c_str(), - default_dada_header.size()); - psrdada_cpp::RawBytes bytes(temp_header, - header_size, - header_size, - false); - Header header_writer(bytes); - header_writer.set("SOURCE", header.source_name); - header_writer.set("RA", header.ra); - header_writer.set("DEC", header.dec); - header_writer.set("NBEAM", stream_data.nbeams()); - header_writer.set("NCHAN", stream_data.nchannels()); - header_writer.set("OBS_NCHAN", header.obs_nchans); - header_writer.set("OBS_FREQUENCY", - header.obs_frequency); - header_writer.set("OBS_BW", header.obs_bandwidth); - header_writer.set("NSAMP", stream_data.nsamples()); - if(stream_data.ndms()) { - header_writer.set("NDMS", stream_data.ndms()); - header_writer.set("DMS", stream_data.dms(), 7); - } - header_writer.set( - "COHERENT_DM", - static_cast(stream_data.reference_dm())); - try{ - header_writer.set("FREQ", std::accumulate( - stream_data.frequencies().begin(), - stream_data.frequencies().end(), - 0.0)/stream_data.frequencies().size()); - } catch(std::runtime_error& ){ - BOOST_LOG_TRIVIAL(warning) << "Warning: Frequencies array was stale, using the centre frequency from the header"; - header_writer.set("FREQ", header.frequency); - } - - header_writer.set("BW", header.bandwidth); - header_writer.set("TSAMP", stream_data.tsamp() * 1e6); - if(config.stokes_mode == "IQUV") { - header_writer.set("NPOL", 4); - } else { - header_writer.set("NPOL", 1); - } - header_writer.set("STOKES_MODE", - config.stokes_mode); - header_writer.set("ORDER", - stream_data.dims_as_string()); - header_writer.set("CHAN0_IDX", header.chan0_idx); - header_writer.set("FILE_SIZE", filesize); - header_writer.set("FILE_NUMBER", file_idx); - header_writer.set("OBS_OFFSET", bytes_written); - header_writer.set("OBS_OVERLAP", 0); - - long double tstart = header.utc_start + stream_data.utc_offset(); - - header_writer.set("UTC_START", tstart); - header_writer.set("MJD_START", MJD_UNIX_EPOCH + tstart / 86400.0); - std::shared_ptr header_ptr( - temp_header, - std::default_delete()); - - return header_ptr; - }); + output_dir << config.output_dir << "/" << std::fixed << std::setfill('0') + << std::setw(9) << static_cast(header.frequency); + + std::stringstream output_basename; + output_basename << config.output_basename << "_" << std::fixed + << std::setfill('0') << std::setw(9) + << static_cast(header.frequency); + + std::unique_ptr file_stream = + std::make_unique( + output_dir.str(), + output_basename.str(), + config.extension, + filesize, + [&, header, stream_data, stream_idx, filesize]( + std::size_t& header_size, + std::size_t bytes_written, + std::size_t file_idx) -> std::shared_ptr { + header_size = config.header_size; + char* temp_header = new char[header_size]; + std::fill(temp_header, temp_header + header_size, 0); + std::memcpy(temp_header, + default_dada_header.c_str(), + default_dada_header.size()); + psrdada_cpp::RawBytes bytes(temp_header, + header_size, + header_size, + false); + Header header_writer(bytes); + header_writer.set("SOURCE", header.source_name); + header_writer.set("RA", header.ra); + header_writer.set("DEC", header.dec); + header_writer.set("NBEAM", stream_data.nbeams()); + header_writer.set("NCHAN", + stream_data.nchannels()); + header_writer.set("OBS_NCHAN", header.obs_nchans); + header_writer.set("OBS_FREQUENCY", + header.obs_frequency); + header_writer.set("OBS_BW", header.obs_bandwidth); + header_writer.set("NSAMP", stream_data.nsamples()); + if(stream_data.ndms()) { + header_writer.set("NDMS", stream_data.ndms()); + header_writer.set("DMS", stream_data.dms(), 7); + } + header_writer.set( + "COHERENT_DM", + static_cast(stream_data.reference_dm())); + try { + header_writer.set( + "FREQ", + std::accumulate(stream_data.frequencies().begin(), + stream_data.frequencies().end(), + 0.0) / + stream_data.frequencies().size()); + } catch(std::runtime_error&) { + BOOST_LOG_TRIVIAL(warning) + << "Warning: Frequencies array was stale, using the " + "centre frequency from the header"; + header_writer.set("FREQ", header.frequency); + } + + header_writer.set("BW", header.bandwidth); + header_writer.set("TSAMP", + stream_data.tsamp() * 1e6); + if(config.stokes_mode == "IQUV") { + header_writer.set("NPOL", 4); + } else { + header_writer.set("NPOL", 1); + } + header_writer.set("STOKES_MODE", + config.stokes_mode); + header_writer.set("ORDER", + stream_data.dims_as_string()); + header_writer.set("CHAN0_IDX", header.chan0_idx); + header_writer.set("FILE_SIZE", filesize); + header_writer.set("FILE_NUMBER", file_idx); + header_writer.set("OBS_OFFSET", bytes_written); + header_writer.set("OBS_OVERLAP", 0); + + long double tstart = + header.utc_start + stream_data.utc_offset(); + + header_writer.set("UTC_START", tstart); + header_writer.set("MJD_START", + MJD_UNIX_EPOCH + + tstart / 86400.0); + std::shared_ptr header_ptr( + temp_header, + std::default_delete()); + + return header_ptr; + }); return file_stream; } -template inline -std::unique_ptr create_sigproc_file_stream(MultiFileWriterConfig const& config, - ObservationHeader const& obs_header, - VectorType const& stream_data, - std::size_t stream_idx) +template +inline std::unique_ptr +create_sigproc_file_stream(MultiFileWriterConfig const& config, + ObservationHeader const& obs_header, + VectorType const& stream_data, + std::size_t stream_idx) { - - - BOOST_LOG_TRIVIAL(debug) << "Creating stream based on stream prototype: " << stream_data.describe(); ObservationHeader header = obs_header; BOOST_LOG_TRIVIAL(info) << "Header: " << header.to_string(); - + // Here we round the file size to a multiple of the stream prototype std::size_t filesize = std::max(1ul, @@ -199,36 +208,40 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon sizeof(typename VectorType::value_type)) * stream_data.size(); BOOST_LOG_TRIVIAL(debug) - << "Maximum allowed file size = " << filesize << " bytes (+header)"; + << "Maximum allowed file size = " << filesize << " bytes (+header)"; - - double foff = -1* static_cast(header.obs_bandwidth / header.nchans)/1e6;// MHz - // 1* foff instead of 0.5* foff below because the dedispersion causes all the frequencies to change by half the bandwidth to refer to the bottom of the channel - double fch1 = static_cast(header.obs_frequency + header.obs_bandwidth / 2.0)/1e6 + foff; // MHz + double foff = -1 * + static_cast(header.obs_bandwidth / header.nchans) / + 1e6; // MHz + // 1* foff instead of 0.5* foff below because the dedispersion causes all + // the frequencies to change by half the bandwidth to refer to the bottom of + // the channel + double fch1 = + static_cast(header.obs_frequency + header.obs_bandwidth / 2.0) / + 1e6 + + foff; // MHz double utc_start = static_cast(header.utc_start); header.mjd_start = (utc_start / 86400.0) + MJD_UNIX_EPOCH; - uint32_t datatype = 0; + uint32_t datatype = 0; uint32_t barycentric = 0; // uint32_t ibeam = 0; - double az = 0.0; - double za = 0.0; + double az = 0.0; + double za = 0.0; uint32_t nifs = header.npol; header.sigproc_params = true; - header.rawfile = std::string("unset"); - header.fch1 = fch1; - header.foff = foff; - header.tsamp = header.tsamp/1e6; - header.az = az; - header.za = za; - header.datatype = datatype; - header.barycentric = barycentric; - header.nifs = nifs; - header.telescopeid = 64; - - + header.rawfile = std::string("unset"); + header.fch1 = fch1; + header.foff = foff; + header.tsamp = header.tsamp / 1e6; + header.az = az; + header.za = za; + header.datatype = datatype; + header.barycentric = barycentric; + header.nifs = nifs; + header.telescopeid = 64; BOOST_LOG_TRIVIAL(info) << "Creating Sigproc file stream"; @@ -246,42 +259,41 @@ std::unique_ptr create_sigproc_file_stream(MultiFileWriterCon // // causing compiler bugs prior to g++ 5.x // char formatted_time[80]; // strftime(formatted_time, 80, "%Y-%m-%d-%H:%M:%S", ptm); - // base_filename << formatted_time; - - //make config.output_dir if it does not exist - - - std::unique_ptr file_stream = std::make_unique( - config.output_dir, - config.output_basename, - config.extension, - filesize, - [header](std::size_t& header_size, - std::size_t bytes_written, - std::size_t file_idx) -> std::shared_ptr { - // We do not explicitly delete[] this array - // Cleanup is handled by the shared pointer - // created below - std::ostringstream header_stream; - // get ostream from temp_header - - - - SigprocHeader sigproc_header(header); - double mjd_offset = (((bytes_written / (header.nbits / 8.0)) / (header.nchans)) * - header.tsamp) / - (86400.0); - sigproc_header.add_time_offset(mjd_offset); - sigproc_header.write_header(header_stream); - std::string header_str = header_stream.str(); - header_size = header_str.size(); - char* header_cstr = new char[header_size]; - std::copy(header_str.begin(), header_str.end(), header_cstr); - std::shared_ptr header_ptr( - header_cstr, - std::default_delete()); - return header_ptr; - }); + // base_filename << formatted_time; + + // make config.output_dir if it does not exist + + std::unique_ptr file_stream = + std::make_unique( + config.output_dir, + config.output_basename, + config.extension, + filesize, + [header](std::size_t& header_size, + std::size_t bytes_written, + std::size_t file_idx) -> std::shared_ptr { + // We do not explicitly delete[] this array + // Cleanup is handled by the shared pointer + // created below + std::ostringstream header_stream; + // get ostream from temp_header + + SigprocHeader sigproc_header(header); + double mjd_offset = (((bytes_written / (header.nbits / 8.0)) / + (header.nchans)) * + header.tsamp) / + (86400.0); + sigproc_header.add_time_offset(mjd_offset); + sigproc_header.write_header(header_stream); + std::string header_str = header_stream.str(); + header_size = header_str.size(); + char* header_cstr = new char[header_size]; + std::copy(header_str.begin(), header_str.end(), header_cstr); + std::shared_ptr header_ptr( + header_cstr, + std::default_delete()); + return header_ptr; + }); return file_stream; } diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 52e4f41..12d9976 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -61,7 +61,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, pipeline_config.coherent_dms()); } /* - * @brief Create a new CoherentDedis§rser object, mostly used only for + * @brief Create a new CoherentDedisperser object, mostly used only for * testing * * @param config The config reference diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index 13255aa..7040a4e 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -10,7 +10,7 @@ std::vector parse_float_list(std::string const& str) { std::vector values; std::size_t start = 0; - std::size_t end = 0; + std::size_t end = 0; while(end != std::string::npos) { end = str.find(',', start); values.push_back(std::stof(str.substr(start, end - start))); @@ -25,7 +25,7 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, Header parser(raw_header); header.order = parser.get("ORDER"); - if(header.order.find("A") != std::string::npos) { + if(header.order.find("A") != std::string::npos) { header.nantennas = parser.get("NANT"); header.sample_clock = parser.get("SAMPLE_CLOCK"); @@ -35,16 +35,16 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.sync_time = parser.get("SYNC_TIME"); } - - header.refdm = parser.get_or_default("COHERENT_DM", 0.0); + header.refdm = + parser.get_or_default("COHERENT_DM", 0.0); - header.npol = parser.get("NPOL"); - header.nbits = parser.get("NBIT"); - header.nchans = parser.get("NCHAN"); + header.npol = parser.get("NPOL"); + header.nbits = parser.get("NBIT"); + header.nchans = parser.get("NCHAN"); - header.bandwidth = parser.get("BW"); - header.frequency = parser.get("FREQ"); + header.bandwidth = parser.get("BW"); + header.frequency = parser.get("FREQ"); header.tsamp = parser.get("TSAMP"); @@ -58,18 +58,21 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, header.chan0_idx = parser.get("CHAN0_IDX"); header.obs_offset = parser.get("OBS_OFFSET"); - header.obs_bandwidth = parser.get_or_default("OBS_BW", 856e6); - header.obs_nchans = parser.get_or_default("OBS_NCHAN", 4096); - header.obs_frequency = parser.get_or_default("OBS_FREQUENCY", - parser.get_or_default("OBS_FREQ", - header.frequency)); + header.obs_bandwidth = + parser.get_or_default("OBS_BW", 856e6); + header.obs_nchans = + parser.get_or_default("OBS_NCHAN", 4096); + header.obs_frequency = + parser.get_or_default( + "OBS_FREQUENCY", + parser.get_or_default( + "OBS_FREQ", + header.frequency)); header.ndms = parser.get_or_default("NDMS", "0"); if(header.ndms != "0") { header.dms = parse_float_list(parser.get("DMS")); } - - } void validate_header(ObservationHeader const& header, PipelineConfig const& config) @@ -104,7 +107,6 @@ void update_config(PipelineConfig& config, ObservationHeader const& header) // TO DO: might need to add other variables in the future. } - bool are_headers_similar(ObservationHeader const& header1, ObservationHeader const& header2) { @@ -153,14 +155,10 @@ std::string ObservationHeader::to_string() const if(ndms != "0") { oss << " ndms: " << ndms << "\n"; oss << " dms: "; - for(auto dm : dms) { - oss << dm << " "; - } + for(auto dm: dms) { oss << dm << " "; } oss << "\n"; } - if(sigproc_params) - { - + if(sigproc_params) { oss << " Sigproc parameters:\n" << " az: " << az << "\n" << " za: " << za << "\n" diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cu index ff7ad8f..fdb9278 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cu +++ b/cpp/skyweaver/src/skycleaver_cli.cu @@ -1,4 +1,11 @@ +#include "skyweaver/SkyCleaver.cuh" +#include "skyweaver/SkyCleaverConfig.hpp" +#include "skyweaver/logging.hpp" + #include +#include +#include +#include #include #include #include @@ -12,20 +19,12 @@ #include #include #include -#include -#include -#include -#include "skyweaver/logging.hpp" -#include "skyweaver/SkyCleaverConfig.hpp" -#include "skyweaver/SkyCleaver.cuh" #define BOOST_LOG_DYN_LINK 1 - -namespace +namespace { - -std::string skycleaver_splash=R"( +std::string skycleaver_splash = R"( __ __ .-----.| |--.--.--.----.| |.-----.---.-.--.--.-----.----. |__ --|| <| | | __|| || -__| _ | | | -__| _| @@ -39,8 +38,7 @@ const size_t ERROR_UNHANDLED_EXCEPTION = 2; const char* build_time = __DATE__ " " __TIME__; - -} +} // namespace namespace std { @@ -51,9 +49,6 @@ std::ostream& operator<<(std::ostream& os, const std::vector& vec) } } // namespace std - - - int main(int argc, char** argv) { std::cout << skycleaver_splash; @@ -62,7 +57,6 @@ int main(int argc, char** argv) skyweaver::SkyCleaverConfig config; - namespace po = boost::program_options; po::options_description generic("Generic options"); @@ -71,158 +65,143 @@ int main(int argc, char** argv) "Skycleaver configuration file"); po::options_description main_options("Main options"); - main_options.add_options() - ("help,h", "Produce help message") - ("root-dir,r", - po::value() - -> required() - ->notifier( - [&config](std::string key) { config.root_dir(key); }), - "The output directory for all results") - ("output-dir", - po::value() - ->default_value(config.output_dir()) - ->notifier( - [&config](std::string key) { config.output_dir(key); }), - "The output directory for all results") - ("root-prefix", - po::value() - ->default_value(config.root_prefix()) - ->notifier( - [&config](std::string key) { config.root_prefix(key); }), - "The prefix for all output files") - ("out-prefix", - po::value() - ->default_value(config.out_prefix()) - ->notifier( - [&config](std::string key) { config.out_prefix(key); }), - "The prefix for all output files") - ("nthreads", - po::value() - ->default_value(config.nthreads()) - ->notifier( - [&config](unsigned int key) { config.nthreads(key); }), - "The number of threads to use for processing") - ("nsamples-per-block", - po::value() - ->default_value(config.nsamples_per_block()) - ->notifier( - [&config](std::size_t key) { config.nsamples_per_block(key); }), - "The number of samples per block") - ("nchans", - po::value() - ->default_value(config.nchans()) - ->notifier( - [&config](std::size_t key) { config.nchans(key); }), - "The number of channels") - ("nbridges", - po::value() - ->default_value(config.nbridges()) - ->notifier( - [&config](std::size_t key) { config.nbridges(key); }), - "The number of bridges") - ("nbeams", - po::value() - ->default_value(config.nbeams()) - ->notifier( - [&config](std::size_t key) { config.nbeams(key); }), - "The number of beams") - ("ndms", - po::value() - ->default_value(config.ndms()) - ->notifier( - [&config](std::size_t key) { config.ndms(key); }), - "The number of DMs") - ("stokes-mode", - po::value() - ->default_value(config.stokes_mode()) - ->notifier( - [&config](std::string key) { config.stokes_mode(key); }), - "The stokes mode") - ("stream-id", - po::value() - ->default_value(config.stream_id()) - ->notifier( - [&config](std::size_t key) { config.stream_id(key); }), - "The stream id") - ("max-ram-gb", - po::value() - ->default_value(config.max_ram_gb()) - ->notifier( - [&config](std::size_t key) { config.max_ram_gb(key); }), - "The maximum amount of RAM to use in GB") - ("max-output-filesize", - po::value() - ->default_value(config.max_output_filesize()) - ->notifier( - [&config](std::size_t key) { config.max_output_filesize(key); }), - "The maximum output file size in bytes") - ("dada-header-size", - po::value() - ->default_value(config.dada_header_size()) - ->notifier( - [&config](std::size_t key) { config.dada_header_size(key); }), - "The size of the DADA header") - ("log-level", - po::value()->default_value("info")->notifier( - [](std::string level) { skyweaver::init_logging(level); }), - "The logging level to use (debug, info, warning, error)"); - - po::options_description cmdline_options; - cmdline_options.add(generic).add(main_options); - - // set options allowed in config file - po::options_description config_file_options; - config_file_options.add(main_options); - po::variables_map variable_map; - try { - po::store(po::command_line_parser(argc, argv) - .options(cmdline_options) - .run(), - variable_map); - if(variable_map.count("help")) { - std::cout << "skycleaver -- A pipeline that cleaves input TDB files, " - "and cleaves them to form output Sigproc Filterbank files." - << std::endl - << cmdline_options << std::endl; - return SUCCESS; - } - } catch(po::error& e) { - std::cerr << "ERROR: " << e.what() << std::endl << std::endl; - return ERROR_IN_COMMAND_LINE; + main_options.add_options()("help,h", "Produce help message")( + "root-dir,r", + po::value()->required()->notifier( + [&config](std::string key) { config.root_dir(key); }), + "The output directory for all results")( + "output-dir", + po::value() + ->default_value(config.output_dir()) + ->notifier([&config](std::string key) { config.output_dir(key); }), + "The output directory for all results")( + "root-prefix", + po::value() + ->default_value(config.root_prefix()) + ->notifier([&config](std::string key) { config.root_prefix(key); }), + "The prefix for all output files")( + "out-prefix", + po::value() + ->default_value(config.out_prefix()) + ->notifier([&config](std::string key) { config.out_prefix(key); }), + "The prefix for all output files")( + "nthreads", + po::value() + ->default_value(config.nthreads()) + ->notifier([&config](unsigned int key) { config.nthreads(key); }), + "The number of threads to use for processing")( + "nsamples-per-block", + po::value() + ->default_value(config.nsamples_per_block()) + ->notifier( + [&config](std::size_t key) { config.nsamples_per_block(key); }), + "The number of samples per block")( + "nchans", + po::value() + ->default_value(config.nchans()) + ->notifier([&config](std::size_t key) { config.nchans(key); }), + "The number of channels")( + "nbridges", + po::value() + ->default_value(config.nbridges()) + ->notifier([&config](std::size_t key) { config.nbridges(key); }), + "The number of bridges")( + "nbeams", + po::value() + ->default_value(config.nbeams()) + ->notifier([&config](std::size_t key) { config.nbeams(key); }), + "The number of beams")( + "ndms", + po::value() + ->default_value(config.ndms()) + ->notifier([&config](std::size_t key) { config.ndms(key); }), + "The number of DMs")( + "stokes-mode", + po::value() + ->default_value(config.stokes_mode()) + ->notifier([&config](std::string key) { config.stokes_mode(key); }), + "The stokes mode")( + "stream-id", + po::value() + ->default_value(config.stream_id()) + ->notifier([&config](std::size_t key) { config.stream_id(key); }), + "The stream id")( + "max-ram-gb", + po::value() + ->default_value(config.max_ram_gb()) + ->notifier([&config](std::size_t key) { config.max_ram_gb(key); }), + "The maximum amount of RAM to use in GB")( + "max-output-filesize", + po::value() + ->default_value(config.max_output_filesize()) + ->notifier([&config](std::size_t key) { + config.max_output_filesize(key); + }), + "The maximum output file size in bytes")( + "dada-header-size", + po::value() + ->default_value(config.dada_header_size()) + ->notifier( + [&config](std::size_t key) { config.dada_header_size(key); }), + "The size of the DADA header")( + "log-level", + po::value()->default_value("info")->notifier( + [](std::string level) { skyweaver::init_logging(level); }), + "The logging level to use (debug, info, warning, error)"); + + po::options_description cmdline_options; + cmdline_options.add(generic).add(main_options); + + // set options allowed in config file + po::options_description config_file_options; + config_file_options.add(main_options); + po::variables_map variable_map; + try { + po::store( + po::command_line_parser(argc, argv).options(cmdline_options).run(), + variable_map); + if(variable_map.count("help")) { + std::cout + << "skycleaver -- A pipeline that cleaves input TDB files, " + "and cleaves them to form output Sigproc Filterbank files." + << std::endl + << cmdline_options << std::endl; + return SUCCESS; } - - auto config_file = variable_map.at("cfg").as(); - - if(config_file != "") { - std::ifstream config_fs(config_file.c_str()); - if(!config_fs.is_open()) { - std::cerr << "Unable to open configuration file: " - << config_file << " (" << std::strerror(errno) - << ")\n"; - return ERROR_UNHANDLED_EXCEPTION; - } else { - po::store(po::parse_config_file(config_fs, config_file_options), - variable_map); - } + } catch(po::error& e) { + std::cerr << "ERROR: " << e.what() << std::endl << std::endl; + return ERROR_IN_COMMAND_LINE; + } + + auto config_file = variable_map.at("cfg").as(); + + if(config_file != "") { + std::ifstream config_fs(config_file.c_str()); + if(!config_fs.is_open()) { + std::cerr << "Unable to open configuration file: " << config_file + << " (" << std::strerror(errno) << ")\n"; + return ERROR_UNHANDLED_EXCEPTION; + } else { + po::store(po::parse_config_file(config_fs, config_file_options), + variable_map); } - po::notify(variable_map); - BOOST_LOG_NAMED_SCOPE("skycleaver_cli"); - BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; - BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); - BOOST_LOG_TRIVIAL(info) << "output_dir: " << config.output_dir(); - BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); - BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); - BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); - BOOST_LOG_TRIVIAL(info) << "nsamples_per_block: " << config.nsamples_per_block(); - BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); - BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); - BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); - BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); - - - skyweaver::SkyCleaver skycleaver(config); - skycleaver.cleave(); - - + } + po::notify(variable_map); + BOOST_LOG_NAMED_SCOPE("skycleaver_cli"); + BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; + BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); + BOOST_LOG_TRIVIAL(info) << "output_dir: " << config.output_dir(); + BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); + BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); + BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); + BOOST_LOG_TRIVIAL(info) + << "nsamples_per_block: " << config.nsamples_per_block(); + BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); + BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); + BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); + BOOST_LOG_TRIVIAL(info) + << "max_output_filesize: " << config.max_output_filesize(); + + skyweaver::SkyCleaver skycleaver(config); + skycleaver.cleave(); } \ No newline at end of file diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 3092164..8204ca3 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -52,7 +52,7 @@ class NullHandler { public: template - void init(Args... args){}; + void init(Args... args) {}; template bool operator()(Args... args) @@ -263,19 +263,22 @@ void setup_pipeline(skyweaver::PipelineConfig& config) IBWriterType ib_handler(config, "ib", create_stream_callback_ib); using StatsWriterType = - skyweaver::MultiFileWriter>; + skyweaver::MultiFileWriter>; typename StatsWriterType::CreateStreamCallBackType create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; - StatsWriterType stats_handler(config, "stats", create_stream_callback_stats); - + StatsWriterType stats_handler(config, + "stats", + create_stream_callback_stats); if constexpr(enable_incoherent_dedispersion) { - using CBWriterType = skyweaver::MultiFileWriter>; + using CBWriterType = + skyweaver::MultiFileWriter>; typename CBWriterType::CreateStreamCallBackType - create_stream_callback_cb = - skyweaver::detail::create_dada_file_stream>; + create_stream_callback_cb = + skyweaver::detail::create_dada_file_stream< + skyweaver::TDBPowersH>; skyweaver::MultiFileWriter> cb_file_writer(config, "cb", create_stream_callback_cb); skyweaver::IncoherentDedispersionPipeline>; + using CBWriterType = + skyweaver::MultiFileWriter>; typename CBWriterType::CreateStreamCallBackType - create_stream_callback_cb = - skyweaver::detail::create_dada_file_stream>; - CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb); + create_stream_callback_cb = + skyweaver::detail::create_dada_file_stream< + skyweaver::TFBPowersD>; + CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb); skyweaver::BeamformerPipeline, true>(config); } else if(config.stokes_mode() == "QU") { - setup_pipeline, - true>(config); + setup_pipeline< + skyweaver::StokesTraits, + true>(config); } else if(config.stokes_mode() == "IV") { - setup_pipeline, - true>(config); + setup_pipeline< + skyweaver::StokesTraits, + true>(config); } else if(config.stokes_mode() == "IQUV") { setup_pipeline( config); @@ -586,13 +593,15 @@ int main(int argc, char** argv) skyweaver::StokesParameter::V>, false>(config); } else if(config.stokes_mode() == "QU") { - setup_pipeline, - false>(config); + setup_pipeline< + skyweaver::StokesTraits, + false>(config); } else if(config.stokes_mode() == "IV") { - setup_pipeline, - false>(config); + setup_pipeline< + skyweaver::StokesTraits, + false>(config); } else if(config.stokes_mode() == "IQUV") { setup_pipeline( config); From 6a331081fdbb4744fe8e6d2f566da8f64b151e96 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 26 Sep 2024 16:02:57 +0000 Subject: [PATCH 22/69] Add .tmp suffix to output filenames, and remove it when they are closed. --- cpp/skyweaver/FileOutputStream.hpp | 2 ++ cpp/skyweaver/src/FileOutputStream.cpp | 29 +++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cpp/skyweaver/FileOutputStream.hpp b/cpp/skyweaver/FileOutputStream.hpp index 53dc988..968cc92 100644 --- a/cpp/skyweaver/FileOutputStream.hpp +++ b/cpp/skyweaver/FileOutputStream.hpp @@ -48,6 +48,8 @@ class FileOutputStream std::string _full_path; std::size_t _bytes_requested; std::size_t _bytes_written; + std::string _temporary_suffix; + std::string _temporary_path; std::ofstream _stream; }; diff --git a/cpp/skyweaver/src/FileOutputStream.cpp b/cpp/skyweaver/src/FileOutputStream.cpp index f4d2ffa..40ca0ad 100644 --- a/cpp/skyweaver/src/FileOutputStream.cpp +++ b/cpp/skyweaver/src/FileOutputStream.cpp @@ -38,15 +38,16 @@ void create_directories(const fs::path& path) } FileOutputStream::File::File(std::string const& fname, std::size_t bytes) - : _full_path(fname), _bytes_requested(bytes), _bytes_written(0) + : _full_path(fname), _bytes_requested(bytes), _bytes_written(0), _temporary_suffix(".tmp") { + _temporary_path = _full_path + _temporary_suffix; _stream.exceptions(std::ofstream::failbit | std::ofstream::badbit); - _stream.open(_full_path, std::ofstream::out | std::ofstream::binary); + _stream.open(_temporary_path, std::ofstream::out | std::ofstream::binary); if(_stream.is_open()) { - BOOST_LOG_TRIVIAL(info) << "Opened output file " << _full_path; + BOOST_LOG_TRIVIAL(info) << "Opened output file " << _temporary_path; } else { std::stringstream error_message; - error_message << "Could not open file " << _full_path; + error_message << "Could not open file " << _temporary_path; BOOST_LOG_TRIVIAL(error) << error_message.str(); throw std::runtime_error(error_message.str()); } @@ -55,15 +56,29 @@ FileOutputStream::File::File(std::string const& fname, std::size_t bytes) FileOutputStream::File::~File() { if(_stream.is_open()) { - BOOST_LOG_TRIVIAL(info) << "Closing file " << _full_path; + BOOST_LOG_TRIVIAL(info) << "Closing file " << _temporary_path; _stream.close(); } + BOOST_LOG_TRIVIAL(info) << "Renaming file " << _temporary_path + << " to " << _full_path; + int res = std::rename(_temporary_path.c_str(), _full_path.c_str()); + + // Can't throw an exception from a destructor, + // but we'll at least put a warning in the log! + if (res) + { + std::stringstream error_message; + error_message << "Error renaming file " << _temporary_path + << " to " << _full_path + << " (" << res << ")."; + BOOST_LOG_TRIVIAL(error) << error_message.str(); + } } std::size_t FileOutputStream::File::write(char const* ptr, std::size_t bytes) { BOOST_LOG_TRIVIAL(debug) - << "Writing " << bytes << " bytes to " << _full_path; + << "Writing " << bytes << " bytes to " << _temporary_path; std::size_t bytes_remaining = _bytes_requested - _bytes_written; try { if(bytes > bytes_remaining) { @@ -89,7 +104,7 @@ std::size_t FileOutputStream::File::write(char const* ptr, std::size_t bytes) reason = "eofbit set."; } - BOOST_LOG_TRIVIAL(error) << "Error while writing to " << _full_path + BOOST_LOG_TRIVIAL(error) << "Error while writing to " << _temporary_path << " (" << e.what() << ") because of reason: " << reason; throw; From 456719ad1891e5f1d13240428512ac5f489979ca Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Mon, 30 Sep 2024 16:40:12 +0200 Subject: [PATCH 23/69] cleave only required samples --- cpp/skyweaver/SkyCleaverConfig.hpp | 13 ++++++++++++- cpp/skyweaver/src/SkyCleaver.cu | 27 +++++++++++++++++++++------ cpp/skyweaver/src/skycleaver_cli.cu | 21 ++++++++++++++++----- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index feadd9b..1972812 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -20,13 +20,16 @@ class SkyCleaverConfig std::size_t _ndms; std::string _stokes_mode; std::size_t _dada_header_size; + std::size_t _start_sample; + std::size_t _nsamples_to_read; public: SkyCleaverConfig() : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), - _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096) + _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096), + _start_sample(0), _nsamples_to_read(0) { } SkyCleaverConfig(SkyCleaverConfig const&) = delete; @@ -55,6 +58,11 @@ class SkyCleaverConfig { _dada_header_size = dada_header_size; } + void start_sample(std::size_t start_sample) { _start_sample = start_sample; } + void nsamples_to_read(std::size_t nsamples_to_read) + { + _nsamples_to_read = nsamples_to_read; + } std::string output_dir() const { return _output_dir; } std::string root_dir() const { return _root_dir; } @@ -71,6 +79,9 @@ class SkyCleaverConfig std::size_t ndms() const { return _ndms; } std::string stokes_mode() const { return _stokes_mode; } std::size_t dada_header_size() const { return _dada_header_size; } + std::size_t start_sample() const { return _start_sample; } + std::size_t nsamples_to_read() const { return _nsamples_to_read; } + }; } // namespace skyweaver #endif // SKYCLEAVERCONFIG_HPP \ No newline at end of file diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index bf195cd..b645414 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -151,7 +151,6 @@ void SkyCleaver::init_readers() long double obs_centre_freq = header.obs_frequency; long double obs_bandwidth = header.obs_bandwidth; - std::size_t obs_nchans = header.obs_nchans; long double start_freq = obs_centre_freq - obs_bandwidth / 2; @@ -159,7 +158,7 @@ void SkyCleaver::init_readers() for(int i = 0; i < nbridges; i++) { - int ifreq = std::lround( std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); + int ifreq = std::lround(std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) << "Expected frequency [" << i << "]: " << ifreq; @@ -224,9 +223,6 @@ void SkyCleaver::init_readers() } } - BOOST_LOG_TRIVIAL(debug) << "Smallest data size: " << smallest_data_size; - BOOST_LOG_TRIVIAL(debug) << "ndm: " << _config.ndms(); - BOOST_LOG_TRIVIAL(debug) << "nbeams: " << _config.nbeams(); if(smallest_data_size % (_config.ndms() * _config.nbeams()) != 0) { std::runtime_error("Data size is not a multiple of ndms * nbeams"); @@ -235,6 +231,12 @@ void SkyCleaver::init_readers() std::size_t smallest_nsamples = std::floor(smallest_data_size / _config.ndms() / _config.nbeams()); + if (smallest_nsamples < _config.start_sample()) { + std::runtime_error("start_sample is greater than the smallest_nsamples in the data."); + } + + smallest_nsamples = smallest_nsamples - _config.start_sample(); + BOOST_LOG_TRIVIAL(info) << "Smallest data size: " << smallest_data_size << " Smallest number of samples: " << smallest_nsamples; @@ -244,7 +246,20 @@ void SkyCleaver::init_readers() "Smallest data size is less than nsamples_per_block"); } - _nsamples_to_read = smallest_nsamples; + if(smallest_nsamples < _config.nsamples_to_read()) { + std::runtime_error( + "Smallest data size is less than nsamples_to_read"); + } + + + _nsamples_to_read = _config.nsamples_to_read(); + + std::size_t bytes_seeking = (_config.start_sample() * _config.ndms() * _config.nbeams() * sizeof(InputVectorType::value_type)); + for(const auto& [freq, reader]: _bridge_readers) { + _bridge_readers[freq]->seekg(bytes_seeking, + std::ios_base::beg); + } + BOOST_LOG_TRIVIAL(info) << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cu index fdb9278..d8fc2bd 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cu +++ b/cpp/skyweaver/src/skycleaver_cli.cu @@ -147,7 +147,16 @@ int main(int argc, char** argv) "log-level", po::value()->default_value("info")->notifier( [](std::string level) { skyweaver::init_logging(level); }), - "The logging level to use (debug, info, warning, error)"); + "The logging level to use (debug, info, warning, error)")( + "start_sample", + po::value() + ->default_value(config.start_sample()) + ->notifier([&config](std::size_t key) { config.start_sample(key); }), + "Start from this sample")("nsamples_to_read", + po::value() + ->default_value(config.start_sample()) + ->notifier([&config](std::size_t key) { config.nsamples_to_read(key); }), + "total number of samples to read from start_sample"); po::options_description cmdline_options; cmdline_options.add(generic).add(main_options); @@ -187,6 +196,7 @@ int main(int argc, char** argv) } } po::notify(variable_map); + BOOST_LOG_NAMED_SCOPE("skycleaver_cli"); BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); @@ -194,13 +204,14 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); - BOOST_LOG_TRIVIAL(info) - << "nsamples_per_block: " << config.nsamples_per_block(); + BOOST_LOG_TRIVIAL(info) << "nsamples_per_block: " << config.nsamples_per_block(); BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); + BOOST_LOG_TRIVIAL(info) << "nbridges: " << config.nbridges(); + BOOST_LOG_TRIVIAL(info) << "ndms: " << config.ndms(); BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); - BOOST_LOG_TRIVIAL(info) - << "max_output_filesize: " << config.max_output_filesize(); + BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); + skyweaver::SkyCleaver skycleaver(config); skycleaver.cleave(); From 48adc14b91afa565369e9419abef356f202c0630 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Wed, 2 Oct 2024 15:24:25 +0200 Subject: [PATCH 24/69] cleave only required dms and beams, code review changes --- cmake/compiler_settings.cmake | 4 +- cpp/skyweaver/MultiFileWriter.cuh | 18 ----- cpp/skyweaver/SkyCleaverConfig.hpp | 14 +++- cpp/skyweaver/src/SkyCleaver.cu | 55 +++++++++++++-- cpp/skyweaver/src/skycleaver_cli.cu | 106 +++++++++++++++++++++++++--- 5 files changed, 160 insertions(+), 37 deletions(-) diff --git a/cmake/compiler_settings.cmake b/cmake/compiler_settings.cmake index 879cf70..500e4f0 100644 --- a/cmake/compiler_settings.cmake +++ b/cmake/compiler_settings.cmake @@ -6,9 +6,9 @@ if (NOT CMAKE_BUILD_TYPE) endif () #set(CMAKE_VERBOSE_MAKEFILE 1) - +set (CMAKE_CXX_STANDARD 20) # Set compiler flags -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fopenmp -std=gnu++20") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fopenmp") set(ARCH "broadwell" CACHE STRING "target architecture (-march=native, x86-64), defautls to broadwell") diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 352b2f5..6533d6a 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -37,24 +37,6 @@ struct MultiFileWriterConfig { stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), output_basename("") {}; - MultiFileWriterConfig(MultiFileWriterConfig const& other) - : header_size(other.header_size), max_file_size(other.max_file_size), - stokes_mode(other.stokes_mode), output_dir(other.output_dir), - base_output_dir(other.base_output_dir), prefix(other.prefix), - extension(other.extension), output_basename(other.output_basename) {}; - - MultiFileWriterConfig& operator=(MultiFileWriterConfig const& other) - { - header_size = other.header_size; - max_file_size = other.max_file_size; - stokes_mode = other.stokes_mode; - output_dir = other.output_dir; - prefix = other.prefix; - extension = other.extension; - output_basename = other.output_basename; - base_output_dir = other.base_output_dir; - return *this; - } std::string to_string() { diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index 1972812..8b77dcc 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -22,6 +22,9 @@ class SkyCleaverConfig std::size_t _dada_header_size; std::size_t _start_sample; std::size_t _nsamples_to_read; + std::vector _required_beams; + std::vector _required_dms; + public: SkyCleaverConfig() @@ -29,7 +32,7 @@ class SkyCleaverConfig _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096), - _start_sample(0), _nsamples_to_read(0) + _start_sample(0), _nsamples_to_read(0), _required_beams({}), _required_dms({}) { } SkyCleaverConfig(SkyCleaverConfig const&) = delete; @@ -63,6 +66,11 @@ class SkyCleaverConfig { _nsamples_to_read = nsamples_to_read; } + void required_beams(std::vector required_beams) { _required_beams = required_beams; } + void required_dms(std::vector required_dms) { _required_dms = required_dms; } + + + std::string output_dir() const { return _output_dir; } std::string root_dir() const { return _root_dir; } @@ -81,6 +89,10 @@ class SkyCleaverConfig std::size_t dada_header_size() const { return _dada_header_size; } std::size_t start_sample() const { return _start_sample; } std::size_t nsamples_to_read() const { return _nsamples_to_read; } + std::vector required_beams() const { return _required_beams; } + std::vector required_dms() const { return _required_dms; } + + }; } // namespace skyweaver diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index b645414..6655ff1 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "skyweaver/types.cuh" #include "skyweaver/SkyCleaver.cuh" @@ -246,13 +247,19 @@ void SkyCleaver::init_readers() "Smallest data size is less than nsamples_per_block"); } - if(smallest_nsamples < _config.nsamples_to_read()) { - std::runtime_error( - "Smallest data size is less than nsamples_to_read"); - } - + if(_config.nsamples_to_read() > 0) { + + if(smallest_nsamples < _config.nsamples_to_read()) { + std::runtime_error( + "Smallest data size is less than nsamples_to_read"); + } + - _nsamples_to_read = _config.nsamples_to_read(); + _nsamples_to_read = _config.nsamples_to_read(); + } + else{ + _nsamples_to_read = smallest_nsamples; + } std::size_t bytes_seeking = (_config.start_sample() * _config.ndms() * _config.nbeams() * sizeof(InputVectorType::value_type)); for(const auto& [freq, reader]: _bridge_readers) { @@ -275,6 +282,8 @@ void SkyCleaver::init_writers() BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_writers") BOOST_LOG_TRIVIAL(debug) << "_config.output_dir(); " << _config.output_dir(); + + if(!fs::exists(_config.output_dir())) { fs::create_directories(_config.output_dir()); } @@ -282,6 +291,7 @@ void SkyCleaver::init_writers() ? "" : _config.out_prefix() + "_"; std::string output_dir = _config.output_dir(); + for(int idm = 0; idm < _config.ndms(); idm++) { @@ -289,8 +299,26 @@ void SkyCleaver::init_writers() std::string prefix = _config.ndms() > 1 ? out_prefix + "idm_" + to_string_with_padding(_header.dms[idm], 9, 3) + "_": out_prefix; + if(! _config.required_dms().empty()) { + const auto& required_dms = _config.required_dms(); + if(std::ranges::find(required_dms, _header.dms[idm]) == required_dms.end()) { + BOOST_LOG_TRIVIAL(info) + << "DM " << _header.dms[idm] << " is not required, skipping from writing"; + continue; + } + } for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + // skip if beam is not used + if (! _config.required_beams().empty()) { + const auto& required_beams = _config.required_beams(); + std::cerr << "required_beams: " << required_beams.size() << "ibeam: " << ibeam << std::endl; + if(std::ranges::find(required_beams, ibeam) == required_beams.end()) { + BOOST_LOG_TRIVIAL(info) + << "Beam " << ibeam << " is not required, skipping from writing"; + continue; + } + } MultiFileWriterConfig writer_config; writer_config.header_size = _config.dada_header_size(); writer_config.max_file_size = _config.max_output_filesize(); @@ -412,6 +440,14 @@ void SkyCleaver::cleave() #pragma omp parallel for schedule(static) collapse(2) for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { for(std::size_t idm = 0; idm < ndms; idm++) { + // _beam_data[idm][ibeam] is not found, skip + if(_beam_data.find(idm) == _beam_data.end()) { + continue; + } + if(_beam_data[idm].find(ibeam) == _beam_data[idm].end()) { + continue; + } + #pragma omp simd for(std::size_t isample = 0; isample < nsamples_per_block; isample++) { const std::size_t base_index = @@ -436,6 +472,13 @@ void SkyCleaver::cleave() #pragma omp parallel for schedule(static) collapse(2) for(int idm = 0; idm < _config.ndms(); idm++) { for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + + if(_beam_data.find(idm) == _beam_data.end()) { + continue; + } + if(_beam_data[idm].find(ibeam) == _beam_data[idm].end()) { + continue; + } _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], _config.stream_id()); } diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cu index d8fc2bd..0b56bac 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cu +++ b/cpp/skyweaver/src/skycleaver_cli.cu @@ -19,6 +19,10 @@ #include #include #include +#include +#include +#include + #define BOOST_LOG_DYN_LINK 1 namespace @@ -38,6 +42,53 @@ const size_t ERROR_UNHANDLED_EXCEPTION = 2; const char* build_time = __DATE__ " " __TIME__; + +template +std::vector +get_list_from_string(const std::string& value, + T epsilon = std::numeric_limits::epsilon()) +{ + std::vector output; + std::vector comma_chunks; + + // Split the input string by commas + std::stringstream ss(value); + std::string token; + while(std::getline(ss, token, ',')) { comma_chunks.push_back(token); } + + for(const auto& comma_chunk: comma_chunks) { + // Check if the chunk contains a colon (indicating a range) + if(comma_chunk.find(':') == std::string::npos) { + output.push_back(static_cast(std::atof(comma_chunk.c_str()))); + continue; + } + + // Split the range chunk by colons + std::stringstream ss_chunk(comma_chunk); + std::vector colon_chunks; + std::string colon_token; + while(std::getline(ss_chunk, colon_token, ':')) { + colon_chunks.push_back( + static_cast(std::atof(colon_token.c_str()))); + } + + // Determine the step size + T step = colon_chunks.size() == 3 ? colon_chunks[2] : static_cast(1); + T start = colon_chunks[0]; + T stop = colon_chunks[1]; + + // Loop and add values to the output vector + if constexpr(std::is_floating_point::value) { + for(T k = start; k <= stop + epsilon; k += step) { + output.push_back(k); + } + } else { + for(T k = start; k <= stop; k += step) { output.push_back(k); } + } + } + return output; +} + } // namespace namespace std @@ -149,14 +200,33 @@ int main(int argc, char** argv) [](std::string level) { skyweaver::init_logging(level); }), "The logging level to use (debug, info, warning, error)")( "start_sample", - po::value() - ->default_value(config.start_sample()) - ->notifier([&config](std::size_t key) { config.start_sample(key); }), - "Start from this sample")("nsamples_to_read", - po::value() - ->default_value(config.start_sample()) - ->notifier([&config](std::size_t key) { config.nsamples_to_read(key); }), - "total number of samples to read from start_sample"); + po::value() + ->default_value(config.start_sample()) + ->notifier( + [&config](std::size_t key) { config.start_sample(key); }), + "Start from this sample")( + "nsamples_to_read", + po::value() + ->default_value(config.start_sample()) + ->notifier( + [&config](std::size_t key) { config.nsamples_to_read(key); }), + "total number of samples to read from start_sample")( + "required_beams", + po::value() + ->default_value("") + ->notifier([&config](std::string key) { + config.required_beams(get_list_from_string(key)); + }), + "Comma separated list of beams to process. Syntax - beam1, beam2, " + "beam3:beam4:step, beam5 etc..")( + "required_dms", + po::value() + ->default_value("") + ->notifier([&config](std::string key) { + config.required_dms(get_list_from_string(key)); + }), + "Comma separated list of DMs to process. Syntax - dm1, dm2, " + "dm1:dm2:step, etc.."); po::options_description cmdline_options; cmdline_options.add(generic).add(main_options); @@ -204,13 +274,29 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); - BOOST_LOG_TRIVIAL(info) << "nsamples_per_block: " << config.nsamples_per_block(); + BOOST_LOG_TRIVIAL(info) + << "nsamples_per_block: " << config.nsamples_per_block(); BOOST_LOG_TRIVIAL(info) << "nchans: " << config.nchans(); BOOST_LOG_TRIVIAL(info) << "nbeams: " << config.nbeams(); BOOST_LOG_TRIVIAL(info) << "nbridges: " << config.nbridges(); BOOST_LOG_TRIVIAL(info) << "ndms: " << config.ndms(); BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); - BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); + BOOST_LOG_TRIVIAL(info) + << "max_output_filesize: " << config.max_output_filesize(); + BOOST_LOG_TRIVIAL(info) << "dada_header_size: " << config.dada_header_size(); + BOOST_LOG_TRIVIAL(info) << "start_sample: " << config.start_sample(); + BOOST_LOG_TRIVIAL(info) << "nsamples_to_read: " << config.nsamples_to_read(); + if(config.required_beams().size() > 0) { + for (auto beam : config.required_beams()) { + BOOST_LOG_TRIVIAL(info) << "required_beam: " << beam; + } + } + if(config.required_dms().size() > 0) { + for (auto dm : config.required_dms()) { + BOOST_LOG_TRIVIAL(info) << "required_dm: " << dm; + } + } + skyweaver::SkyCleaver skycleaver(config); From 0cfe1f91b9348bbd073c21022a51dcf9b14f3cc5 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 3 Oct 2024 16:00:24 +0200 Subject: [PATCH 25/69] seek to start sample after dedisp delay correction --- cpp/skyweaver/src/SkyCleaver.cu | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cu index 6655ff1..4f73cce 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cu @@ -261,13 +261,19 @@ void SkyCleaver::init_readers() _nsamples_to_read = smallest_nsamples; } - std::size_t bytes_seeking = (_config.start_sample() * _config.ndms() * _config.nbeams() * sizeof(InputVectorType::value_type)); - for(const auto& [freq, reader]: _bridge_readers) { - _bridge_readers[freq]->seekg(bytes_seeking, - std::ios_base::beg); + std::size_t bytes_seeking = + (_config.start_sample() * _config.ndms() * _config.nbeams() * + sizeof(InputVectorType::value_type)); + + if(bytes_seeking > 0) { + BOOST_LOG_TRIVIAL(info) << "Seeking " << bytes_seeking + << " bytes in bridge readers to start sample: " + << _config.start_sample(); + for(const auto& [freq, reader]: _bridge_readers) { + _bridge_readers[freq]->seekg(bytes_seeking, std::ios_base::cur); + } } - BOOST_LOG_TRIVIAL(info) << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; @@ -371,6 +377,8 @@ void SkyCleaver::cleave() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::cleave") + /** read data **/ + for(std::size_t nsamples_read = 0; nsamples_read < _nsamples_to_read; nsamples_read += _config.nsamples_per_block()) { std::size_t gulp_samples = @@ -430,6 +438,8 @@ void SkyCleaver::cleave() _timer.stop("skyweaver::read_data"); _timer.start("skyweaver::process_data"); + /** Process data **/ + omp_set_num_threads(_config.nthreads()); std::size_t nbridges = _config.nbridges(); @@ -467,12 +477,15 @@ void SkyCleaver::cleave() BOOST_LOG_TRIVIAL(info) << "Processed data"; _timer.stop("skyweaver::process_data"); + /** Write data **/ + + _timer.start("skyweaver::write_data"); #pragma omp parallel for schedule(static) collapse(2) for(int idm = 0; idm < _config.ndms(); idm++) { for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - + if(_beam_data.find(idm) == _beam_data.end()) { continue; } From ee35ae22cf5b982108cc11982b0cec28773e4252 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 3 Oct 2024 16:54:25 +0200 Subject: [PATCH 26/69] CDD all data including overlap --- cpp/skyweaver/src/CoherentDedisperser.cu | 118 ++++++++++++++++++----- 1 file changed, 92 insertions(+), 26 deletions(-) diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 12d9976..73c9f27 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -13,6 +13,49 @@ #include #include #include + + +namespace{ + +#define NCHANS_PER_BLOCK 128 + +// Function to check if a number's prime factors are only 2, 3, 5, or +bool has_small_prime_factors(size_t n) { + if (n == 0) return false; + while (n % 2 == 0) n /= 2; + while (n % 3 == 0) n /= 3; + while (n % 5 == 0) n /= 5; + while (n % 7 == 0) n /= 7; + return n == 1; +} + +// Function to find the next optimal FFT size +size_t next_optimal_fft_size(size_t N) { + size_t n = N; + while (!has_small_prime_factors(n)) { + n++; + } + return n; +} + +// Function to compute padding amount +size_t compute_padding(size_t N) { + size_t optimal_size = 0; + size_t n = N-1; + + do{ + n = n+1; + optimal_size = next_optimal_fft_size(n); + BOOST_LOG_TRIVIAL(info) << "Trying optimal size: " << optimal_size; + } while (optimal_size % NCHANS_PER_BLOCK != 0); + return optimal_size - N; +} + + + +} + + namespace skyweaver { @@ -45,7 +88,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, "Gulp length must be at least 2 times the maximum DM delay"); } - if(max_dm_delay_samps %2 !=0) { + if((pipeline_config.gulp_length_samps() + max_dm_delay_samps) %2 !=0) { max_dm_delay_samps++; } @@ -78,7 +121,6 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, std::vector dms) { config.gulp_samps = gulp_samps; - config.overlap_samps = overlap_samps; config.num_coarse_chans = num_coarse_chans; config.npols = npols; config.nantennas = nantennas; @@ -88,9 +130,13 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, config.high_freq = low_freq + bw; config.coarse_chan_bw = bw / num_coarse_chans; config.filter_delay = tsamp * overlap_samps / 2.0; - BOOST_LOG_TRIVIAL(warning) << "tsamp in create_coherent_dedisperser_config: " << config.tsamp; - BOOST_LOG_TRIVIAL(warning) << "overlap_samps in create_coherent_dedisperser_config: " << config.overlap_samps; - BOOST_LOG_TRIVIAL(warning) << "Filter delay: " << config.filter_delay; + BOOST_LOG_TRIVIAL(debug) << "tsamp in create_coherent_dedisperser_config: " << config.tsamp; + BOOST_LOG_TRIVIAL(info) << "overlap_samps just due to dedispersion: " << overlap_samps; + BOOST_LOG_TRIVIAL(info) << "Filter delay: " << config.filter_delay; + + config.overlap_samps = compute_padding(gulp_samps + overlap_samps) + overlap_samps; + + BOOST_LOG_TRIVIAL(info) << "overlap_samps after padding for optimal FFT size " << config.overlap_samps; /* Precompute DM constants */ @@ -99,7 +145,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, config._d_dm_prefactor.resize(dms.size()); config._d_ism_responses.resize(dms.size()); for(int i = 0; i < dms.size(); i++) { - config._d_ism_responses[i].resize(num_coarse_chans * gulp_samps); + config._d_ism_responses[i].resize(num_coarse_chans * (config.gulp_samps + config.overlap_samps)); } thrust::transform(config._d_dms.begin(), @@ -147,10 +193,7 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, } -namespace -{ -#define NCHANS_PER_BLOCK 128 -} // namespace + void CoherentDedisperser::dedisperse( TPAVoltagesD const& d_tpa_voltages_in, @@ -159,7 +202,8 @@ void CoherentDedisperser::dedisperse( unsigned int dm_idx) { BOOST_LOG_NAMED_SCOPE("CoherentDedisperser::dedisperse"); - _d_fpa_spectra.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); + //d_tpa_voltages_in.size() is with overlap + _d_fpa_spectra.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); _d_tpa_voltages_in_cufft.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); _d_tpa_voltages_dedispersed.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); @@ -216,6 +260,10 @@ void CoherentDedisperser::dedisperse( BOOST_LOG_TRIVIAL(debug) << "copying from input from " << discard_size << " to " << _d_tpa_voltages_dedispersed.size() - discard_size; + BOOST_LOG_TRIVIAL(debug) << "Total elements copied: " + << _d_tpa_voltages_dedispersed.size() - 2 * discard_size; + BOOST_LOG_TRIVIAL(debug) << "Remaining space in output: " + << d_ftpa_voltages_out.size() - out_offset; BOOST_LOG_TRIVIAL(debug) << "copying to output from " << out_offset << " to " << out_offset + _d_tpa_voltages_dedispersed.size() - 2 * discard_size; @@ -223,8 +271,8 @@ void CoherentDedisperser::dedisperse( std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; + - // transform: divide by d_tpa_voltages_in.size() thrust::transform(_d_tpa_voltages_dedispersed.begin() + discard_size, _d_tpa_voltages_dedispersed.end() - discard_size, d_ftpa_voltages_out.begin() + out_offset, @@ -236,6 +284,8 @@ void CoherentDedisperser::dedisperse( static_cast(__float2int_rn(val.y / fft_size)); return char2_val; }); + + BOOST_LOG_TRIVIAL(debug) << "Transformed dedispersed voltages to char2"; d_ftpa_voltages_out.reference_dm(_config._h_dms[dm_idx]); } @@ -246,15 +296,22 @@ void CoherentDedisperser::multiply_by_chirp( unsigned int freq_idx, unsigned int dm_idx) { - std::size_t total_chans = _config._d_ism_responses[dm_idx].size(); - std::size_t response_offset = freq_idx * _config.gulp_samps; + std::size_t total_chans = _config._d_ism_responses[dm_idx].size(); // all coarse + fine chans + std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; // ONLY FINE CHANS + std::size_t response_offset = freq_idx * fft_size; BOOST_LOG_TRIVIAL(debug) << "Freq idx: " << freq_idx; BOOST_LOG_TRIVIAL(debug) << "_config.gulp_samps: " << _config.gulp_samps; + BOOST_LOG_TRIVIAL(debug) << "_config.overlap_samps: " << _config.overlap_samps; BOOST_LOG_TRIVIAL(debug) << "response_offset: " << response_offset; - - dim3 blockSize(_config.nantennas * _config.npols); - dim3 gridSize(_config.gulp_samps / NCHANS_PER_BLOCK); + BOOST_LOG_TRIVIAL(debug) << "total_chans: " << total_chans; + BOOST_LOG_TRIVIAL(debug) << "chirp multiply input size: " + << _d_fpa_spectra_in.size(); + BOOST_LOG_TRIVIAL(debug) << "chirp multiply output size: " + << _d_fpa_spectra_out.size(); + + dim3 blockSize(_config.nantennas * _config.npols); + dim3 gridSize(fft_size / NCHANS_PER_BLOCK); kernels::dedisperse<<>>( thrust::raw_pointer_cast(_config._d_ism_responses[dm_idx].data() + response_offset), @@ -273,14 +330,14 @@ __global__ void dedisperse(cufftComplex const* __restrict__ _d_ism_response, cufftComplex* out, unsigned total_chans) { - const unsigned pa_size = blockDim.x; + const unsigned pa_size = blockDim.x; // NANT * NPOL volatile __shared__ cufftComplex response[NCHANS_PER_BLOCK]; - const unsigned block_start_chan_idx = blockIdx.x * NCHANS_PER_BLOCK; + const unsigned block_start_chan_idx = blockIdx.x * NCHANS_PER_BLOCK; // coarse chan idx const unsigned remainder = - min(total_chans - block_start_chan_idx, NCHANS_PER_BLOCK); + min(total_chans - block_start_chan_idx, NCHANS_PER_BLOCK); // how many channels to process for(int idx = threadIdx.x; idx < remainder; idx += pa_size) { cufftComplex const temp = _d_ism_response[block_start_chan_idx + idx]; @@ -331,7 +388,7 @@ struct DMResponse { int fine_chan = tid % num_fine_chans; // fine channel double nu_0 = low_freq + chan * coarse_chan_bw - - 0.5f * coarse_chan_bw; // + fine_chan * fine_chan_bw; + 0.5f * coarse_chan_bw; double nu = fine_chan * fine_chan_bw; // fine_chan_freq @@ -341,7 +398,7 @@ struct DMResponse { cufftDoubleComplex weight; sincos(phase, &weight.y, - &weight.x); // TO DO: test if it is not approximate + &weight.x); // TODO: test if it is not approximate cufftComplex float_weight; float_weight.x = static_cast(weight.x); float_weight.y = static_cast(weight.y); @@ -357,20 +414,29 @@ void get_dm_responses(CoherentDedisperserConfig& config, thrust::device_vector& response) { BOOST_LOG_TRIVIAL(debug) << "Generating DM responses"; - thrust::device_vector indices(config.num_coarse_chans * - config.gulp_samps); - thrust::sequence(indices.begin(), indices.end()); + std::size_t fft_size = config.gulp_samps + config.overlap_samps; + thrust::device_vector indices(config.num_coarse_chans * fft_size); + thrust::sequence(indices.begin(), indices.end()); + + // store raw responses in a temporary variable + thrust::device_vector temp_response(response.size()); + BOOST_LOG_TRIVIAL(warning) << "DOING FFTSHIFT" << std::endl; // Apply the DMResponse functor using thrust's transform thrust::transform(indices.begin(), indices.end(), - response.begin(), + temp_response.begin(), kernels::DMResponse(config.num_coarse_chans, config.gulp_samps, config.low_freq, config.coarse_chan_bw, config.fine_chan_bw, dm_prefactor)); + + // rotate the response to match cufft output + std::size_t shift = fft_size / 2; + thrust::copy(temp_response.begin() + shift, temp_response.end(), response.begin()); + thrust::copy(temp_response.begin(), temp_response.begin() + shift, response.begin() + (fft_size - shift)); } } // namespace skyweaver From 52b1ab31178d821807f105c185df904b5f555439 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Mon, 7 Oct 2024 17:05:19 +0200 Subject: [PATCH 27/69] fix FFT shift, change left over gulps -> fft_size where appropriate --- cpp/skyweaver/src/CoherentDedisperser.cu | 202 ++++++++++++----------- 1 file changed, 108 insertions(+), 94 deletions(-) diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 73c9f27..79fc4be 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -14,47 +14,46 @@ #include #include - -namespace{ +namespace +{ #define NCHANS_PER_BLOCK 128 -// Function to check if a number's prime factors are only 2, 3, 5, or -bool has_small_prime_factors(size_t n) { - if (n == 0) return false; - while (n % 2 == 0) n /= 2; - while (n % 3 == 0) n /= 3; - while (n % 5 == 0) n /= 5; - while (n % 7 == 0) n /= 7; +// Function to check if a number's prime factors are only 2, 3, 5, or +bool has_small_prime_factors(size_t n) +{ + if(n == 0) + return false; + while(n % 2 == 0) n /= 2; + while(n % 3 == 0) n /= 3; + while(n % 5 == 0) n /= 5; + while(n % 7 == 0) n /= 7; return n == 1; } // Function to find the next optimal FFT size -size_t next_optimal_fft_size(size_t N) { +size_t next_optimal_fft_size(size_t N) +{ size_t n = N; - while (!has_small_prime_factors(n)) { - n++; - } + while(!has_small_prime_factors(n)) { n++; } return n; } // Function to compute padding amount -size_t compute_padding(size_t N) { +size_t compute_padding(size_t N) +{ size_t optimal_size = 0; - size_t n = N-1; - - do{ - n = n+1; + size_t n = N - 1; + + do { + n = n + 1; optimal_size = next_optimal_fft_size(n); BOOST_LOG_TRIVIAL(info) << "Trying optimal size: " << optimal_size; - } while (optimal_size % NCHANS_PER_BLOCK != 0); + } while(optimal_size % NCHANS_PER_BLOCK != 0); return optimal_size - N; } - - -} - +} // namespace namespace skyweaver { @@ -62,16 +61,16 @@ namespace skyweaver void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, PipelineConfig const& pipeline_config) { - // the centre frequency and bandwidth are for the bridge. This is taken from Observation Header (not from the user) + // the centre frequency and bandwidth are for the bridge. This is taken from + // Observation Header (not from the user) float f_low = pipeline_config.centre_frequency() - pipeline_config.bandwidth() / 2.0f; - float f_high = f_low + pipeline_config.bandwidth()/pipeline_config.nchans(); - - // pipeline_config.centre_frequency() + pipeline_config.bandwidth() / 2.0f; - float tsamp = pipeline_config.nchans() / pipeline_config.bandwidth(); + float f_high = + f_low + pipeline_config.bandwidth() / pipeline_config.nchans(); - + // pipeline_config.centre_frequency() + pipeline_config.bandwidth() / 2.0f; + float tsamp = pipeline_config.nchans() / pipeline_config.bandwidth(); if(pipeline_config.coherent_dms().empty()) { throw std::runtime_error("No coherent DMs specified"); @@ -81,14 +80,15 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, pipeline_config.coherent_dms().end()); float max_dm = *it; BOOST_LOG_TRIVIAL(debug) << "Constructing coherent dedisperser plan"; - std::size_t max_dm_delay_samps = DMSampleDelay(max_dm, f_low, tsamp)(f_high); + std::size_t max_dm_delay_samps = + DMSampleDelay(max_dm, f_low, tsamp)(f_high); if(max_dm_delay_samps > 2 * pipeline_config.gulp_length_samps()) { throw std::runtime_error( "Gulp length must be at least 2 times the maximum DM delay"); } - if((pipeline_config.gulp_length_samps() + max_dm_delay_samps) %2 !=0) { + if((pipeline_config.gulp_length_samps() + max_dm_delay_samps) % 2 != 0) { max_dm_delay_samps++; } @@ -125,19 +125,23 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, config.npols = npols; config.nantennas = nantennas; config.tsamp = tsamp; - config.low_freq = low_freq; - config.bw = bw; - config.high_freq = low_freq + bw; - config.coarse_chan_bw = bw / num_coarse_chans; - config.filter_delay = tsamp * overlap_samps / 2.0; - BOOST_LOG_TRIVIAL(debug) << "tsamp in create_coherent_dedisperser_config: " << config.tsamp; - BOOST_LOG_TRIVIAL(info) << "overlap_samps just due to dedispersion: " << overlap_samps; + config.low_freq = low_freq; + config.bw = bw; + config.high_freq = low_freq + bw; + config.coarse_chan_bw = bw / num_coarse_chans; + config.filter_delay = tsamp * overlap_samps / 2.0; + BOOST_LOG_TRIVIAL(debug) + << "tsamp in create_coherent_dedisperser_config: " << config.tsamp; + BOOST_LOG_TRIVIAL(info) + << "overlap_samps just due to dedispersion: " << overlap_samps; BOOST_LOG_TRIVIAL(info) << "Filter delay: " << config.filter_delay; - config.overlap_samps = compute_padding(gulp_samps + overlap_samps) + overlap_samps; - - BOOST_LOG_TRIVIAL(info) << "overlap_samps after padding for optimal FFT size " << config.overlap_samps; + config.overlap_samps = + compute_padding(gulp_samps + overlap_samps) + overlap_samps; + BOOST_LOG_TRIVIAL(info) + << "overlap_samps after padding for optimal FFT size " + << config.overlap_samps; /* Precompute DM constants */ config._h_dms = dms; @@ -145,7 +149,8 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, config._d_dm_prefactor.resize(dms.size()); config._d_ism_responses.resize(dms.size()); for(int i = 0; i < dms.size(); i++) { - config._d_ism_responses[i].resize(num_coarse_chans * (config.gulp_samps + config.overlap_samps)); + config._d_ism_responses[i].resize( + num_coarse_chans * (config.gulp_samps + config.overlap_samps)); } thrust::transform(config._d_dms.begin(), @@ -164,16 +169,16 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, // data is FTPA order, we will loop over F, so we are left with TPA order. // Let's fuse PA to X, so TX order. // We stride and batch over X and transform T - std::size_t X = config.npols * config.nantennas; - std::size_t fft_size = config.gulp_samps + config.overlap_samps; - int n[1] = {static_cast(fft_size)}; // FFT size - int inembed[1] = {static_cast(fft_size)}; - int onembed[1] = {static_cast(fft_size)}; - int istride = X; - int ostride = X; - int idist = 1; - int odist = 1; - int batch = X; + std::size_t X = config.npols * config.nantennas; + std::size_t fft_size = config.gulp_samps + config.overlap_samps; + int n[1] = {static_cast(fft_size)}; // FFT size + int inembed[1] = {static_cast(fft_size)}; + int onembed[1] = {static_cast(fft_size)}; + int istride = X; + int ostride = X; + int idist = 1; + int odist = 1; + int batch = X; if(cufftPlanMany(&config._fft_plan, 1, @@ -192,9 +197,6 @@ void create_coherent_dedisperser_config(CoherentDedisperserConfig& config, BOOST_LOG_TRIVIAL(debug) << "FFT plan created"; } - - - void CoherentDedisperser::dedisperse( TPAVoltagesD const& d_tpa_voltages_in, FTPAVoltagesD& d_ftpa_voltages_out, @@ -202,8 +204,8 @@ void CoherentDedisperser::dedisperse( unsigned int dm_idx) { BOOST_LOG_NAMED_SCOPE("CoherentDedisperser::dedisperse"); - //d_tpa_voltages_in.size() is with overlap - _d_fpa_spectra.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); + // d_tpa_voltages_in.size() is with overlap + _d_fpa_spectra.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); _d_tpa_voltages_in_cufft.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); _d_tpa_voltages_dedispersed.resize(d_tpa_voltages_in.size(), {0.0f, 0.0f}); @@ -250,8 +252,8 @@ void CoherentDedisperser::dedisperse( BOOST_LOG_TRIVIAL(debug) << "Executed inverse FFT"; - std::size_t out_offset = freq_idx * _config.nantennas * _config.npols * - (_config.gulp_samps); + std::size_t out_offset = + freq_idx * _config.nantennas * _config.npols * (_config.gulp_samps); std::size_t discard_size = _config.nantennas * _config.npols * _config.overlap_samps / 2; @@ -260,30 +262,28 @@ void CoherentDedisperser::dedisperse( BOOST_LOG_TRIVIAL(debug) << "copying from input from " << discard_size << " to " << _d_tpa_voltages_dedispersed.size() - discard_size; - BOOST_LOG_TRIVIAL(debug) << "Total elements copied: " - << _d_tpa_voltages_dedispersed.size() - 2 * discard_size; + BOOST_LOG_TRIVIAL(debug) + << "Total elements copied: " + << _d_tpa_voltages_dedispersed.size() - 2 * discard_size; BOOST_LOG_TRIVIAL(debug) << "Remaining space in output: " << d_ftpa_voltages_out.size() - out_offset; BOOST_LOG_TRIVIAL(debug) << "copying to output from " << out_offset << " to " << out_offset + _d_tpa_voltages_dedispersed.size() - 2 * discard_size; + std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; - std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; - - - - thrust::transform(_d_tpa_voltages_dedispersed.begin() + discard_size, - _d_tpa_voltages_dedispersed.end() - discard_size, - d_ftpa_voltages_out.begin() + out_offset, - [=] __device__(cufftComplex const& val) { - char2 char2_val; - char2_val.x = static_cast( - __float2int_rn(val.x / fft_size)); // scale the data back - char2_val.y = - static_cast(__float2int_rn(val.y / fft_size)); - return char2_val; - }); + thrust::transform( + _d_tpa_voltages_dedispersed.begin() + discard_size, + _d_tpa_voltages_dedispersed.end() - discard_size, + d_ftpa_voltages_out.begin() + out_offset, + [=] __device__(cufftComplex const& val) { + char2 char2_val; + char2_val.x = static_cast( + __float2int_rn(val.x / fft_size)); // scale the data back + char2_val.y = static_cast(__float2int_rn(val.y / fft_size)); + return char2_val; + }); BOOST_LOG_TRIVIAL(debug) << "Transformed dedispersed voltages to char2"; d_ftpa_voltages_out.reference_dm(_config._h_dms[dm_idx]); @@ -296,21 +296,24 @@ void CoherentDedisperser::multiply_by_chirp( unsigned int freq_idx, unsigned int dm_idx) { - std::size_t total_chans = _config._d_ism_responses[dm_idx].size(); // all coarse + fine chans - std::size_t fft_size = _config.gulp_samps + _config.overlap_samps; // ONLY FINE CHANS + std::size_t total_chans = + _config._d_ism_responses[dm_idx].size(); // all coarse + fine chans + std::size_t fft_size = + _config.gulp_samps + _config.overlap_samps; // ONLY FINE CHANS std::size_t response_offset = freq_idx * fft_size; BOOST_LOG_TRIVIAL(debug) << "Freq idx: " << freq_idx; BOOST_LOG_TRIVIAL(debug) << "_config.gulp_samps: " << _config.gulp_samps; - BOOST_LOG_TRIVIAL(debug) << "_config.overlap_samps: " << _config.overlap_samps; + BOOST_LOG_TRIVIAL(debug) + << "_config.overlap_samps: " << _config.overlap_samps; BOOST_LOG_TRIVIAL(debug) << "response_offset: " << response_offset; BOOST_LOG_TRIVIAL(debug) << "total_chans: " << total_chans; - BOOST_LOG_TRIVIAL(debug) << "chirp multiply input size: " - << _d_fpa_spectra_in.size(); - BOOST_LOG_TRIVIAL(debug) << "chirp multiply output size: " - << _d_fpa_spectra_out.size(); + BOOST_LOG_TRIVIAL(debug) + << "chirp multiply input size: " << _d_fpa_spectra_in.size(); + BOOST_LOG_TRIVIAL(debug) + << "chirp multiply output size: " << _d_fpa_spectra_out.size(); - dim3 blockSize(_config.nantennas * _config.npols); + dim3 blockSize(_config.nantennas * _config.npols); dim3 gridSize(fft_size / NCHANS_PER_BLOCK); kernels::dedisperse<<>>( thrust::raw_pointer_cast(_config._d_ism_responses[dm_idx].data() + @@ -334,10 +337,12 @@ __global__ void dedisperse(cufftComplex const* __restrict__ _d_ism_response, volatile __shared__ cufftComplex response[NCHANS_PER_BLOCK]; - const unsigned block_start_chan_idx = blockIdx.x * NCHANS_PER_BLOCK; // coarse chan idx + const unsigned block_start_chan_idx = + blockIdx.x * NCHANS_PER_BLOCK; // coarse chan idx const unsigned remainder = - min(total_chans - block_start_chan_idx, NCHANS_PER_BLOCK); // how many channels to process + min(total_chans - block_start_chan_idx, + NCHANS_PER_BLOCK); // how many channels to process for(int idx = threadIdx.x; idx < remainder; idx += pa_size) { cufftComplex const temp = _d_ism_response[block_start_chan_idx + idx]; @@ -387,8 +392,7 @@ struct DMResponse { int chan = tid / num_fine_chans; // Coarse channel int fine_chan = tid % num_fine_chans; // fine channel - double nu_0 = low_freq + chan * coarse_chan_bw - - 0.5f * coarse_chan_bw; + double nu_0 = low_freq + chan * coarse_chan_bw - 0.5f * coarse_chan_bw; double nu = fine_chan * fine_chan_bw; // fine_chan_freq @@ -414,11 +418,11 @@ void get_dm_responses(CoherentDedisperserConfig& config, thrust::device_vector& response) { BOOST_LOG_TRIVIAL(debug) << "Generating DM responses"; - std::size_t fft_size = config.gulp_samps + config.overlap_samps; + std::size_t fft_size = config.gulp_samps + config.overlap_samps; thrust::device_vector indices(config.num_coarse_chans * fft_size); thrust::sequence(indices.begin(), indices.end()); - + // store raw responses in a temporary variable thrust::device_vector temp_response(response.size()); BOOST_LOG_TRIVIAL(warning) << "DOING FFTSHIFT" << std::endl; @@ -427,16 +431,26 @@ void get_dm_responses(CoherentDedisperserConfig& config, indices.end(), temp_response.begin(), kernels::DMResponse(config.num_coarse_chans, - config.gulp_samps, + fft_size, config.low_freq, config.coarse_chan_bw, config.fine_chan_bw, dm_prefactor)); - // rotate the response to match cufft output + // rotate the response to match cufft output + + if(fft_size %2 != 0) { + throw std::runtime_error("FFT size must be even."); + } + std::size_t shift = fft_size / 2; - thrust::copy(temp_response.begin() + shift, temp_response.end(), response.begin()); - thrust::copy(temp_response.begin(), temp_response.begin() + shift, response.begin() + (fft_size - shift)); + + for(auto it = temp_response.begin(), it_out = response.begin(); + it < temp_response.end(); + it += fft_size, it_out += fft_size) { + thrust::copy(it, it + shift, it_out + shift); + thrust::copy(it + shift, it + fft_size, it_out); + } } } // namespace skyweaver From 4bf4df14edd3d25f975674a5f1afa343bf3cd3c7 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Fri, 11 Oct 2024 17:40:50 +0200 Subject: [PATCH 28/69] Make skycleaver a CPP application --- cpp/skyweaver/CMakeLists.txt | 2 +- cpp/skyweaver/SkyCleaver.cuh | 81 ------------------- cpp/skyweaver/SkyCleaver.hpp | 68 ++++++++++++---- .../src/{SkyCleaver.cu => SkyCleaver.cpp} | 4 +- .../{skycleaver_cli.cu => skycleaver_cli.cpp} | 0 5 files changed, 57 insertions(+), 98 deletions(-) delete mode 100644 cpp/skyweaver/SkyCleaver.cuh rename cpp/skyweaver/src/{SkyCleaver.cu => SkyCleaver.cpp} (99%) rename cpp/skyweaver/src/{skycleaver_cli.cu => skycleaver_cli.cpp} (100%) diff --git a/cpp/skyweaver/CMakeLists.txt b/cpp/skyweaver/CMakeLists.txt index 0f49287..76d6ae6 100644 --- a/cpp/skyweaver/CMakeLists.txt +++ b/cpp/skyweaver/CMakeLists.txt @@ -111,7 +111,7 @@ target_link_libraries(skyweavercpp OpenMP::OpenMP_CXX) install(TARGETS skyweavercpp DESTINATION bin) -cuda_add_executable(skycleaver src/skycleaver_cli.cu) +add_executable(skycleaver src/skycleaver_cli.cpp) target_link_libraries(skycleaver ${SKYWEAVER_LIBRARIES} ${DEPENDENCY_LIBRARIES} diff --git a/cpp/skyweaver/SkyCleaver.cuh b/cpp/skyweaver/SkyCleaver.cuh deleted file mode 100644 index 4c7be58..0000000 --- a/cpp/skyweaver/SkyCleaver.cuh +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef SKYWEAVER_SKYCLEAVER_HPP -#define SKYWEAVER_SKYCLEAVER_HPP -#include "boost/log/trivial.hpp" -#include "psrdada_cpp/psrdadaheader.hpp" -#include "psrdada_cpp/raw_bytes.hpp" -#include "skyweaver/DescribedVector.hpp" -#include "skyweaver/MultiFileReader.cuh" -#include "skyweaver/MultiFileWriter.cuh" -#include "skyweaver/ObservationHeader.hpp" -#include "skyweaver/SkyCleaverConfig.hpp" -#include "skyweaver/Timer.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace skyweaver -{ - -struct BridgeReader { - public: - std::vector _tdb_filenames; - std::unique_ptr _tdb_reader; - std::string freq; - -}; // BridgeReader - - -class SkyCleaver -{ - public: - using InputType = int8_t; - using OutputType = uint8_t; - using InputVectorType = TDBPowersH; - using OutputVectorType = TFPowersH; - using FreqType = std::size_t; // up to the nearest Hz - using BeamNumberType = std::size_t; - using DMNumberType = std::size_t; - - private: - SkyCleaverConfig const& _config; - std::map> _bridge_readers; - std::map> _bridge_data; - - std::vector _expected_freqs; - std::vector _available_freqs; - std::size_t _nsamples_to_read; - ObservationHeader _header; - - std::vector _beam_filenames; - std::map>>> - _beam_writers; - std::map>> - _beam_data; - std::size_t _total_beam_writers; - - void init_readers(); - void init_writers(); - - Timer _timer; - - public: - SkyCleaver(SkyCleaverConfig const& config); - SkyCleaver(SkyCleaver const&) = delete; - void operator=(SkyCleaver const&) = delete; - - void cleave(); - -}; // class SkyCleaver - -} // namespace skyweaver - -#endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaver.hpp b/cpp/skyweaver/SkyCleaver.hpp index c22ae3d..46449de 100644 --- a/cpp/skyweaver/SkyCleaver.hpp +++ b/cpp/skyweaver/SkyCleaver.hpp @@ -1,8 +1,14 @@ -#include "SigprocFileWriter.hpp" +#ifndef SKYWEAVER_SKYCLEAVER_HPP +#define SKYWEAVER_SKYCLEAVER_HPP #include "boost/log/trivial.hpp" #include "psrdada_cpp/psrdadaheader.hpp" #include "psrdada_cpp/raw_bytes.hpp" +#include "skyweaver/DescribedVector.hpp" +#include "skyweaver/MultiFileReader.cuh" +#include "skyweaver/MultiFileWriter.cuh" #include "skyweaver/ObservationHeader.hpp" +#include "skyweaver/SkyCleaverConfig.hpp" +#include "skyweaver/Timer.hpp" #include #include @@ -12,32 +18,64 @@ #include #include #include + namespace skyweaver { -struct Bridge -{ - private: - using PowerType = std::vector; +struct BridgeReader { public: std::vector _tdb_filenames; + std::unique_ptr _tdb_reader; std::string freq; -}; // class Bridge +}; // BridgeReader -class MultiBeamWriter + +class SkyCleaver { + public: + using InputType = int8_t; + using OutputType = uint8_t; + using InputVectorType = TDBPowersStdH; + using OutputVectorType = TFPowersStdH; + using FreqType = std::size_t; // up to the nearest Hz + using BeamNumberType = std::size_t; + using DMNumberType = std::size_t; + private: - using FreqType = unsigned int; // up to the nearest Hz - std::map> _bridges; + SkyCleaverConfig const& _config; + std::map> _bridge_readers; + std::map> _bridge_data; + + std::vector _expected_freqs; + std::vector _available_freqs; + std::size_t _nsamples_to_read; + ObservationHeader _header; + std::vector _beam_filenames; + std::map>>> + _beam_writers; + std::map>> + _beam_data; + std::size_t _total_beam_writers; + + void init_readers(); + void init_writers(); + + Timer _timer; + + public: + SkyCleaver(SkyCleaverConfig const& config); + SkyCleaver(SkyCleaver const&) = delete; + void operator=(SkyCleaver const&) = delete; + void cleave(); - public - : add_bridge(FreqType freq, std::vector tdb_filenames); - remove_bridge(FreqType freq); +}; // class SkyCleaver - init(); +} // namespace skyweaver -} // class MultiFileWriter -} // namespace skyweaver \ No newline at end of file +#endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/src/SkyCleaver.cu b/cpp/skyweaver/src/SkyCleaver.cpp similarity index 99% rename from cpp/skyweaver/src/SkyCleaver.cu rename to cpp/skyweaver/src/SkyCleaver.cpp index 4f73cce..4420950 100644 --- a/cpp/skyweaver/src/SkyCleaver.cu +++ b/cpp/skyweaver/src/SkyCleaver.cpp @@ -281,6 +281,7 @@ void SkyCleaver::init_readers() BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver: " << _header.to_string(); _header.nchans = _header.nchans * _config.nbridges(); _header.nbeams = _config.nbeams(); + } void SkyCleaver::init_writers() @@ -449,7 +450,8 @@ void SkyCleaver::cleave() #pragma omp parallel for schedule(static) collapse(2) for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { - for(std::size_t idm = 0; idm < ndms; idm++) { + for(std::size_t idm = 0; idm < ndms; idm++) { // cannot separate loops, so do checks later + // _beam_data[idm][ibeam] is not found, skip if(_beam_data.find(idm) == _beam_data.end()) { continue; diff --git a/cpp/skyweaver/src/skycleaver_cli.cu b/cpp/skyweaver/src/skycleaver_cli.cpp similarity index 100% rename from cpp/skyweaver/src/skycleaver_cli.cu rename to cpp/skyweaver/src/skycleaver_cli.cpp From 70647a981c30c94d9abb55ac406cbfc384dea1e1 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Fri, 11 Oct 2024 17:42:01 +0200 Subject: [PATCH 29/69] correctly set frequencies for described vectors, CDD, IDD --- cpp/skyweaver/DescribedVector.hpp | 12 +++++++++--- .../detail/IncoherentDedispersionPipeline.cu | 5 ++--- cpp/skyweaver/detail/file_writer_callbacks.cpp | 6 ++---- cpp/skyweaver/src/CoherentDedisperser.cu | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cpp/skyweaver/DescribedVector.hpp b/cpp/skyweaver/DescribedVector.hpp index 6fcdd0a..4950f30 100644 --- a/cpp/skyweaver/DescribedVector.hpp +++ b/cpp/skyweaver/DescribedVector.hpp @@ -366,12 +366,13 @@ struct DescribedVector { * @param freqs */ void frequencies(FrequenciesType const& freqs) - { + { if(freqs.size() != get_dim_extent()) { throw std::runtime_error("Invalid number of frequecies passed."); } _frequencies_stale = false; _frequencies = freqs; + } /** @@ -386,6 +387,7 @@ struct DescribedVector { } _frequencies_stale = false; _frequencies.resize(1, freq); + _frequencies[0] = freq; } /** @@ -691,9 +693,13 @@ using FPAStatsD = DescribedVector, FreqDim, PolnDim, AntennaDim>; //skycleaver output vectors - template -using TFPowersH = DescribedVector>, +using TDBPowersStdH = DescribedVector>, + TimeDim, + DispersionDim, + BeamDim>; +template +using TFPowersStdH = DescribedVector>, TimeDim, FreqDim>; diff --git a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu index 62f722b..f295f5b 100644 --- a/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu +++ b/cpp/skyweaver/detail/IncoherentDedispersionPipeline.cu @@ -73,10 +73,9 @@ void IncoherentDedispersionPipeline:: // Set the correct DMs on the block _output_buffers[ref_dm_idx].dms(plan[ref_dm_idx].incoherent_dms); _output_buffers[ref_dm_idx].reference_dm(plan[ref_dm_idx].coherent_dm); - _output_buffers[ref_dm_idx].frequencies({_config.centre_frequency() - _config.bandwidth() / 2.0}); - - BOOST_LOG_TRIVIAL(debug) << "setting centre frequency to " << _output_buffers[ref_dm_idx].frequencies()[0]; + _output_buffers[ref_dm_idx].frequencies(_config.channel_frequencies().front()); + BOOST_LOG_TRIVIAL(debug) << "setting centre frequency to " << std::setprecision(15) << _output_buffers[ref_dm_idx].frequencies()[0]; BOOST_LOG_TRIVIAL(debug) << "Passing output buffer to handler: " << _output_buffers[ref_dm_idx].describe(); _timer.start("file writing"); diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 10d9929..f2b9601 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -54,7 +54,6 @@ RECEIVER L-band FREQ 1284000000.000000 BW 856000000.000000 TSAMP 4.7850467290 -STOKES I NBIT 8 NDIM 1 @@ -62,7 +61,6 @@ NPOL 1 NCHAN 64 NBEAM 800 ORDER TFB - CHAN0_IDX 2688 )"; @@ -125,9 +123,9 @@ create_dada_file_stream(MultiFileWriterConfig const& config, header_writer.set("RA", header.ra); header_writer.set("DEC", header.dec); header_writer.set("NBEAM", stream_data.nbeams()); + header_writer.set("OBS_NCHAN", header.obs_nchans); header_writer.set("NCHAN", stream_data.nchannels()); - header_writer.set("OBS_NCHAN", header.obs_nchans); header_writer.set("OBS_FREQUENCY", header.obs_frequency); header_writer.set("OBS_BW", header.obs_bandwidth); @@ -135,7 +133,7 @@ create_dada_file_stream(MultiFileWriterConfig const& config, if(stream_data.ndms()) { header_writer.set("NDMS", stream_data.ndms()); header_writer.set("DMS", stream_data.dms(), 7); - } + } header_writer.set( "COHERENT_DM", static_cast(stream_data.reference_dm())); diff --git a/cpp/skyweaver/src/CoherentDedisperser.cu b/cpp/skyweaver/src/CoherentDedisperser.cu index 79fc4be..94b0d31 100644 --- a/cpp/skyweaver/src/CoherentDedisperser.cu +++ b/cpp/skyweaver/src/CoherentDedisperser.cu @@ -392,7 +392,7 @@ struct DMResponse { int chan = tid / num_fine_chans; // Coarse channel int fine_chan = tid % num_fine_chans; // fine channel - double nu_0 = low_freq + chan * coarse_chan_bw - 0.5f * coarse_chan_bw; + double nu_0 = low_freq + chan * coarse_chan_bw; double nu = fine_chan * fine_chan_bw; // fine_chan_freq From 0dde728270860a8ca853dd1016e311052e4e4153 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Mon, 21 Oct 2024 05:21:23 +0200 Subject: [PATCH 30/69] Added multiple tiling plot code, targets file, mosaic command file, psf, neighbouring and overlapping beam generator --- python/skyweaver/cli.py | 40 +-- python/skyweaver/skyweaver.py | 442 ++++++++++++++++++++++++++++++++-- 2 files changed, 452 insertions(+), 30 deletions(-) diff --git a/python/skyweaver/cli.py b/python/skyweaver/cli.py index c29fbfb..184b898 100644 --- a/python/skyweaver/cli.py +++ b/python/skyweaver/cli.py @@ -150,7 +150,8 @@ def delays_create( bfconfig: str, pointing_idx: int = None, step: float = 4.0, - outfile: str = None): + outfile: str = None, + hex: bool = True): """Create a delay file Args: @@ -160,6 +161,7 @@ def delays_create( step (float, optional): The time step between delay solutions. Defaults to 4.0 seconds. outfile (str, optional): The file to write delay models to. Defaults to a standard output filename. + hex (bool, optional): Use hex encoding for the output filename. Defaults to True. """ sm = skyweaver.SessionMetadata.from_file(metafile) bc = skyweaver.BeamformerConfig.from_file(bfconfig) @@ -173,23 +175,30 @@ def delays_create( raise ValueError("Pointing idx {} requested but only {} pointings in session") step = step * u.s pointing = pointings[pointing_idx] - delays, targets, _ = skyweaver.create_delays(sm, bc, pointing, step=step) + if outfile is None: - fname = "swdelays_{}_{}_to_{}_{}.bin".format( - pointing.phase_centre.name, - int(pointing.start_epoch.unix), - int(pointing.end_epoch.unix), - secrets.token_hex(3) - ) + if hex: + fname = "swdelays_{}_{}_to_{}_{}".format( + pointing.phase_centre.name, + int(pointing.start_epoch.unix), + int(pointing.end_epoch.unix), + secrets.token_hex(3) + ) + else: + fname = "swdelays_{}_{}_to_{}".format( + pointing.phase_centre.name, + int(pointing.start_epoch.unix), + int(pointing.end_epoch.unix) + ) else: fname = outfile - log.info("Writing delay model to file %s", fname) - with open(fname, "wb") as fo: + delays, targets, _ = skyweaver.create_delays(sm, bc, pointing, step=step, outfile=fname) + + log.info(f"Writing delay model to file {fname}.bin") + with open(fname + ".bin", "wb") as fo: for delay_model in delays: fo.write(delay_model.to_bytes()) - with open(fname + ".targets", "w") as fo: - for target in targets: - fo.write(target.format_katcp() +"\n") + def parse_default_args(args): @@ -255,13 +264,16 @@ def cli(): help="An HDF5 FBFUSE-BVR metadata file") delays_create_parser.add_argument("bfconfig", metavar="BFCONFIG", help="A YAML beamformer configuration file") + delays_create_parser.add_argument("--no-hex", action="store_false", dest="hex", + help="Do not use hex encoding for the output filename") delays_create_parser.set_defaults( func=lambda args: delays_create( args.metafile, args.bfconfig, args.pointing_idx, args.step, - args.outfile)) + args.outfile, + args.hex)) # parse and execute args = parser.parse_args() diff --git a/python/skyweaver/skyweaver.py b/python/skyweaver/skyweaver.py index 55ff79f..38e99d5 100644 --- a/python/skyweaver/skyweaver.py +++ b/python/skyweaver/skyweaver.py @@ -7,18 +7,28 @@ import logging import textwrap import ctypes -from typing import Any +from typing import Any, Tuple from dataclasses import dataclass from typing_extensions import Self +import sys +from collections import defaultdict +from matplotlib.patches import Ellipse +import matplotlib.patches as mpatches +from adjustText import adjust_text # 3rd party imports import h5py import yaml import numpy as np +import pandas as pd +import random from rich.progress import track from astropy.time import Time, TimeDelta from astropy import units as u +from astropy.coordinates import SkyCoord from astropy.units import Quantity +from astropy import wcs +import matplotlib.pyplot as plt from katpoint import Target, Antenna from mosaic.beamforming import ( DelayPolynomial, @@ -228,6 +238,8 @@ def add_tiling( for ii, (ra, dec) in enumerate(coordinates): self._targets.append( (Target(f"{prefix}_{ii:04d},radec,{ra},{dec}"), sub_array_idx)) + + def add_beam(self, target: Target, subarray: Subarray = None) -> None: """Add a single beam to the engine @@ -245,6 +257,7 @@ def add_beam(self, target: Target, subarray: Subarray = None) -> None: sub_array_idx = len(self._subarray_sets) - 1 self._targets.append((target, sub_array_idx)) + def _extract_weights(self) -> np.ndarray: # Here we extract the weights for each beam/antenna # as an optimisation we cache the antenna mask per @@ -649,6 +662,7 @@ class BeamSet: A beam set is a collection of beams which are formed from a common subarray. """ + name: str anntenna_names: list[str] beams: list[Target] tilings: list[dict] @@ -691,6 +705,7 @@ def from_file(cls, config_file: str) -> Self: for bs in data["beam_sets"]: beam_sets.append( BeamSet( + bs["name"], bs["antenna_set"], bs["beams"], bs["tilings"], @@ -707,10 +722,27 @@ def from_file(cls, config_file: str) -> Self: ) +class BeamSetOutput: + def __init__(self): + self.beam_names = [] + self.beam_set_id = None + self.beam_shape_x = None + self.beam_shape_y = None + self.beam_shape_z = None + self.overlap = None + self.ra = None # Right Ascension + self.dec = None # Declination + + def set_ra_dec(self, ra, dec): + """Set RA and Dec for the beam set.""" + self.ra = ra + self.dec = dec + + def make_tiling( pointing: PointingMetadata, subarray: Subarray, - tiling_desc: dict) -> Tiling: + tiling_desc: dict) -> Tuple[Tiling, BeamShape]: """Make a tiling using the complete mosaic tiling options Args: @@ -779,15 +811,37 @@ def make_tiling( antenna_strings: list[str] = [ ant.format_katcp() for ant in subarray.antenna_positions ] + psfsim: PsfSim = PsfSim(antenna_strings, ref_freq) - beam_shape: BeamShape = psfsim.get_beam_shape(target, epoch.unix) + psf_beam_shape: BeamShape = psfsim.get_beam_shape(target, epoch.unix) + #Build Mosaic command here. Remove T + mosaic_epoch = epoch.iso.replace("T", " ") + mosaic_epoch = mosaic_epoch.replace("-", ".") + mosaic_antenna_string = ','.join([item.replace('m', '') for item in subarray.names]) + mosaic_command=f"python maketiling.py --freq {ref_freq} --source {target.body._ra} {target.body._dec} --datetime {mosaic_epoch} --subarray {mosaic_antenna_string} --verbose --tiling_method {method} --tiling_shape {shape} --ants antenna.csv --beamnum {nbeams} --overlap {overlap}" + + tiling: Tiling = generate_nbeams_tiling( - beam_shape, nbeams, overlap, + psf_beam_shape, nbeams, overlap, method, shape, parameter=shape_params, coordinate_type=coordinate_type) - return tiling - + return tiling, psf_beam_shape, mosaic_command + +def pad_ra_dec(ra, dec): + # Split RA and Dec into components + ra_parts = ra.split(':') + dec_parts = dec.split(':') + + # Pad RA hours and Dec degrees to two digits + ra_parts[0] = ra_parts[0].zfill(2) # Ensure two digits for RA hours + dec_parts[0] = dec_parts[0].zfill(2) # Ensure two digits for Dec degrees + + # Join the parts back together + padded_ra = ':'.join(ra_parts) + padded_dec = ':'.join(dec_parts) + + return padded_ra, padded_dec def create_delays( session_metadata: SessionMetadata, @@ -795,7 +849,8 @@ def create_delays( pointing: PointingMetadata, start_epoch: Time = None, end_epoch: Time = None, - step: TimeDelta = 4 * u.s) -> list[DelayModel]: + step: TimeDelta = 4 * u.s, + outfile: str = None) -> list[DelayModel]: """Create a set of delay models Args: @@ -808,6 +863,7 @@ def create_delays( Defaults to the end of the pointing. step (TimeDelta, optional): The step size between consequtive solutions. Defaults to 4*u.s. + outfile (str, optional): The path to write the delay models to. Defaults is None. Returns: list[DelayModel]: A list of delay models @@ -824,27 +880,381 @@ def create_delays( log.info("Step size: %s", step.to(u.s)) full_subarray = om.get_subarray() de = DelayEngine(full_subarray, pointing.phase_centre) - bs_tilings = [] + # Initialize beam_set_lookup dictionary to track unique beam sets + beam_set_lookup = {} + beam_set_id = 0 + plot_beams = [] + neighbouring_beams = [] + #Initialise the known beams. PSF size is calculated for 50% overlap + nbeams_requested = 1 + overlap = 0.5 + #Iterate beams first through all beam sets for bs in bc.beam_sets: - tilings = [] - subarray_subset = om.get_subarray(bs.anntenna_names) - # add beams first as these are likely more important than tilings + sorted_antennas = sorted(bs.anntenna_names) + subarray_subset = om.get_subarray(sorted_antennas) + antenna_string = ','.join(sorted_antennas) if bs.beams is not None: for target_desc in bs.beams: - de.add_beam(Target(target_desc), subarray_subset) + target = Target(target_desc) + #Add the beam to the delay engine + de.add_beam(target, subarray_subset) + beam_key = (antenna_string, overlap, nbeams_requested) + if beam_key not in beam_set_lookup: + beam_set_lookup[beam_key] = beam_set_id + beam_set_id += 1 + ra = str(target.body._ra) + dec = str(target.body._dec) + # Pad RA and Dec to two digits + ra, dec = pad_ra_dec(ra, dec) + name = target.name + current_beam_set_id = beam_set_lookup[beam_key] + #Get the PSF Beam shape at 50 % overlap for beams defined in yaml. + tiling_desc = { + "nbeams": nbeams_requested, + "overlap": overlap, + "target": target_desc, + } + _, psf_beam_shape,_ = make_tiling(pointing, subarray_subset, tiling_desc) + #Add the beam to the plot beams list + plot_beams.append((name, ra, dec, round(psf_beam_shape.axisH, 5), round(psf_beam_shape.axisV, 5), round(psf_beam_shape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'known')) + neighbouring_beams.append((name, ra, dec, round(psf_beam_shape.axisH, 5), round(psf_beam_shape.axisV, 5), round(psf_beam_shape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'known')) + + #Iterate through all tilings + bs_tilings = [] + for bs in bc.beam_sets: + tilings = [] + sorted_antennas = sorted(bs.anntenna_names) + subarray_subset = om.get_subarray(sorted_antennas) + antenna_string = ','.join(sorted_antennas) if bs.tilings is not None: + output_prefix = f"{outfile}_{bs.name}" if outfile is not None else None for tiling_desc in bs.tilings: - tiling = make_tiling(pointing, subarray_subset, tiling_desc) + tiling, psf_beamshape, mosaic_command = make_tiling(pointing, subarray_subset, tiling_desc) + #Add the tiling to the delay engine de.add_tiling(tiling, subarray_subset) tilings.append(tiling) + cb_beamshape = tiling.meta["axis"][:3] #axisH, axisV, angle + overlap = tiling.meta["axis"][-1] + nbeams_requested = tiling_desc['nbeams'] + beam_key = (antenna_string, overlap, nbeams_requested) + if beam_key not in beam_set_lookup: + beam_set_lookup[beam_key] = beam_set_id + beam_set_id += 1 + current_beam_set_id = beam_set_lookup[beam_key] + coords = tiling.get_equatorial_coordinates() + coords = SkyCoord(coords, unit=u.deg) + mosaic_command+=f" --tiling_plot {output_prefix}_bid_{beam_set_id}.png --tiling_coordinate {output_prefix}_bid_{beam_set_id}.csv" + log.info("Mosaic command written to %s", f"{outfile}.mosaic") + log.info(f"Writing PSF of BeamSet {bs.name} for target {target.name} to {output_prefix}_bid_{beam_set_id}.fits") + psf_beamshape.psf.write_fits(f"{output_prefix}_bid_{beam_set_id}.fits") + log.info(f"PSF Plot of BeamSet {bs.name} for target {target.name} to {output_prefix}_bid_{beam_set_id}.png") + psf_beamshape.plot_psf(f"{output_prefix}_bid_{beam_set_id}.png") + with open(f"{outfile}.mosaic", "a") as f: + f.write(mosaic_command + "\n") + for index, coord in enumerate(coords): + ra_hms = coord.ra.to_string(unit=u.hour, sep=':', precision=2, pad=True) + dec_dms = coord.dec.to_string(unit=u.degree, sep=':', precision=1, alwayssign=True, pad=True) + plot_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(cb_beamshape[0], 5), round(cb_beamshape[1], 5), round(cb_beamshape[2], 5), current_beam_set_id, overlap, len(sorted_antennas), 'tiling')) + neighbouring_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(psf_beam_shape.axisH, 5), round(psf_beam_shape.axisV, 5), round(psf_beam_shape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'tiling')) bs_tilings.append(tilings) - log.info( - "Calculating solutions for %d antennas and %d beams", - full_subarray.nantennas, de.nbeams) + + target = tiling_desc.get("target", None) + if target is None: + print("No target specified") + target = pointing.phase_centre + else: + target = Target(target) + column_list = ['name', 'ra', 'dec', 'x', 'y', 'angle', 'beam_set_id', 'overlap', 'nantennas', 'type'] + plot_beams_df = pd.DataFrame(plot_beams, columns=column_list) + #Plot beams has beam shape for the overlap requested, wheras neighbouring beams has the PSF shape (50% overlap) + neighbouring_beams_df = pd.DataFrame(neighbouring_beams, columns=column_list) + log.info("Calculating solutions for %d antennas and %d beams", full_subarray.nantennas, de.nbeams) delays = de.calculate_delays(start_epoch, end_epoch, step) + log.info("Beams and tilings written to %s.targets", outfile) + plot_beams_df.to_csv(outfile + ".targets", index=False) + boresight_coords = psf_beamshape.bore_sight.equatorial + plot_multiple_tilings(pointing, neighbouring_beams_df, plot_beams_df, boresight_coords, outfile) return delays, de.targets, bs_tilings +def plot_multiple_tilings(pointing, neighbouring_beams_df, plot_beams_df, boresight_coords, outfile, HD=True, beam_size_scaling=1.0, annotate_beam_names=False): + # Initialize WCS projection + wcs_properties = wcs.WCS(naxis=2) + wcs_properties.wcs.crpix = [0, 0] + wcs_properties.wcs.ctype = ["RA---TAN", "DEC--TAN"] + wcs_properties.wcs.crval = boresight_coords + center = boresight_coords + + # Use neutral scaling for overlap detection + wcs_properties.wcs.cdelt = [1, 1] # Neutral pixel scale + + detector = BeamOverlapDetector(neighbouring_beams_df, wcs_properties) + overlaps_df = detector.find_overlapping_beams() + overlaps_df.to_csv(outfile + "_overlapping_beams.csv", index=False) + + # Update WCS projection for plotting + step = 1 / 10000000000.0 + wcs_properties.wcs.cdelt = [-step, step] + resolution = step + + thisDPI = 300 + if HD: + width = 3200. + extra_source_text_size = 8 + else: + width = 800. + extra_source_text_size = 3 + + fig = plt.figure(figsize=(width/thisDPI, width/thisDPI), dpi=thisDPI) + axis = fig.add_subplot(111, aspect='equal', projection=wcs_properties) + + # Define color palette + color_palette = ['#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33', '#a65628', '#f781bf'] + + # Extract relevant data from DataFrame in one step for efficiency and clarity + beam_ra, beam_dec, beam_name, beam_set_id, beam_x, beam_y, beam_angle, beam_type, nantennas, overlap = ( + plot_beams_df['ra'].astype(str), + plot_beams_df['dec'].astype(str), + plot_beams_df['name'].astype(str), + plot_beams_df['beam_set_id'].astype(int), + plot_beams_df['x'].astype(float), + plot_beams_df['y'].astype(float), + plot_beams_df['angle'].astype(float), + plot_beams_df['type'].astype(str), + plot_beams_df['nantennas'].astype(int), + plot_beams_df['overlap'].astype(float) + ) + + # Get equatorial coordinates + equatorialCoordinates = SkyCoord(beam_ra, beam_dec, frame='fk5', unit=(u.hourangle, u.deg)) + beam_coordinate = np.array(wcs_properties.wcs_world2pix(np.array([equatorialCoordinates.ra.deg, equatorialCoordinates.dec.deg]).T, 0)) + + # Plot the boresight for reference + axis.plot(0, 0, marker='+', markersize=15, color='black') + + # Store labels for legend creation based on beam_set_id + labels = {} + + # Loop through all beams to create ellipses and prepare legend text + for idx in range(len(beam_coordinate)): + coord = beam_coordinate[idx] + + # Create the ellipse for each beam + ellipse = Ellipse( + xy=coord, + width=2.0 * beam_x.iloc[idx] * beam_size_scaling / resolution, + height=2.0 * beam_y.iloc[idx] * beam_size_scaling / resolution, + angle=beam_angle.iloc[idx] + ) + ellipse.fill = False + color = color_palette[beam_set_id.iloc[idx] % len(color_palette)] + ellipse.set_edgecolor(color) + axis.add_artist(ellipse) + + # Prepare legend text based on the beam type criteria + if beam_type.iloc[idx] == 'known': + legend_text = f"Beam set{beam_set_id.iloc[idx]}: {beam_type.iloc[idx]}, {nantennas.iloc[idx]} antennas" + axis.annotate( + beam_name.iloc[idx], + xy=coord, + xytext=(coord[0] + 5, coord[1] + 5), # Text position, slightly offset + fontsize=8 + ) + else: + legend_text = f"Beam set{beam_set_id.iloc[idx]}: {beam_name.iloc[idx].split('_')[0]}, {nantennas.iloc[idx]} antennas, {overlap.iloc[idx]} overlap" + # Add an annotation with the beam name. Be careful with the font size, as it can be too large for the plot + if annotate_beam_names: + axis.annotate(beam_name.iloc[idx], xy=coord, xytext=(coord[0], coord[1]), fontsize=extra_source_text_size) + # If the legend entry for this beam_set_id does not exist yet, create it + if beam_set_id.iloc[idx] not in labels: + labels[beam_set_id.iloc[idx]] = (ellipse, legend_text) + + # Create a legend for all beam categories (all types) + handles = [labels[k][0] for k in labels] + legend_texts = [labels[k][1] for k in labels] + axis.legend(handles, legend_texts, loc='upper right') + margin = 1.1 * max(np.sqrt(np.sum(np.square(beam_coordinate), axis=1))) + axis.set_xlim(center[0] - margin, center[0] + margin) + axis.set_ylim(center[1] - margin, center[1] + margin) + axis.set_xlabel('RA', fontsize=30) + axis.set_ylabel('Dec', fontsize=30) + + # Save the original, unzoomed plot + # output_filename_unzoomed = f"{outfile}_original.png" + # log.info(f"Saving original unzoomed plot to {output_filename_unzoomed}") + # plt.tight_layout() + # plt.savefig(output_filename_unzoomed) + + # Now zoom into the tiling beam sets and save separate plots + tiling_df = plot_beams_df[plot_beams_df['type'] == 'tiling'] + + for beam_set, group_df in tiling_df.groupby('beam_set_id'): + equatorialCoordinates_tiling = SkyCoord(group_df['ra'].astype(str), group_df['dec'].astype(str), frame='fk5', unit=(u.hourangle, u.deg)) + beam_coordinate_tiling = np.array(wcs_properties.wcs_world2pix(np.array([equatorialCoordinates_tiling.ra.deg, equatorialCoordinates_tiling.dec.deg]).T, 0)) + # Recalculate margin for this specific beam set + margin = 1.3 * max(np.sqrt(np.sum(np.square(beam_coordinate_tiling), axis=1))) + + # Set the new axis limits for zooming into this region + axis.set_xlim(center[0] - margin, center[0] + margin) + axis.set_ylim(center[1] - margin, center[1] + margin) + + # Save the zoomed plot for the current beam_set_id + output_filename_zoomed = f"{outfile}_tiling_beamset_{beam_set}.png" + log.info(f"Saving zoomed plot for beam_set_id {beam_set} to {output_filename_zoomed}") + plt.title(f"Boresight: {pointing.phase_centre.name}, UTC Start {pointing.start_epoch.isot}, Zoomed into Beam set {beam_set}") + plt.tight_layout() + plt.savefig(output_filename_zoomed) + + +class BeamOverlapDetector: + def __init__(self, neighbour_df, wcs_properties): + self.neighbour_df = neighbour_df + self.wcs_properties = wcs_properties + self._convert_coordinates() + self._convert_to_pixel_coordinates() + + def _convert_coordinates(self): + # Convert RA and Dec from strings to degrees + coords = SkyCoord(ra=self.neighbour_df['ra'].values, dec=self.neighbour_df['dec'].values, unit=(u.hourangle, u.deg)) + self.neighbour_df['ra_deg'] = coords.ra.deg + self.neighbour_df['dec_deg'] = coords.dec.deg + + def _convert_to_pixel_coordinates(self): + # Convert RA and Dec to pixel coordinates using WCS + sky_coords = SkyCoord(ra=self.neighbour_df['ra_deg'].values, dec=self.neighbour_df['dec_deg'].values, unit='deg') + pixel_coords = np.array(self.wcs_properties.wcs_world2pix(np.array([sky_coords.ra.deg, sky_coords.dec.deg]).T, 0)) + self.neighbour_df['x_pix'], self.neighbour_df['y_pix'] = pixel_coords[:, 0], pixel_coords[:, 1] + + + def ellipse_parametric(self, t, a, b, x0, y0, theta): + cos_t = np.cos(t) + sin_t = np.sin(t) + x = x0 + a * cos_t * np.cos(theta) - b * sin_t * np.sin(theta) + y = y0 + a * cos_t * np.sin(theta) + b * sin_t * np.cos(theta) + return x, y + + def point_in_ellipse(self, x, y, ellipse): + x0, y0, a, b, theta = ellipse["x0"], ellipse["y0"], ellipse["a"], ellipse["b"], ellipse["theta"] + cos_theta = np.cos(-theta) + sin_theta = np.sin(-theta) + xr = cos_theta * (x - x0) - sin_theta * (y - y0) + yr = sin_theta * (x - x0) + cos_theta * (y - y0) + return (xr**2 / a**2) + (yr**2 / b**2) <= 1 + + def check_containment(self, ellipse1, ellipse2): + # Check if the center of ellipse1 is inside ellipse2 + x0, y0 = ellipse1["x0"], ellipse1["y0"] + return self.point_in_ellipse(x0, y0, ellipse2) + + def discrete_overlap(self, ellipse1, ellipse2, num_points=100): + # First, check if one ellipse contains the center of the other + if self.check_containment(ellipse1, ellipse2) or self.check_containment(ellipse2, ellipse1): + return True + + # Check points on the perimeters of both ellipses + t_values = np.linspace(0, 2 * np.pi, num_points) + #psf_x, psf_y -> semi-major axis (a), semi-minor axis (b) of the ellipse + x1, y1 = self.ellipse_parametric(t_values, ellipse1["a"], ellipse1["b"], ellipse1["x0"], ellipse1["y0"], ellipse1["theta"]) + + # Check if any points on the perimeter of ellipse1 lie inside ellipse2 + for x, y in zip(x1, y1): + if self.point_in_ellipse(x, y, ellipse2): + return True + return False + + def find_nearest_neighbours(self, beam, beams, n=6): + # Create a SkyCoord object for the target beam + target_coord = SkyCoord(ra=beam['ra_deg'] * u.deg, dec=beam['dec_deg'] * u.deg) + + # Create SkyCoord objects for all other beams + all_coords = SkyCoord(ra=[b['ra_deg'] for b in beams] * u.deg, + dec=[b['dec_deg'] for b in beams] * u.deg) + + # Calculate separations + separations = target_coord.separation(all_coords) + + # Sort by separation and get the nearest neighbours + nearest_indices = np.argsort(separations)[1:n+1] # Skip the first one (itself) + + # Return the list of nearest beam names in descending order of separation + nearest_neighbours = [beams[i]['name'] for i in nearest_indices] + return nearest_neighbours + + def find_overlapping_beams(self): + beams = [] + for _, row in self.neighbour_df.iterrows(): + beams.append({ + "name": row['name'], + "x0": row['x_pix'], + "y0": row['y_pix'], + "a": row['x'], + "b": row['y'], + "theta": np.radians(row['angle']), + "beam_set_id": row['beam_set_id'], + "ra_deg": row['ra_deg'], + "dec_deg": row['dec_deg'] + }) + beam_overlap_dict = {} + for i, beam1 in enumerate(beams): + # Initialize overlap list for beam1 if not already in the dict + if beam1['name'] not in beam_overlap_dict: + beam_overlap_dict[beam1['name']] = { + "name": beam1['name'], + "x_pix": beam1['x0'], + "y_pix": beam1['y0'], + "a": beam1['a'], + "b": beam1['b'], + "angle": np.degrees(beam1['theta']), + "beam_set_id": beam1['beam_set_id'], + "overlapping_beams": [], # Empty list initially + "neighbouring_beams": [] # Empty list for nearest neighbours + } + + # Inner loop to check overlaps + for j in range(i + 1, len(beams)): # Only check pairs once + beam2 = beams[j] + + # Avoid checking beams with the same beam_set_id. They have same PSF shape + if beam1['beam_set_id'] == beam2['beam_set_id']: + continue + + # Initialize overlap list for beam2 if not already in the dict + if beam2['name'] not in beam_overlap_dict: + beam_overlap_dict[beam2['name']] = { + "name": beam2['name'], + "x_pix": beam2['x0'], + "y_pix": beam2['y0'], + "a": beam2['a'], + "b": beam2['b'], + "angle": np.degrees(beam2['theta']), + "beam_set_id": beam2['beam_set_id'], + "overlapping_beams": [], # Empty list initially + "neighbouring_beams": [] # Empty list for nearest neighbours + } + + # Check if beam1 and beam2 overlap + if self.discrete_overlap(beam1, beam2): + # Update overlap lists for both beams + beam_overlap_dict[beam1['name']]['overlapping_beams'].append(beam2['name']) + beam_overlap_dict[beam2['name']]['overlapping_beams'].append(beam1['name']) + + # Find the 6 nearest neighbours for beam1 only if beam set IDs match + matching_beams = [b for b in beams if b['beam_set_id'] == beam1['beam_set_id']] + nearest_neighbours = self.find_nearest_neighbours(beam1, matching_beams) + beam_overlap_dict[beam1['name']]['neighbouring_beams'] = nearest_neighbours + + # Convert the dictionary to a DataFrame + overlap_results = pd.DataFrame(beam_overlap_dict.values()) + overlap_results = overlap_results.sort_values(by=['beam_set_id', 'name']) + + return overlap_results + + + + + + + def main(): """ What does this thing actually do? From 5ca8addac0aa97f0ac6d2dec41174c0b6bad7da5 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Mon, 21 Oct 2024 05:24:47 +0200 Subject: [PATCH 31/69] removed unused class --- python/skyweaver/skyweaver.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/python/skyweaver/skyweaver.py b/python/skyweaver/skyweaver.py index 38e99d5..61eb04d 100644 --- a/python/skyweaver/skyweaver.py +++ b/python/skyweaver/skyweaver.py @@ -721,24 +721,6 @@ def from_file(cls, config_file: str) -> Self: beam_sets ) - -class BeamSetOutput: - def __init__(self): - self.beam_names = [] - self.beam_set_id = None - self.beam_shape_x = None - self.beam_shape_y = None - self.beam_shape_z = None - self.overlap = None - self.ra = None # Right Ascension - self.dec = None # Declination - - def set_ra_dec(self, ra, dec): - """Set RA and Dec for the beam set.""" - self.ra = ra - self.dec = dec - - def make_tiling( pointing: PointingMetadata, subarray: Subarray, From 8fbb204d7e2cc3dda2e25b5d0b717624d34018a3 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Wed, 6 Nov 2024 19:04:03 +0100 Subject: [PATCH 32/69] changed python delay creation code to plot, even if no known beams are given by user --- .bdm.db | Bin 0 -> 12288 bytes .nextflow/.bdm.db | Bin 0 -> 12288 bytes .nextflow/plr/.bdm.db | Bin 0 -> 12288 bytes Dockerfile.skyweavercpp | 4 ++-- Dockerfile.skyweaverpy | 3 ++- cmake/.bdm.db | Bin 0 -> 12288 bytes cpp/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/CMakeLists.txt | 4 ++-- cpp/skyweaver/benchmark/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/detail/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/src/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/data/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/data/dada_files/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/data/dedispersion/.bdm.db | Bin 0 -> 12288 bytes cpp/skyweaver/test/src/.bdm.db | Bin 0 -> 12288 bytes doc/.bdm.db | Bin 0 -> 12288 bytes examples/.bdm.db | Bin 0 -> 12288 bytes examples/config_files/.bdm.db | Bin 0 -> 12288 bytes examples/metadata_files/.bdm.db | Bin 0 -> 12288 bytes prototyping/.bdm.db | Bin 0 -> 12288 bytes python/.bdm.db | Bin 0 -> 12288 bytes python/skyweaver/.bdm.db | Bin 0 -> 12288 bytes python/skyweaver/__pycache__/.bdm.db | Bin 0 -> 12288 bytes python/skyweaver/skyweaver.py | 9 ++++----- python/skyweaver_test_utils/.bdm.db | Bin 0 -> 12288 bytes python/tests/.bdm.db | Bin 0 -> 12288 bytes 28 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 .bdm.db create mode 100644 .nextflow/.bdm.db create mode 100644 .nextflow/plr/.bdm.db create mode 100644 cmake/.bdm.db create mode 100644 cpp/.bdm.db create mode 100644 cpp/skyweaver/.bdm.db create mode 100644 cpp/skyweaver/benchmark/.bdm.db create mode 100644 cpp/skyweaver/detail/.bdm.db create mode 100644 cpp/skyweaver/src/.bdm.db create mode 100644 cpp/skyweaver/test/.bdm.db create mode 100644 cpp/skyweaver/test/data/.bdm.db create mode 100644 cpp/skyweaver/test/data/dada_files/.bdm.db create mode 100644 cpp/skyweaver/test/data/dedispersion/.bdm.db create mode 100644 cpp/skyweaver/test/src/.bdm.db create mode 100644 doc/.bdm.db create mode 100644 examples/.bdm.db create mode 100644 examples/config_files/.bdm.db create mode 100644 examples/metadata_files/.bdm.db create mode 100644 prototyping/.bdm.db create mode 100644 python/.bdm.db create mode 100644 python/skyweaver/.bdm.db create mode 100644 python/skyweaver/__pycache__/.bdm.db create mode 100644 python/skyweaver_test_utils/.bdm.db create mode 100644 python/tests/.bdm.db diff --git a/.bdm.db b/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..ad9cfd5b69b9ad4dccece7d52f8f73c40ab5ce9e GIT binary patch literal 12288 zcmeHM&2QX96!(54yW4iNEhy@Ws=`%OwTaTj9?$rL3W=m$RZ)`=lC&i%2)S9?wX*gu zwzo+*AQlNuaDW>pB*XzB^-ti!5h3nSgv1|!gv1pI-mGU_JK9P;ml9)je&7_wl;tMe0_ZHq3;|FoiK@fC+5%V zTixZIu6$>;dspT<*^cGd={tLg%vN_6m*jX91a@@P@xwvSp-YD8$c+X8Uze?t4<0y? zBS(O7G<055yqVhk<;(T)85+Yw2IWPA(Xb%oqbW?}LRyq39vQ|lN2ph=&0o4yAGh+3 zMzIrdI69u@hU%Zyi)vSD^A|7H$IZN#o}W2^;WQthepVOCCm)n+XV0B0eYTz0_xlb@ zmpv*BeP?-Pt;=`jirFiJ70-g`2P{r9uw_xt-raj>@^b8|cK!8HR8^1@zTpdNEU9@R#=xQzoen>-PS=fg;!4GM~*-TFG!%4*1R89hK{bc`%uOKh< zlOR7+5WM_M@g@ZK<{dlljht}L2}eOXzwlANG&z7DzcM+1>`TZV<-!4E%IEU{ve$tf zV!qV{5phb>S*d%zZnz+5EftRdbW-MH37N z1_T3w0l|P^KrkQ}5DW+g1Ov|#0~?LDbfKkH8s&1i(x_Ie%~N&Sn$3>qC%zXBBFDvs zLM&ZFilQP@Hx*(i()gPfS6=!-`nfJi-j$y|b-(`c7uwhBZRzC}p?IYt9#s5ptam#> z&s7PwOjA`cA*P{eNJkc7c(*BB`?rRB``3u}ty){U%)=`c!VBRN=k^B&o&LacO~W*a zj!lbTtP=>`Dcenk6L^{GVc_3hBRn*ImdMHcK3!n$E11rba%!(^DXF95R& zu~s;_>y@^2ZfdM+tJk|5+g(>x3{_WD!$8EsnxPv2&)}*{6JB$B^-j4hU7HFwjn}>5 z&{eTwLE;D!2PP24s--gc`u6}o<1Laqyzi9S(!x~u>+ANuv*yQ~S7e!3C8mxIqyf8F z(R7B|{0mTP-r`ciIiUCaQtPAyf1(Kn1OtKr!GK^uFd!HZ3D?HqL``$_xb5MWa%ct ZP)NwF1C)Fts-`L!X@;h28qvUk{vT9hANv3R literal 0 HcmV?d00001 diff --git a/.nextflow/.bdm.db b/.nextflow/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..4ecbe7d1786bb1b6bd37c0995fe46d526466ef5b GIT binary patch literal 12288 zcmeI1-D(p-6vuay*rb)1iUh$@94Muxl(^Z=ry`h0D8>4f1~v3*n@rLnyPLSXX;Ux6 z_;u%n@8GR3;e+@FzJs&bndxp?p-4a|XUMYiKWEN4Gr!qcShDwGm$(>p2ezrZNRyT& zS(Y9mBuP@7UQv4Swm>^EF6dnj^#9h4OIhV!l6n^wThg^}SC^N5ENw6Tq5>Ka00KY& z2mk>f00b@@fk|W`o?cx|O!kimJ|1GrwF!3k{ZXS{ZPwAto%$=pZAf>JgALqv5gVP| z+(ORCG!{vq;_KQ^!j>Yk_r|YIoRguy#9G*Jo`nfUj^J-A@MN24O-bLa!S61Y&d#-+9V76R#i?Zy^YHhd9cjgZ1pq;19`gXm6o;P-$ zR2wbyxZc`AmTqFytiNts~DK%Sa)69LABjyH9F&g+a+}Ud;tU38_lIjEN_Zz75X?J zh~>>#cmjf0(-{j_K+$T3+dr*k28H38K~G5Zf*$u&1ai~4`l}X&Ly}lNAmI&3V)-nE zHzbL5U`)9ScSzG!O_}=B>foV!C^m&o%AW3p2n}ocm^cox+#FkzsgJ`3nmlZv>+}uk zYzQ}q5N;45+H43nh!AcNA^F)5ZV(~dz}`lD>FoXGuiL?IM%Zd&Yh?QV%m+iB9>C9c zrw5R|3E3?YI)Kc0IS(Lv9M~b|TOE=rN9udEu*?Ufi^t+gqJur@KLZ)>+nj#%LjwXp z00;m9AOHk_01yBIKmZ5;0U!VbE-?Y``(I9em*@`~5C8%|00;m9AOHk_01yBIKmZ8* zX9AyE@_p%cst}PQ5jnaTi^Y=3R7#t+l;3=zX{m(QqdxXaXo;*v|LyG>gX2z4DHn?Q zQc2U)qLR-QbH!r0`a61~cI#&(AxXW=`~6<#W7E47rJ|~uR#FwMST4|*LRl%DHI>hr m{(Y3&k}u>dikh!zrAk?;D1QJa;JHr# literal 0 HcmV?d00001 diff --git a/.nextflow/plr/.bdm.db b/.nextflow/plr/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..2211b5050431e4053ce512252953f14d51e6884f GIT binary patch literal 12288 zcmeH~?Q7FO9LLiK*K|@l8AWgm4#t=rVwF>B zr+2>a-w^yK{6qW?{2#m|cekWd7}6Iqen(60`}^JR;XaoHvioeCco-e_9n0{LCe2BS zgmfPvNs==3pQ3+0X6Pi%1-&LB^S@0q(z>3_#j>{=&262Z%q`METaR1%roMxo?rc43 z?Chb3`raC{4GW`|{<5`({L^65@bp5naNa+6g2Kt`AnoEn=ev51u#wa3) z?RP9Z5kc&rW8sP@TFvx&XSJ-TFg+{kiHUC1&O(I7HN8z-m)M@l_GB5OxPc~*8|XemLuoRE z8$<{^BffR^{tEZ);@3lLx3N96LVxkSnFt=hPq%{y zki7}nB^i4FnelQyfb4c)4>3RLm@ILmp;wE`azuJ~B<>`-*q7lwQ1rje=|?{dAOHk_ z01yBIKmZ5;0U!VbfB+Bx0zlvr6Y#(P6S;2^{lNeNKmZ5;0U!VbfB+Bx0zd!=0D=F4 zz{lOhU1>gFNhXrXY$}^hr*pY{UJFJ(9HG#(yzI}YkNxC|JTRS(TB<3UR<2j-RaM)l zmDP<(x$!GG-@N%lmZWa+-K%c#!@mDll#NnJ(`qF}tJW*!TCGx7YUfS$^QM1arG#Ue gDW{ohszffBY|6`2rCzDh!_>4=RVk}gwOXzJ21JCnKL7v# literal 0 HcmV?d00001 diff --git a/Dockerfile.skyweavercpp b/Dockerfile.skyweavercpp index 90df0fc..2b3f093 100644 --- a/Dockerfile.skyweavercpp +++ b/Dockerfile.skyweavercpp @@ -60,7 +60,7 @@ WORKDIR /usr/src/skyweaver COPY . . RUN cmake -S . -B build/ -DARCH=native -DPSRDADA_INCLUDE_DIR=/usr/local/include/psrdada \ -DPSRDADACPP_INCLUDE_DIR=/usr/local/include/psrdada_cpp -DSKYWEAVER_NANTENNAS=64 \ - -DSKYWEAVER_NBEAMS=128 -DSKYWEAVER_NCHANS=64 -DSKYWEAVER_IB_SUBTRACTION=1 -DBUILD_SUBMODULES=1 \ - -DENABLE_TESTING=1 -DENABLE_BENCHMARK=1 &&\ + -DSKYWEAVER_NBEAMS=800 -DSKYWEAVER_NCHANS=64 -DSKYWEAVER_IB_SUBTRACTION=1 -DBUILD_SUBMODULES=1 \ + -DENABLE_TESTING=1 -DENABLE_BENCHMARK=1 -DSKYWEAVER_CB_TSCRUNCH=4 -DSKYWEAVER_IB_TSCRUNCH=4 -DSKYWEAVER_CB_FSCRUNCH=1 -DSKYWEAVER_IB_FSCRUNCH=1 &&\ make -C build/ -j 16 && make -C build/ install diff --git a/Dockerfile.skyweaverpy b/Dockerfile.skyweaverpy index 27077fd..ac7686a 100644 --- a/Dockerfile.skyweaverpy +++ b/Dockerfile.skyweaverpy @@ -20,7 +20,8 @@ RUN apt-get update && apt-get install -y \ # Install Python dependencies RUN pip install --upgrade pip && \ - pip install pytest + pip install pytest && \ + pip install pandas # Copy the rest of your application code into the container diff --git a/cmake/.bdm.db b/cmake/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..dbda81b78cb98217787608da6b38e96752797c7b GIT binary patch literal 12288 zcmeHLL2ukd6!v;I+1=3129+oaRV7zd6jFuYnXx@K6%q+rMTJmGl0u^jA!pawSXtYP z?M)L7NDG1+AdXx(Rfr3Jf-4d~00H9AD+dlp9C@~996L#+UMVxm#Pi;p_r3SN8L#BG z{m!NvJIa0#dUmW3X-=wCq}LQhlB7D!Doj46VNv4(JS&CuirJJ5g_Le6X+(x`CJ^q|>hp_>jY+#0XdF2q zhfCHMg-P^J8U@+a`uz3ljY+E*rSIlJV0lptP(Nu1_4AMA`ofhfl`pnqdvEBla@nKG z(0A6>H@o~~UUd7)t=l`@jqa9mXY1BmYg@a@o88?N#kW02+3DWfSy7VHV9V_EGPRc+ zDfk*1>>j%QAs5COW}+XQE+hA&eEZ(;uq+0>!-%`vu?wLYuvZj%MInh;Nb=Z@<1kXz zH{V`|jDvXJh5C5`L7eR7qUZWaimVFzxFEQGl5uV+2(F)IoSO;~L>tD3r?I@EFpRyT zr6dfBmiSaCU_-81)p_8O==vE!xwRy^e#T{PEs3tbH%xiY%_SMCN;%Ea>N}(OAU_mH zDooEJcP-0sO`^#4W5jCm?BlWmz{?7#GgIKRE?kkja7FGy&bn|#?!pzhi+0w9D{>dE zV82GZboTp|)g3t>jvc?}_+u}#FMl$uq%YuSFQ+dc`z2%#x%2`u<RzQIp$p-mtK|57xU#$V3vD7-WVupJ+aj%YdmhH+`F1)MuB8LkjZDG+H%KFG81)TkX;6#| zovP3@(liZeYk$f~zi0+NzLQQ#`c=RD>9(b67D*>`g4ARZszR#)X}YDVD#Dsh5rr0k qilIk1_@z|=CKLGUjn)+lxW literal 0 HcmV?d00001 diff --git a/cpp/.bdm.db b/cpp/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..c91b928366f3b30bfc88e45704c4df8289c98571 GIT binary patch literal 12288 zcmeH~-D(p-6vuay*la5?l@bI?VW5=SQrhiqJ{7@KLMc|QF@6-Uw#g(NlHJ7JNt=2h zM6GvT_#k}=!I$vDH}Dn2+3d`8H&*CHE~K0xVdsC&oR9gURgcL!Dc36KH6ob>R0y>%^_SzgwLH~k)tc&e7$ z?e`)g+)cz0YH1aXD)Jm-g^WfbnOR!WhUsvjzK30r&YxdY#HYs#RXs){uW1d>WiyV7C1uWwH4zTiqc}TZn#;)3>+IryhAHmmhT6 zQQ57xJrQnE!bXD{nbL+#PM6?yRZf! zK^(th=?Mwqj5?OCkYd$3w0&O74vP-8!=8v}g+26KgmSZ9gI&wQ5lNf?kn~0*aRQdo z8CL$ z(F4feg!~eVJb>JIH6K8JJMf2C9Cbt*0%_niqcR(k6c6N`#1{K9xCd7JZ*%t14+97Q z0U!VbfB+Bx0zd!=00AHX1b_e#xWWYd?|(IQrm!CjAOHk_01yBIKmZ5;0U!VbfB+Eq zUkH5KQ|~Er>5Z5gi>dMHL?V$&rPJnUq=OL%(@bmrjQRLaGqrlJ*D!P0LQ&7_W+9g^ zmW*tn{3|}Ub^BXPQ@X6Qi5 zr6nmGF~)deAhl5}EuKDIKCGn&8u(71!?FFWVOalX9YwLnON*yYl@F`wQ5tSC2zIl^ zK=)@ILjK;HW2Iv!P85E)5!kzJC%)bIXFNvVys&a*jYsB-ZbQ2K>gL*|wRP$G`sJ$^ z*0-fs*0#?{p51Yz&9z&b=Oh-5csGh()ZJkr#kEw!*=^U`=f)tyWc{t9UElq7a{6w2 zf7tBS_kDi39k>`e2JWPddfH$krkp&mgP`wAD_5?qz~gSvbaDUu2D$+|o0}ciV=l&b zVIQOn*JB?J{gQ~fX&$)&?K(!}>&FHqup>evUv z9RMHhzXVo7j^q;Fkz7Kayo7fom++3{679)Lct>&x?}*ii-+HWH(S7^Q8w1Cy zJKmrZ9Y6b4yAT%O+vmdqjFm9{85=6Vxbx{Oz&IV^BIcnUGF47#bd(I6TFMNZgCvu1 zGBcwbILqG6rQ+{|+OKMNYtys8oqc__Sp9kRm6?CdJe*mY{&0F9@9ZB;ZB_nL`AKD^ z{MYi^Wm5XM^xe|Zw~?fqRO&PWQ|ysrK=QwnkF3nY+>d2AIr~_1!1Bk zKAvickKSW#=Xp^uY9s^tLaWtxT6hTPdMkrwGX}Rbsu`+o=`vAeQz5EGW9-U5V(d!u zryqXA+MO?m!b_QMU-s%<_W14vD~{b^E3({oq9~Sb5{=5LhRu|9E5@(=2KZ(8JcGu5 z%QsJo!k07gFFI}e!ByL{TM4dCG~KXN)zlP970b}o7h_ z+iKydZ+WlRYbqKsOxZLI*@By8k%M(CjN5@%J(=ZZvRKD?TDi??blH;;+J`Ailz+*YsOt*3A`T{leKR1h`Q z0>Sc8b^JQz^F!wI_0NgIX^vYdrV7uy7aZIj1a8}3-fI#hEFp@GSi8L_3KmBkD&>aLaPiXI_fc>$GOK1}fhud3K{SJ)q77p4#GQBx*yC`M6-1b2 zcb*+paCpTFTt9H@{;J)s588InjnhIQGS!JiDN&IYssVChnydc?ekpc7Ot#k+nDbdW zv^t`kQ6H45I#fc|WNK+d;plT;1v>MZd5=*Zm3E#H1w9j;3EZ_o&>I9BK_7w<)t{^* z;V7{b1W_{;S*Hf~e3^R=+;>^qo@dFYWXN2cAZ&=OKE4k{iA|`i8=4_&iUP4?cMwC* zy%)J1<{rzDhqvb#bQUR*Y&wmLEUsQsh?Lc}Wm4U=WYwgyq3JPr=>vds-H14;jpF2} z5?8xk(~UKWm@3hsOUN08P)kRG$JjNdN#)j(`Ykve7UcFU!_Jca-9(gAi3pEX6Do?r z(V)~pe z(Oh^8cuZI*q92RDVC~L~D10r`^P`F7A}=Sdf7CQB6sclCuP|{Tv@tWqzXKn26ixu2 zph!R2n-+x|neJ!cqg=1MJqK$BP9(I7VyG&l7MVj$nL@v09^b!VB*ZtAgK(r zia|27$63j=Km{Vx1B2)?bQd!Z55d9==Llg4CGmD;l+YYaxkG;I5EzRhe`JlSh6=e& zQs)xDncgNd`)g$u-z>Me5Ijl00q$j4h8h@{HIbLeOi?`#be8PNyuMu;^|*7<@#}qe z*J<3uvL6&I${bZk;us3%HOpf2ro|(>@g+bnF~Hi%d*8W&sls0l4uU2@ z)w7D%RnxLeFw7qs7IuI+)mkDlB;^rn*TzKQ`Ajc2u|yZg3Xh_!wt))$LzzHDVkalw z1R$%z-Z4>vj!vo8SB>;0puz zP^4K{RYIdCmOjz2M8g)C6oudm5|)J&O(QzR5(d`gETU_QLGW=2%l>3UpM`H2vZco7 Xu{8>f;iNaf;2bJrd4*zOTQmL(E=4di literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/CMakeLists.txt b/cpp/skyweaver/CMakeLists.txt index 76d6ae6..d81e740 100644 --- a/cpp/skyweaver/CMakeLists.txt +++ b/cpp/skyweaver/CMakeLists.txt @@ -18,7 +18,7 @@ set(skyweaver_src src/Timer.cpp src/Transposer.cu src/WeightsManager.cu - src/SkyCleaver.cu + src/SkyCleaver.cpp src/SigprocHeader.cpp ) @@ -138,4 +138,4 @@ if(ENABLE_BENCHMARK) COMMAND beamformer_bench --benchmark_counters_tabular=true WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_subdirectory(benchmark) -endif() \ No newline at end of file +endif() diff --git a/cpp/skyweaver/benchmark/.bdm.db b/cpp/skyweaver/benchmark/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..db73c6c42a37555417a366b82c31ab1ae591ec79 GIT binary patch literal 12288 zcmeHNPm9||6qjUot=+^`LNHwtLZ^g~ZdhE6{#j}XbT^o=bknB0rtQ*0k?pZp*pgjo zoOpXEI3c<9&||M9r-XiqLN7h`T=EV29Xcavw9=Bmo|iXxtoM6w-kR&slj@Ub2=XJd);r5 z&>=rT37L`sMfvFV-EEXC!qAUT+CelM5- zckZaGtGtHCd8lzbTg)qQY z7VOs=ty{O$RkNIB7!*mE__iEOeclxMi=UOo2RClWzZ_D(KPCBe^Ghv{-rL#T?~0ZA zI2fY6FMHhw-2?RKVDI71!4dkRd$f%rKP0Hv{jRr-_-f>0wtAU5^Y*uSF z>mqRl22n;(VXeR*%D6176&OVQY07(Hu3$E+l+&!NA(_+TVo_L8ak>_TYZbHGBuRpZ z;(Sh_zpN@)@T!8%GgBC6U4)`=5sJdaJnJG9g^N%WF4kEWp(tF0B7Yi*sprqHY~Bg^ zaY3R1i56jI-+DTg(+&9ilXL^-Ps047R5oC)yqFs>za8>TES9VZfaVKwQDUcZLN>erd`$4Od-BiTTKu2>~C6=AvQ4f?2cvIriC5HGz`z! z`A^{|`Wic8^CS1@uNT+HThC4m-SW(|JFczsZtAXWyAH;NuIskzS-4|hQ};|el|0Mp kc?ZAITa{I|RFzy+rKE~dk3Iecsvv=-M5Lr7p>0%&%GsUWTCu&0 z?QPOZMJxzz95~R5D*_=Q@lS9pXD$f!%!U7fH@3&Iqpj2fr!u3R^?Yw;zW3fY_DX)c z_0hT;6S+SOeJ7SR;iMpn!rQVe2to~d1v+apFsL#BSFtdD-moStD{nMlc4l@*nE&?p z$+=(WuFXCJz#CV9E5H@t3UCFu0$hQYO@WCtQ)`_&SD)Ou?~?ry3F6Qt5xc&#*;(1{ z$T!zIw`69Mok)&|N4l|`tzN#oBu8W4cftqDZZPZ-x@44&`r*)L>+)6d{yh>BIRuUe zBXU(49ILf1U8+w`(HQP$RD*Ch9+f~bn$koeq|0*RG2=MmNbOZ?t&11ylV;J;I3gjV zqvL6AsQpE~Ag5AmUAR!6G>Tq&Ztetz(|myWMO`Rgec9!J8@rk_MxxKw4)6>X?>FK3rpB^df zH88C9++dG^amq>UgTrCuKFpWzdV6I!?CwQuyA!+MIwSUrpj!mgVhZDX3%#78e*1(b5HJN;FNw>XA`=FFzCrDooBI z-&)3Sn?#Wt#3;+jcMi%1ATJvr&(xqE-NFp{7G}t|Xh*j&L%xL>@-6u27G}t|Fhllh z#8S_GztX%T^68iaT@sA_bbj#*PfQ-budXByVD?LxU8K?jm>FNr2Qa%GvWJ)*bqT6W z(sWiULtKC{ImquMSWIDh4=mDm^IGGP(EO>{ng4tK^ZC~re*ol-E5H@t3UCFu0$c&E z09Sx3z!l&Ma0UKL1#Z^b!j-08nUy4|GFz=y8z<_tH5yy)U=$9!?-Qp-!sYI0)JLkO zX_jhg7$MEHlaDa@@$~Aezlo3Pf-q?RH8*Jg{+RZSnYM7gsinMPk+(o-SSm6sT|-#6 zbfl_C#~Gir3w#nK?0?_>h4$OW+5%#MmI$wpz1V#ZDn++K_#YWU9rux@8;Y%BLj`6F zsfulz8FMQD=Htas`_;CvF*Ve+pgW}1%piW3^jxT%hLJlAZnz@?l_)|TskUkBx`qw# zXPTyB8!UrYaKX=aRFXTDwqQ=pV3jyN{rj7QMQ&s$SjV8Zj&<8qEa;3}`5DO10O<+n zv$TIAwS~8)CP8axclQ%%?sgr|+jXE!mz<|HELf zoD<-~8&`lUz!l&Ma0R#mTmh~CSAZ+P72pbR1^x#LOd|13;Y?GLL`f3UYD}ZqY(YD{ z&`jFG`J^>ZHS4K?eb@DRkHn7a^;AVspj4n(22>5OshGC1@{dG|GPVnYTtOekkiS6DL)!$kVO(2WY- literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/src/.bdm.db b/cpp/skyweaver/src/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..d6a1ed30f03308fb941481998247ccff02948828 GIT binary patch literal 12288 zcmeHMO>Er873QvF?~k-vD~cpGf2I+*jvZH?8P4!0L4ZoK;<~Y8N0y^VMp96_+%=(H z?v~`*mJy^?isY66Ii)_eXmco96hVLjxd-i~?X@Tn1Wnyzfn0lS-jFjCSD=8L4Gwm| zdGqtVH{ZNB1mE6x?TQzINr`v!~4Ih7VJz^=AV7Gy5GoZX?v*>Xy)mF?(eDsz5hFth-gVt^+mphRM zss(UAE3~o#*O)Q#o)bkusI6SNwgMIpqK*gqmm3&Fe6|$(p3hAbyKwGh1kdLdrz07` z^J9zCp^OCDdeQEIu6|bNMg8o^n9$9RLohlv^JwG8R9UT)rKjAVR9TPo& zrx){{4v)#OtC-WothPIhZl{L=Mg{R%q)W$TaFa0f{0NIZ^__d;9RNPw0sBlk$l)cj zBV8gp(k1%gC9)%3B0JJ0^zahdkuH%P;x>|77x!1P@6i3}$n{&UKk6srXW#9W;sAW_ zLL7kNCKQkISOAL7M>7D0JBT2bM?EGINog{wkBc-TM((}TlVCCz6CXIszs(n_uPDu5 zH(#Fl@63lYtnqc@=Z!Pd|D3)*{mj%~ro5?o{nPrb+W%|6tG!nJU-c81T$F-=f`Nj8 zf`Nj8f`Nj8gn`WkRk_e)6SZ==JW-mQoNP>2d1*9mc->(zX#L1_+HSDa8V)-c8yHhU zbwn98Y~41Pt?Yj?xAMdn^?y|1?}D!SRlTeJ?U%e>pI4PrxxiUOWFp(bmd+?62GR-B zsX(k@Ks+W8KL*5)w;op&*3@%Mz38?*_(~5#Z{T0=Ieu)dfsh4|6l2?_2BJ2R(2YL< zbcI8A9;mIu<6+{1&jQS(4XJsLGjKCN=`c z4~GK*Y7oYl$$$XeG>J_xmF%AU8Zb`?dmn~<89wCKMeL0_CIMUn4o){rV4c#Gljr^b!1Gz)?Wv=TULEjElOL_P zPM`mInD~c{P2d6i!#3eesflev5OU#jz*fPNr=jx5uHUMwik)k;#7<)|0XN{vuqnY7 zC6;bk7&93z%YO#&Q^N2#z+bJY%E?^t4M>e7-T|I$La-npniyEZa7~;iSL)cn(nI zWyJmM!RE1}Zu(8v>)wvS>~(}}8)BMS$Tks#iGfmoTlg6OgFnFKnb7>7?a8D3O}s+5 zD{ltyk9Bf{gkqv&2oIZp0%}>N48Vm;hp-=RPaJjCanNg{XgG>)L;?77LI}c?!P^Pz zwn-oi!9^q?PrLxwGhp`=kP*Av8)fbudDrN5Z|Fki%v~M$9j`0ieI{iKm$Iv7{n{7*H_5Fp3bin5{#;l3*PYt&Bp;HcX2^-Y&n7k3t$Go@WtC XbplDx@q!I%K}hwUz3e~05jb%~2oU1RiGP4sc2}p{^au`|pi3!rz4~~+ zdcXHxB~`ih!k!noW`7)voXDicd81S^o-s|sFltaMP<2^?MpX;&Ev4FbYu1cS^kfr8 zmzHlDYwtDAul#rA^72<86s2IGV4z^2V4z^2V4z^&x5mI}d8xMX&_ngpm*4Q*+Y{H1 z0?!TgcW1w|b4@UL>QWWe|Y8wwfTt985QF>ki0?j-O=>DY;=pTP8*Y1Dt z!P2|eBIj`E#^%PKO04~0YkRMwJ@c5?Gk2do=v?mXo6qm>UftTiX0GDkADlHq@2|=9!{KpO9CwdHz1)dBfQ}W9 z(n2>aD8$qzPn;+ULUViXxoz0-IO=;~zcyeTso7c_dA{0WY=v`@5!{Lng zWH=*(RWnZK+v>TK=qPatY$}-fB3YW1Axy&1^CKMFG;&U|9iTkh0d~$i$l?;+ku1?2 z$r8G_M0X@hbVss;EiTa=$r9ZWpGMmB`23pN9lEbiUBB!4)6snV!rzCbSpvTO#4G{h zlQ902GYJ@X{x%aZ4u?32wW~8C(Ui_dX;x$@F>+6mNP@{qoX5ZgbvIvMe#dBi(0X$1 z>$Nx69%%l*d1Lj5)qk!&(fF+4HI`QXwX(hZ4NNIY!9c-4!9c-4!9c-4!9c;lugbu| zdfT|rvMS5va=B8fR;$g`x+=})&Q<5w-SfgI+>B15K4y$aMhRyUBg7d&0vV?t-Lw6` z$Cc0PhEX20zg+3R-~P9%ubpojJ1v@{^|?cJJ%{_u*#C>`^xWXUg+B~~&7;YrFC>=| z)H5b2#X@jOVubR2F+u~h|3_7)r4aJ=y*=&s;W?sr$y4thfEFWkr$=A}=Zq7}V$>o6 z+tQ|38)45OK3yHOKV4Dv)>_;6Q!d2S=`iyC3>Ve?vZcKvQl)g`y593F%aSveLlB9I`&28*bm6#TA9JGTUI z6(Fsl2vhaivdW3P=#3y)Z%rd_7;Y+W3kz{9q~z3M7~06v0ek)&C|!mqQqj~uo|L5dvebk^%|KmRu9{&LUAS*V=~6-T5VV=-j#L_|71sol|6SAq0h_ zAQ%hAwDR)aqVn!i+qlxgIZ2)4g$#;iNx_9J!O`GvN-5J8tj8cKuV*=N>)g*K-Sq(e zcbv&;(8-VusZA^?SiCV~@;;B1AU zq6mIh#00H;2PjMFV@cirORa_hKSe1RC>SUhC>SUhC>SUhC>SUhC>SUhC>SUh_&qRi z8kC+gHd?e?Dwj)@_glt^5BtE6}P#O_jP;R}WG>0xxL-ZrsR1 zluALRMYg`Ls+ZY&;o{wY@r&EDN2m*L-N80IOv2S1Dp!wJP%X+BMi}0gU=SffiICWa tS2852WkE}F^(KMu(l;L^vw4Crxb#cVk}eotT;aJ=!sY~{oO2snKLVuF5oiDa literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/test/data/.bdm.db b/cpp/skyweaver/test/data/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..a6ad13ad3ab19348c3edb844363b2884bacb0401 GIT binary patch literal 12288 zcmeHM-HzKt6t5?OlX4P3y3$#V$G4n9!DhJ#|I?H?gDb*l%^2b#47fd+FyFuUz`|(w+6c z7=Qr`00Y1PFaQhy1Hi!Z#=y#0>-4T)H&upfP6bY8i3P5-E z_0w~)AYa$hkCW3ToSmE&(mkexr51rx6`ZJmXH<$@#+2p-+B^7Yk5xRQ6T$X0J(6kc>K_r!-zR z6)bpD!TPKe?nM`=s9mI@cJVH{NJZ@;6}5|h(M2k17pV|mBiXw6epP)J_^SoZPH?tJ zEBn@WiC%8NAKolCp!gDshtb%8Liv1dK=C?=O)QtX0bNOHW%ZiSS0Kg9`b}b!Ijr7+ zEq*rFt;ZVs!2kw;0bl?a00w{oU;r2Z27m!z02lxUfPrVrz&-t{_J$R-)(xZ8GTQBS z_p-@Hw@YzBzZ~NvJ}Y*PNj7mpCyLxK4Ew(4M1x?^j~s3F%f{ZTzqP*o(PXFnjp@}t zFHb)GW67`T{NL6r^`HG`+bd)T27m!z02lxUfB|3t7yt%<0bl?a00#aS23BYK+gi^G z4BasF);d33cP*>O#^$C~j?LG~v9V#9l|tT4&m6}z9y|Te9}I%X^TRN5Z7+&^d-v}) zKXtd+nY%n~|NaO~x4wVu*nZ@dS2zf4ez`6?u#3CxIJPZhr|*Q3?FO#>Uy>Vz1K0O` dr!OV9AFv+_r4HAQ+(F>kc3^v6H1MLp{RcVRgHr$i literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/test/data/dada_files/.bdm.db b/cpp/skyweaver/test/data/dada_files/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..c9d5502c5359a6820a0328b4aff41e41d9c146f8 GIT binary patch literal 12288 zcmeHN&u`;I6n30!leS5{Wff7VR(MsdN|&lTGaip;SRv7(XjN^2rCUKO4wc(Dsnysi ziL=|~fRsh_#(@I|t_X1f{s|;5ocIG)NQg5MH#qVf|EOJUqnyi(GV*+HX1@34n;A=f z%A*gq+`!RW-JWL$x}jWG3I*kDT~`#P47CU~DO1oWB>^=($cmdA_sV zSl?;rA8s~2(i59*_w~NhaSj7L9=&#FRqqcx&+a{4bNz1935T3StyZt=CF9aj`jaE4 z=jc7)czWWzh1%E3_2uQta6XLTaZJ_jbq6Ookc}qSkqPOJKJplITyR91rE>lD?aENg zIvVtyUP2ePU*smVPwEBR#d3XVsWPl(y)@m_2{d2i1I&}UKxdz+<(C&13tv78?1PRI zFE@S^WA>f(jjcwqGq1Z%ee=DY#{I^&{&0Kq-SzD~{Xt`IRrhVr(RUiVJF9xQ8*ww* zy~yl^TMBa-h0z_?KTg0P;tcKW^Jd@uG#!4>InKlG;c-8iZU-*7j)}c2ILw03V^j&soUBZ1Q19CZCi%d|HKUH_mn(mn0YVOZ5j(l8*8@Qb^n3mCr%Kr7Aq*1Sthcp8N=QKS@FgS$K8CVIw&1b7iiuS$sVD9g^ujlU7{-~YR z-l+ao9e`ODsen{KDj*e*3P=T{0#X5~fK=fBrNG141?4r(C{C+tv8a|xrCPNTmRhZ8 z2liUiZrUwk5{n3CnS=^%31Km+48Oav@#@dZx>8Y;m(=#c^J?qcU;hm2-pqn>S2HhR z+&?@T_{aNJ#Ec2HIHuI1h;u}_g$a+Ti~okyh37y$n^I3tKuuz512NMO1|ghU$RbE! zF^;-20X2%Lv4BtoCJ33rFa@EOHIBMG0X0seHnCwa!A-*u)F9L}#!&B0h18erUaYkd zQG*^78J58)H<3j-;$xYwO~A}zW)OpOVPK13K`g`s8x!+hX#zRSSPp_HMu-za7*tFV zLgT256Htqo+7v=yW-$&Y!w>`x`PlQJPC(5QYQ`}!xM5-@EKadyjyoT_g$bml5w*!U z0ktU?oSJYSIURRj!tejW+&>EZWRVI;1*8H}0jYpgKq?>=kP1izqyka_slZiJ;Mr8+ zo^new)q<)Piqj=XYSreoI+RxywfT8X(-v=y%8j~K0hB2fvl6Df_D6q@X<}?KVi}xR z96>e-a$qJ||4R)sUsXs>wZ2*SvGvQ@a(m_TA23E#7^BJ{zY$Yi5b356s~lr$OF67b1SX*e!wLAH!h22}Y1p;FEvA4M4x1Q|k5^dEecbBq80 literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/test/data/dedispersion/.bdm.db b/cpp/skyweaver/test/data/dedispersion/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..e72a2446636c9637f0144a7c4aa0253343f92d4c GIT binary patch literal 12288 zcmeHM&u`;I6tZOMbck z?k@lSh4DWPYtkluxdGm#Re{;x^<&C~_z6AV%5j%{vhG<1m;_OQ5)$lB2MtJ1Aky)>t6YhSgeYZLL0U7Kvt&8}jAY z_$(S`KQD{wRBEkTx9amok!4785*VJv0p{~k;3uCbwHr5Y%3s`%o&Lnl(#2dogIT&JvyV@Q5&bxy-k%(oVK6w3c(@Z&pw1TdieOL#jIprE6DN+t2yO4ay$w4K z;t_@Xc>+PqW^?FKpKUSA!Z|4b>a!i^o&unLvg6!UK)^PMk59S0BACQp(NhBBq9;BD zg|%U?Y1TP$2~s~5DEF2i^;0c#ZwXSrKS^}Y-6afJC7Pz&8oJZ?C_fa~RG6Ga9$Ln5 zn?w=yV>Qdkb56(9K@zy5e4ClBD~FC`CP_9o0OO6dX2j2H6(%x;J5A?8P2f*O}J_3C9v z3NUt0@;eD8Gnn23>+IXSy5dUBubSG*FE9{IC?FIN3J3*+0zv_yfKWgvAQTV^2nB=! z|5t&2wJp8YG%CxAQmrV}YPE5-&RU}}fC|`Tb9ewlbAFPinZRuXhQ8@?4p*lEn{FF&2*y?Du zVQ98(63bF`J7a>MyMai43Pe{%U$?(!y|1*Tt$!y9X7OS+IwXKq($P#}Vyqfwhh!YV zr;J0lipd9#$bsaKNxo$r)Ch7NtDa(d^|H~_XN$@9{P(Uak z6c7ps1%v`Z0il3UKqw#-5DEwdE|vn1>+&noTGLQuMUgAZRVYX_n#~rpXIGm^TYWxh z*KaiIse%8m8}+ne;f4_n{2>%63~X5)(>9^$x`lOZ>raK1DK5dsc2YF>`N<38_0PT| znAm#KP0PTnt13}DQ0HL94b{X%(=gTu6bLLR5r9d@wr-egWD{axn0%(qHZ6$K!JCd? b9UV#;tZq_jzy;L}!`6vu+8Wkvo8Z3z2~^yS literal 0 HcmV?d00001 diff --git a/cpp/skyweaver/test/src/.bdm.db b/cpp/skyweaver/test/src/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..e2283a31f947f644e618d2b4b00633b33def48a3 GIT binary patch literal 12288 zcmeHM&2QX96!&hn*-f$=Ck-Jhs!B$slvY*ncs!mlZ6%U)DG_a2(zGRNMdR#xw^nw& zi|x$^2T(voNC*jzy>LKC98e|x0WKUkz=boaIKioKK!`hU{Izzvm3pOkl#%`B z9a|SBKcAQyzdwFq>{k%7mNAeqkTH-kkTH-kkTLMIF|eN>EzTZ2TH3$4ZJ9eg)Ak(8 zbff3eYHeYyCS6~y-H@U&$#5mtY@2mY3RlmcIxV^VPRDTe<}JJ1F!_>R@YZy?ooHRW zO4_+&I;P}+j=i3FPHr74&Yn0?+Mnh&Y=^2^PPgAn2}x^G6A6<}Nj_qz;{cG-C=_Rp zA202TNua)KI*~g5J`9GBKPd__MvAk?j+OQ+NtA{agTU)B3>bb=2>GM;^2NC$M{=KD z^^DE78M+%jMndgt3yT+PQDk1S8q)I0TJ1t@Rl2ged}(2IU3#Ooep<4Pjw!9x-dZ~? z@o0pvLG*&*4i72p)l^vBw(PA)=m$Facx&+LTJOfoH``ljv0LAAqwR)g0d%OilN9Pn zfg>i2+%-JUaiztJmlwg}uGh4{|Hy%^$5%&U$FjMJp%=z(La=OZaXga{EZes@o=Ql7 z*7mjrSe>NM_BzR#l+a4fcmpA!8*B}{j)7C6Wd{VsvnkQC11{s)lxW$TZJ+meIwgZw zKBs|M4YTLniGu=0IeskSt!Ww3#C0v(L!nO{V>dkj%F_ejXE2}~+!77MTcUw@3qH6d z8i=<<1MwF1;Ff40-VzOjw^8JJcz*?cyXHH6)2^F#zZ1-#{Ggrl58&OG{R0@@gyAEf zI)LHe(|G{HbO;Y|6zY_yM3e@zI4!CP(KC1BOoGK+407Nk|2Cf*dnkxsh-W7sOzun` zu6$d$R{pE}S$Tfq=Lu(GcKo~X6m^HyakzK^)iW+E=We3E!1w5{_z&C^FJz zl`)-Bjc6fU`4fOEwyF=x%`boajrZ%5RRM`O@MtXQQOg4dZo6B1JEpN6IHFONX+$Q9 ziWCLuib^BJ!d+18>%I2NZWH>=N>!jk-8gTyjlD~TZM4!HQ6?)ylMz-hVT55h(mnPM z=suJ*`c}CroQaH1h2e&2weEQCBkD0`L{l}YU|FUzVTvAUf97M*&Y3R&9l!H_YoaQg z6VZ?$U+T9#>oxeMUp0+J(s~UMU6ysEG9v515XK?lGk=5j@uck=<2;2z!c(>nQs>c_ zLds}}Ayoy-DF!$eAw2s7AVBs!;2Cs}_uFGtVPQBzzuAN=ZJg)XV?)_!kTeidO-GuB zDd+}v5K%%Gs#^e10mX5OZkDRTg`ogY@$oFzv+C}W(XRK~hSyDCAWU@y(qCnWs7Ti_ zi-It>8aOcs!g{eP;Gy}m<`}ly>!z%S6Q=5_EGsg_DnUr6G~(y*`#}T-sy9X-7lF&0 zpmiHu=-RKRBHZgWG1fE&^vi^5I@Jjrg^-|90}z7zyjQ*7gkC?y2^w}y49@T%+Jhh< zhE<3IQ*}xdCR2@45`w(&2Y{q5ih7|cEDw#s03^OBlA&fQoCc~e@Q4yfR-i7*))zyr z;!wXm!q37mPvWz1!+~GFlToIMj49EHs$v;wgkV_>^%qw_e<7Kx8~LhG8+z8F3w5Xq zDkLeDDIrJ&G{!U(BAk5~&@{N2)pE?1AMHi|U?c*8WXLFf9jUrP`GuiI zH+E$n5P~Z-j5h!N&xsQP{ADd;AY&k7AY&k7AY&k7AY&k7AY&k7AY&k7;QzqDzLPs6 z%!oLj%ja_=emSu+EzUxlnG>g{MNyo4-f!ZxSOO{0O1R`Y^#&9rWSuEcw?tG^pqxg@ z!k>9wDxHS!ZeCgR%UmZv{08xlbEqvqhjORi5yps6h7lq#3k5j^Dq2t*(sU>-LP3;5 nVT&>aRt+8VXJw}P>zJZKEe2|+>85~E1|8``mL?#2(SOQ!c*G_j})8pYvI|JA5?oQ>1uN z9Jr|hh3kSS3hye4AP5cIYq-<4f(La9@Ka>Q|1@j}J@u_N)~>XUg=^nky}tU(>iyOW z40z)Va0WO7oB_@NXMi*Csxh#XRvMj~H=E1Pp84o(io!JZQ9_^Z9rSmP`pV<|{-+9+ zDQ==9Xo7~RQq1n%-BFTR5V-N8=ZDb8_GREPPxDq>bv0&h6XHrOXDJB`Buu$oR4_ME`ycHd94jC8k9U;N4Ulrwe{!^=kku%L`Re83fGjDtHFZ3D;)kac z%yOSZK0Y5N{ukx+lgVinM#IyDn!BlwLl=$%77STHA~H$N-8798WpD6s4`&>uo{#IN z1w<*CO<~}NB*mf%cg_GmBpH_)2KZr?aj9Yujy6e8&tnBFn4|%#sem!7NzVb}8|Ru= zT?Vc|KgrT+; zGZYR{I16(9_E!@z+koHP%Qj%~B`h9NWdjz%S91dvuS2nk=~7pqMk&p;unKhsrf6Qi zNq91W`8%*p&gS)NzY6lt@)LO#4|wAYa0WO7oB_@NXMi)n8Q=_X1~>zp0nWgGhk-}+ zuCOJ;T1%2@HL+f=x7V7awcGDU^98xN!WX8V11#KZ&9n?%*Pv>fKv@2`vG@9K;`c9_ zg7Cq(`{$bX$sY^S2eq!SnY)y#E{9R*^`;9?*MMqUrfsN>Vc1YrRkLun{sZ>b8sGZq zjCy+@b%k59K@yZ|EjB?v9HDtHPErp5&4v&_!+@Hl8@2^BmF8z-hirC_#jbGg((FjV zV-ybDMKnu$^I+m>jsq5u-i84mz@7a`Ko|_ym1CN1DpZQ0B3+R zz!~5Sa0WO7oB_@NXW;+9z%mx!7B*x<5+zB@&+Khk?%=j@L(bZbH?k(L%gtOs|C_Km ziiRVgL7SYtp<~-7v{YNQ;qG4&xp26G58n80VDA0?To`YEeWuUoV)B zr30wrs|5fo$FP6}Oh?rWe4784k>;4Trt3PepbGGHf`uIaZN+`wpm8RO_)i#-=L$aH=yJ=G| zgsAn-3m?KuUqbLDyzmWt1i{(t%yhS{P$VFfGbG*lpEKuUezUs}=+^U1>JilJI+p1n zU71!?Re6Arq9{rBjkB*96Ks+Yf?d_n{BP5wvV!lXnRjAxSGo52>h#ptsrAX9OuzsF zKmZ5;0U!VbfWT!VFpNzkv$M0Q;j05m4tvD*97c@ zs+QC3_aY+PO~esuX$g%g@*JlMX^liOJ3pTqX2ON~E^$OUe|}yW-+s|5IWwNj&dsHU z>2Q?>l||U}yc*1W(GvLM+gS4E%#8YJ%QN>ngm0H$;yin;ytY{pCv%%N(8iNmWxY~G z&#D`b%hg@5MhuZ`som1kRie#LyXfQ!XQJ0fxnH!*7^G@*tbhw_laF6cHav8i|;z> z=m9*wGkO5|n~+~(kq3|)FXscuZwLMmi=&Q6O&|@tdQ=)A>5+lFlh|Tk2KT_C|J$7X z=!XFWfB+Bx0zd!=00AHX1b_e#00KY&2wY+U{_lS^{Y7CP7(f6B00AHX1b_e#00KY& z2mk>f@V^Os*ir8(vzdHMjm6aXWFnDBr!yISG%~>mgsx{&{*3wfC#9#zz_fboUpX4N zoPqIbf&I;4L(At&80X8s5;xXveT%0QrMdm%sQLZ)cWd$eHpV4Q)AfRe_0?kDC=~KV rTsUhgo;CgZDjbY)rW|Lku@bppvN12y^BOktMhR<1NiUR&xP<=zBuBc} literal 0 HcmV?d00001 diff --git a/examples/config_files/.bdm.db b/examples/config_files/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..c6df408cbd73cc91d39da0d324300d6e1e323099 GIT binary patch literal 12288 zcmeHN-HO{r6qaOntxe)OrI?0Gp5qwpn@+wD#B;R+3$5 zoZan3!7a^QFZvFpcYTRMU!s)0LEoV>(#)*1BqVo%h&vsTzRnm zF9$Gy0bl?a00w{oU;r4nYz(ZdwO04`ZF}|YQ%X)2BxM;Txw?LQFx(jp(bs#!Z;;ZV zC`UPo$(SLzdi&lM%9lwJWoO$ooll5ZvM8l#Hc!+#y$YQ?AsIm#k8!pjA3C!)THU*M z?bTbNhNm*rESoPEHBhalW~h{O4;2|@j0=g}Nwd}6*sxcfYNBOMG8In9&r2ipubKr% zjaK*09edTTW|>f(gp1G1!PT#t!g==YG78iuyas}&1)^Y)`@G7 zrX@kzT7xt#xzyGgr0G#y@UG1@%)2T$Ez6pa1$&}5g%_0-d!b$HnBOLOPE&?uPf2uI zSMcC<1@E&|xEEcNLc1u1cIjPoQ3~y%6xzkR=%N(bMJeRlNVP8CUuEAp`EE(lF-ez6 zY2W-YHj4-F=l6>TP`(M}#i~7kQh7NaKzTdJhgfZO4Z4cb(%P#-uL2o4)prtKEMR#L zY>KnF+49*^hgl+GSI+hx!b(*i~VdXmiY{3c&7a!Tv9^rmTZ@gGJ}&_mqWiU%Gky z`|FS9|71Wmsen{KDj*e*3P=T{0#{9enKoBzt*q2%pPhK_lOs2XL(h%G{o!tBv)j=> z-tK&&3!m;pdgPAXeynG`*YBUn8VQ!XdT6Iobf?kjclt)X@3sNofXbE~jZ=C3!hfA1Mm1vr#HE@sOgM3j~QemWZ?wGg|uN#_*%x&-fKqI3>M2 zB}8os+SuCfQNzk8p@}J>(4ZItiU_nV17)1;w@(=7E58&ukF^4)dWkY2;GD$RHxV*v zS|-G{X&8{&*oL-g5rinsIF;{yV4SZPDI+ykwpyuNf|4l<%{H<2$$*eyN{JAp(1H}( zEDI*BWq^!P{cVFwtNi+7k(7V`s|$ZC>`ykSfK)&#AQg}bNCl(1QWgXGvPua~c0dUfgk z>fa2&00w{oU;r2Z27m!z;9X;2sjW0xw{GdnZ=MF^Sxh3D1SFOBPy4;SK@UAX=zWEx z4*4lc$&8FB60>*j?4WcJhJNyVH;CqA!l%T!G)d;6oL9_3&z_KkpoGPE9+OY-^u0!F zYfE3Q@fx0rP}5|-h%2C6O~p_u=?=;=iWuh-S>t-6b^EryG|GtYT@I#x+Z6ccAGF5$&70~kkElPK5z(%=)kO5c-u_`vuFQwQ z7#(~*=-u!2(U<*$&-VI9=t1vj2St8J(4hDAU`Dn6A2c1 zOs=1viD~d%F?~2Yslxf_B$e(y4OnUsI4pxv8SspyB2RrvlN9YAezDIgp3_Od`j-vN zDW5H27(~28(S?6n0zt$pE-WPwL|Mg!sf1Xz89h166_&w_hGk0yOv@H}21>D6t$Ei) z;tC9+oS?#5fkBjWSy(GDh=#L__rhGktgDREysR;a>62nnSW!u~7KLjSv&SS&gNT}< zr_evGDp>HUg7ujztn)5XQMgD&;bNb6k&41aDhe0pyo*#6E>a=hMzVGB{>uAK$+rs< zjYzZzbNlv>Gd0_QFFwjPpm-CCn^xI?LiuiPK=C+;O)QtX0xd~tZndk>DM3n3izkUq z<}iN-w)xpyHC`(04+by*3;+Yb05AXy00Y1PFaQhy1Hb?<01R9-14P|W9vE(IRnxSZ zTCdleS9Cs_O*)($)5-8SCQ0^hd-nCoF(oNw-@NUP;&@^@4(@g>yX$n^w%f67&$N`~ zZ`by3{IT`Bt|-^18-HDyyxe%jul)S4DTa2@4MIL(02lxUfB|3t7yt%<0bl?a00w{o zVBjJdSbneGRW=P*Q#DPkt@4w0(=b|Wtgjo{SpOg!Yiov{E99qc>v6(8sZ*4Vi_J97W|;@|1^Pkj?Ro}FFou8a9)V$*GBAJ{Z8 s7Pw=XHuh|{W4=l9+Fi?W95zo#7PeTJ%CKDvJFeqli)HV4UF_li0G>vRWB>pF literal 0 HcmV?d00001 diff --git a/python/.bdm.db b/python/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..5b29949d35bde710bb557cfeb875939822c80532 GIT binary patch literal 12288 zcmeI0&u`pB6vu7vHtQ4?w?ZodRn$RMRZ>NZf39s15(!vUL_#6m0ud)CJG-&6wikQ0 z-Q|F^Ah>bhUqFZ(2mS?)@E>sC#*sTG#Ek8k*ijKxkL??+*YkaE-g`5ju`Sv6KHQH8 zM&mS#LxL>niX_X@+XzXL)S*X<9=xs5PMZsQmTUe0>UN~Q{zjL2*Vd1u7ruM$%H^Lf z-&y~I3TQw82mk>f00e*l5O~@MtdzA*Z*x;!efl864`(4*)$xrDcwdz#VW^zLguL5>D|1kuC#igd5$xlj`bf^hS?vt ziV?Oty&E^wRkvPc6p13V`=}b!{J16bZ@y4EFI~GPe|wLF$5UJ_w>q_|?8BYi{ebVx zYtabpy*~`@1c&H@!@Yx@!z1)waI}q*Fve&Yd@|fdY&WWIxqGEKW?M>Q2R0o&jgm7i z=4DQ1pPYB|=yNgrczV{9>B(8n!$T6$(pBQPE>7x#Rjf^N5fYN+Xm|haE}d~o#t~gV zUqDLOXfER@VNMtDX}#ir0nWyM~I(59xBNuEau zF{(Ai;i74v$(shc&eC9B4B-Y5!VMzCx){O@B7_@6hLlt~%|RNJqfFm(zpRD^P7&q)p9Y>6DgSPnTmdpY>=8w?!Ja6QMWl6C%}+1iuqKQ4GW z_WmzxnnZulfB+Bx0zd!=00AHX1b_e#00KY&2s|qUR*C$k^s;6tvUpqXYFdxBt39n~ zy;qAy`)X<6|Ei_VmgF=|Muy}1hV5CN@0-5o`1G{%*CqBczeZob$M=6ZMc?hm@W-5Y?`FrES8*o OTQF%^`lUhl^?w10^pYz8 literal 0 HcmV?d00001 diff --git a/python/skyweaver/.bdm.db b/python/skyweaver/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..8f3911772db3f250d64458fb92d49b1d5256f6d8 GIT binary patch literal 12288 zcmeHLO>Y}T7~T&zc1q2*s;s0`RR=|(xQfhvjdvG>M1rg+YJifMmI#SxV~^dDy=!-O zo!A_Zj06V|2aa43;)3`S9Lt#t2W}ks56tY&Y<8m{^&HK};oCvMjv^fFwy((pA!VTOpGQ2V|5B^Z%PxrI!A7ooH89kEB&bj z{!M^rf&syRU_dY+7!V8y2Cf+X$cgoDJcpiJp^e+7UM!9vtOw;HaBZat(a&Qqlky2^UKmu`O9W8 ze6`xx*r+Y*#VmcCCn3|zauD&$rqI9nQmL-py(@oznD{4S#Fonjl|}Dv@9cN@&b*2H zVDH0jXSZ_z9v|#|uzhd@9(9hkK2|cZ(Fp}^S`WB zCGrzZFd!HZ3uJ+&X|+ti|7%t)KY#ZP-SpbzYQTn$>A0?=w@q*RUzOetTq8&C!6$z$_{rPS zZ7=I>7wWV_%YlYx(qp@>>upOj-@*exh!zW)x#w+OE*l17&1H8hU9{R Vgef)yI-X&>mg{+@=R%JR{{d;eq+|d9 literal 0 HcmV?d00001 diff --git a/python/skyweaver/__pycache__/.bdm.db b/python/skyweaver/__pycache__/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..e0ff9839ce4f6d500ed074352837584012b31339 GIT binary patch literal 12288 zcmeHLO>Y}T7~WmGu~TrjRb?fjDjF0;ZK`1Q`vX)#jI1gmP)H+61gf>MC(g*;wb@-K zHb@8~!2whxu5jVNl{@GDjY=Fiae?2!JG(oZ-6%-8Hky&gGtc|Z`}NGuO0$nX+95GU zhvU%qVx$Y#1yK~80yef4}-* z`A-16aRxX8oB_@NXMi)n8Mtf=ETpAc^Y-of;?rY-PbN5sLxLkVzPsDm+UuZ?w>ys! zlOZoc5gy@QjI!0Odz&bl`o0&=TO=6wF9(b2SLZ(@6;wvneSGDcwU!#LUN;Kw7_AYp$=?7jiMsG{PZsN9X5-q4JAH zQN2p7xwckcG>TFBBo6}9^J0Mbq9K&;zLIJyw{D3~AI09m2xsMHLnZURx7FV1u${R} z`e^%uz0QNqF8XkH`~9umee_;ue-j0skI`P|lf6wucO#poyO)Z6x}{)i8L)apf*}Lb z)F+h>PNxz1B42(m8kXU>H;kCM7ZV7bIrfX7R|GU-A<3B+$6t8z0w4j+IM);a36hL+RRKY?Q9L}2m2d+QQoS9Du7Y$(98RhIoPU#Hps_$p-n zJ?-@VFUnU0`0>UW;0$mEI0Kvk&H!hCGr$?(3~&ZG1Dt`&&A?(Nz9p>5x+F@HnBL(V zvfPArb45b9Lvvim)wceT=$C^k zz1Xil57@<9R}44pmSZZk8?K?dj^e1At|*G_I)Jpp f(sh;vUoT7x+!ULtfHzyy;p2v8I}Uu|(A@t3dBL`5 literal 0 HcmV?d00001 diff --git a/python/skyweaver/skyweaver.py b/python/skyweaver/skyweaver.py index 61eb04d..263999a 100644 --- a/python/skyweaver/skyweaver.py +++ b/python/skyweaver/skyweaver.py @@ -14,8 +14,6 @@ from collections import defaultdict from matplotlib.patches import Ellipse import matplotlib.patches as mpatches -from adjustText import adjust_text - # 3rd party imports import h5py import yaml @@ -703,6 +701,7 @@ def from_file(cls, config_file: str) -> Self: bfc = data["beamformer_config"] beam_sets = [] for bs in data["beam_sets"]: + print(bs) beam_sets.append( BeamSet( bs["name"], @@ -927,9 +926,9 @@ def create_delays( coords = SkyCoord(coords, unit=u.deg) mosaic_command+=f" --tiling_plot {output_prefix}_bid_{beam_set_id}.png --tiling_coordinate {output_prefix}_bid_{beam_set_id}.csv" log.info("Mosaic command written to %s", f"{outfile}.mosaic") - log.info(f"Writing PSF of BeamSet {bs.name} for target {target.name} to {output_prefix}_bid_{beam_set_id}.fits") + log.info(f"Writing PSF of BeamSet {bs.name} to {output_prefix}_bid_{beam_set_id}.fits") psf_beamshape.psf.write_fits(f"{output_prefix}_bid_{beam_set_id}.fits") - log.info(f"PSF Plot of BeamSet {bs.name} for target {target.name} to {output_prefix}_bid_{beam_set_id}.png") + log.info(f"Writing PSF Plot of BeamSet {bs.name} to {output_prefix}_bid_{beam_set_id}.png") psf_beamshape.plot_psf(f"{output_prefix}_bid_{beam_set_id}.png") with open(f"{outfile}.mosaic", "a") as f: f.write(mosaic_command + "\n") @@ -937,7 +936,7 @@ def create_delays( ra_hms = coord.ra.to_string(unit=u.hour, sep=':', precision=2, pad=True) dec_dms = coord.dec.to_string(unit=u.degree, sep=':', precision=1, alwayssign=True, pad=True) plot_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(cb_beamshape[0], 5), round(cb_beamshape[1], 5), round(cb_beamshape[2], 5), current_beam_set_id, overlap, len(sorted_antennas), 'tiling')) - neighbouring_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(psf_beam_shape.axisH, 5), round(psf_beam_shape.axisV, 5), round(psf_beam_shape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'tiling')) + neighbouring_beams.append((f"{bs.name}_{index:03d}", ra_hms, dec_dms, round(psf_beamshape.axisH, 5), round(psf_beamshape.axisV, 5), round(psf_beamshape.angle, 5), current_beam_set_id, 0.5, len(sorted_antennas), 'tiling')) bs_tilings.append(tilings) target = tiling_desc.get("target", None) diff --git a/python/skyweaver_test_utils/.bdm.db b/python/skyweaver_test_utils/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..bfaf50da17f0f961ca0965e65c4c716a1776eadb GIT binary patch literal 12288 zcmeHMPjBNy6t|twxC>eD3Pn_`Rt#FLN>^2}$98P9LZU^{s-i4xH)6LSRJq9{9VK>x zJ!umTD+R>fIB?{O5Vw8_BrbdiBn}+;3@~HQxOP=U&((}NnfH5d-kOHozj0{|37Y0zGyU9=S{)CHiUT9W@uY$#jidrfLxSv^p0{_yIp8)r8j zt^PrQY?1-VfMh^2AQ_MhNCvJO150hC(YkwAUw(0d(diV$B*7>Ze3CvuOpCtEm`DB|QLHMjqorBG|7tTKDhkOQW1HhTqL35WjUz&swvEuFSN!RckZa)KPBPO81d!up~j8D@!8A1w9F;+@3|Piek@Jv{Bn3NrpX|_zCuD%>`o#h! zgv}N(!Z9n6uOgh60FGJ3g{1^=oK;+yN{D6~lhJvus0_v=DqAXGShkRJP>M}!%~xF{ zu0R~;1Qpf_#Bt7LVXZ(MAB{8K3v&h2Rb`y!W%bdN92c8Hi%PP+C|s+UJ|=03V`A|& zMd7@vpuwvOy3SkyFS-at;UW};i+#~WC<+&$C|sP2E<#bb2nBx|iKX-RSHA8PeKkXI z55==6w{LtiRyhvG(!?%7}41W6kFV>%^Ys$J|T@t>J`j~!PPEy2z+JWyt%V~Qy^a8K#nvjQF zOK8X@4LKJf+5KO=`Hw>XWRna?1|$QL0m*=5Kr$d1kPJu$Bme-+m5FC$O{L_Fa>8Xj)zyy1ogaX_`LtT+4%2 n;5fedzmjcQ(4!MAk4u(g*)&SIS*{mYj&J+)J44{R0UiDYG&Z8w literal 0 HcmV?d00001 diff --git a/python/tests/.bdm.db b/python/tests/.bdm.db new file mode 100644 index 0000000000000000000000000000000000000000..ae7ee8a9a50558dd59fb68b09f72265a4d0d79fa GIT binary patch literal 12288 zcmeHK-Hy{n6t{bz4b^Esj=#~rQ-Asa&*a`Mz zcf&=cfXZDj`VLil)t9K$m*_>`ptrs5jP04VS0K155;~(ye7lyv&Nz@BPJex2A*Hq6^iW?fp|Q$G0l-1)P=pS`>M zHv=$$0bl?a00w{oU;r3+-xyeEOO4jmtNP;6VMvaqB&JD7Qt|wJx4YTvq6gdEhe+s9 zkfM}~$pJ;h>W!Q0D4j)7kjyv2c+w|)$uyS+$s`i%XFdu0 zX#1;P_ilF=eY3m$<>u}l`l7qHj^ZF9sMr0jw~qL36k@)6xjN!o%6vVCtsaN*NPt-$ zlk0~kVj4b^%lF5lDx4gQQsExZkfkmHM`dtO20UY_$m4+0Bt=_0UvIIBCv*_9{zU^5 z%2x{*g)uL&=pr~SfiUJ3OG^oaaaOT3l@QA|rlXTwQ5lSBRJK&Wuxz0xpcI?cns+S| zS6~?D1W9WJhH=iNv{qmk?~gOyrMZGxR~e^yS$#64PvoYsqLOScq-zzk*Cb8DnBtF1~32&00Y1PFaQhy1Hb?<01N;FzyL4+47^(g9;%m=TZUU()-xwhGMOt0-&u4``oqw%}9 z#;)Gkt^3cHXNGGpzQ?BH+u8Iwu9;2G@jNz7j7`(z5^iJ9H!at~Z<1^iTWzOf;kJ}) XY_l-sf!(%`osQk{ear8-K70HJ$Jc`7 literal 0 HcmV?d00001 From 0595fa99ea765213501d31f2b68658ee04315f66 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Wed, 6 Nov 2024 19:07:29 +0100 Subject: [PATCH 33/69] Remove .bdm.db files and update .gitignore --- .bdm.db | Bin 12288 -> 0 bytes .nextflow/.bdm.db | Bin 12288 -> 0 bytes .nextflow/plr/.bdm.db | Bin 12288 -> 0 bytes cmake/.bdm.db | Bin 12288 -> 0 bytes cpp/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/benchmark/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/detail/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/src/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/data/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/data/dada_files/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/data/dedispersion/.bdm.db | Bin 12288 -> 0 bytes cpp/skyweaver/test/src/.bdm.db | Bin 12288 -> 0 bytes doc/.bdm.db | Bin 12288 -> 0 bytes examples/.bdm.db | Bin 12288 -> 0 bytes examples/config_files/.bdm.db | Bin 12288 -> 0 bytes examples/metadata_files/.bdm.db | Bin 12288 -> 0 bytes prototyping/.bdm.db | Bin 12288 -> 0 bytes python/.bdm.db | Bin 12288 -> 0 bytes python/skyweaver/.bdm.db | Bin 12288 -> 0 bytes python/skyweaver/__pycache__/.bdm.db | Bin 12288 -> 0 bytes python/skyweaver_test_utils/.bdm.db | Bin 12288 -> 0 bytes python/tests/.bdm.db | Bin 12288 -> 0 bytes 24 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .bdm.db delete mode 100644 .nextflow/.bdm.db delete mode 100644 .nextflow/plr/.bdm.db delete mode 100644 cmake/.bdm.db delete mode 100644 cpp/.bdm.db delete mode 100644 cpp/skyweaver/.bdm.db delete mode 100644 cpp/skyweaver/benchmark/.bdm.db delete mode 100644 cpp/skyweaver/detail/.bdm.db delete mode 100644 cpp/skyweaver/src/.bdm.db delete mode 100644 cpp/skyweaver/test/.bdm.db delete mode 100644 cpp/skyweaver/test/data/.bdm.db delete mode 100644 cpp/skyweaver/test/data/dada_files/.bdm.db delete mode 100644 cpp/skyweaver/test/data/dedispersion/.bdm.db delete mode 100644 cpp/skyweaver/test/src/.bdm.db delete mode 100644 doc/.bdm.db delete mode 100644 examples/.bdm.db delete mode 100644 examples/config_files/.bdm.db delete mode 100644 examples/metadata_files/.bdm.db delete mode 100644 prototyping/.bdm.db delete mode 100644 python/.bdm.db delete mode 100644 python/skyweaver/.bdm.db delete mode 100644 python/skyweaver/__pycache__/.bdm.db delete mode 100644 python/skyweaver_test_utils/.bdm.db delete mode 100644 python/tests/.bdm.db diff --git a/.bdm.db b/.bdm.db deleted file mode 100644 index ad9cfd5b69b9ad4dccece7d52f8f73c40ab5ce9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHM&2QX96!(54yW4iNEhy@Ws=`%OwTaTj9?$rL3W=m$RZ)`=lC&i%2)S9?wX*gu zwzo+*AQlNuaDW>pB*XzB^-ti!5h3nSgv1|!gv1pI-mGU_JK9P;ml9)je&7_wl;tMe0_ZHq3;|FoiK@fC+5%V zTixZIu6$>;dspT<*^cGd={tLg%vN_6m*jX91a@@P@xwvSp-YD8$c+X8Uze?t4<0y? zBS(O7G<055yqVhk<;(T)85+Yw2IWPA(Xb%oqbW?}LRyq39vQ|lN2ph=&0o4yAGh+3 zMzIrdI69u@hU%Zyi)vSD^A|7H$IZN#o}W2^;WQthepVOCCm)n+XV0B0eYTz0_xlb@ zmpv*BeP?-Pt;=`jirFiJ70-g`2P{r9uw_xt-raj>@^b8|cK!8HR8^1@zTpdNEU9@R#=xQzoen>-PS=fg;!4GM~*-TFG!%4*1R89hK{bc`%uOKh< zlOR7+5WM_M@g@ZK<{dlljht}L2}eOXzwlANG&z7DzcM+1>`TZV<-!4E%IEU{ve$tf zV!qV{5phb>S*d%zZnz+5EftRdbW-MH37N z1_T3w0l|P^KrkQ}5DW+g1Ov|#0~?LDbfKkH8s&1i(x_Ie%~N&Sn$3>qC%zXBBFDvs zLM&ZFilQP@Hx*(i()gPfS6=!-`nfJi-j$y|b-(`c7uwhBZRzC}p?IYt9#s5ptam#> z&s7PwOjA`cA*P{eNJkc7c(*BB`?rRB``3u}ty){U%)=`c!VBRN=k^B&o&LacO~W*a zj!lbTtP=>`Dcenk6L^{GVc_3hBRn*ImdMHcK3!n$E11rba%!(^DXF95R& zu~s;_>y@^2ZfdM+tJk|5+g(>x3{_WD!$8EsnxPv2&)}*{6JB$B^-j4hU7HFwjn}>5 z&{eTwLE;D!2PP24s--gc`u6}o<1Laqyzi9S(!x~u>+ANuv*yQ~S7e!3C8mxIqyf8F z(R7B|{0mTP-r`ciIiUCaQtPAyf1(Kn1OtKr!GK^uFd!HZ3D?HqL``$_xb5MWa%ct ZP)NwF1C)Fts-`L!X@;h28qvUk{vT9hANv3R diff --git a/.nextflow/.bdm.db b/.nextflow/.bdm.db deleted file mode 100644 index 4ecbe7d1786bb1b6bd37c0995fe46d526466ef5b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI1-D(p-6vuay*rb)1iUh$@94Muxl(^Z=ry`h0D8>4f1~v3*n@rLnyPLSXX;Ux6 z_;u%n@8GR3;e+@FzJs&bndxp?p-4a|XUMYiKWEN4Gr!qcShDwGm$(>p2ezrZNRyT& zS(Y9mBuP@7UQv4Swm>^EF6dnj^#9h4OIhV!l6n^wThg^}SC^N5ENw6Tq5>Ka00KY& z2mk>f00b@@fk|W`o?cx|O!kimJ|1GrwF!3k{ZXS{ZPwAto%$=pZAf>JgALqv5gVP| z+(ORCG!{vq;_KQ^!j>Yk_r|YIoRguy#9G*Jo`nfUj^J-A@MN24O-bLa!S61Y&d#-+9V76R#i?Zy^YHhd9cjgZ1pq;19`gXm6o;P-$ zR2wbyxZc`AmTqFytiNts~DK%Sa)69LABjyH9F&g+a+}Ud;tU38_lIjEN_Zz75X?J zh~>>#cmjf0(-{j_K+$T3+dr*k28H38K~G5Zf*$u&1ai~4`l}X&Ly}lNAmI&3V)-nE zHzbL5U`)9ScSzG!O_}=B>foV!C^m&o%AW3p2n}ocm^cox+#FkzsgJ`3nmlZv>+}uk zYzQ}q5N;45+H43nh!AcNA^F)5ZV(~dz}`lD>FoXGuiL?IM%Zd&Yh?QV%m+iB9>C9c zrw5R|3E3?YI)Kc0IS(Lv9M~b|TOE=rN9udEu*?Ufi^t+gqJur@KLZ)>+nj#%LjwXp z00;m9AOHk_01yBIKmZ5;0U!VbE-?Y``(I9em*@`~5C8%|00;m9AOHk_01yBIKmZ8* zX9AyE@_p%cst}PQ5jnaTi^Y=3R7#t+l;3=zX{m(QqdxXaXo;*v|LyG>gX2z4DHn?Q zQc2U)qLR-QbH!r0`a61~cI#&(AxXW=`~6<#W7E47rJ|~uR#FwMST4|*LRl%DHI>hr m{(Y3&k}u>dikh!zrAk?;D1QJa;JHr# diff --git a/.nextflow/plr/.bdm.db b/.nextflow/plr/.bdm.db deleted file mode 100644 index 2211b5050431e4053ce512252953f14d51e6884f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeH~?Q7FO9LLiK*K|@l8AWgm4#t=rVwF>B zr+2>a-w^yK{6qW?{2#m|cekWd7}6Iqen(60`}^JR;XaoHvioeCco-e_9n0{LCe2BS zgmfPvNs==3pQ3+0X6Pi%1-&LB^S@0q(z>3_#j>{=&262Z%q`METaR1%roMxo?rc43 z?Chb3`raC{4GW`|{<5`({L^65@bp5naNa+6g2Kt`AnoEn=ev51u#wa3) z?RP9Z5kc&rW8sP@TFvx&XSJ-TFg+{kiHUC1&O(I7HN8z-m)M@l_GB5OxPc~*8|XemLuoRE z8$<{^BffR^{tEZ);@3lLx3N96LVxkSnFt=hPq%{y zki7}nB^i4FnelQyfb4c)4>3RLm@ILmp;wE`azuJ~B<>`-*q7lwQ1rje=|?{dAOHk_ z01yBIKmZ5;0U!VbfB+Bx0zlvr6Y#(P6S;2^{lNeNKmZ5;0U!VbfB+Bx0zd!=0D=F4 zz{lOhU1>gFNhXrXY$}^hr*pY{UJFJ(9HG#(yzI}YkNxC|JTRS(TB<3UR<2j-RaM)l zmDP<(x$!GG-@N%lmZWa+-K%c#!@mDll#NnJ(`qF}tJW*!TCGx7YUfS$^QM1arG#Ue gDW{ohszffBY|6`2rCzDh!_>4=RVk}gwOXzJ21JCnKL7v# diff --git a/cmake/.bdm.db b/cmake/.bdm.db deleted file mode 100644 index dbda81b78cb98217787608da6b38e96752797c7b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHLL2ukd6!v;I+1=3129+oaRV7zd6jFuYnXx@K6%q+rMTJmGl0u^jA!pawSXtYP z?M)L7NDG1+AdXx(Rfr3Jf-4d~00H9AD+dlp9C@~996L#+UMVxm#Pi;p_r3SN8L#BG z{m!NvJIa0#dUmW3X-=wCq}LQhlB7D!Doj46VNv4(JS&CuirJJ5g_Le6X+(x`CJ^q|>hp_>jY+#0XdF2q zhfCHMg-P^J8U@+a`uz3ljY+E*rSIlJV0lptP(Nu1_4AMA`ofhfl`pnqdvEBla@nKG z(0A6>H@o~~UUd7)t=l`@jqa9mXY1BmYg@a@o88?N#kW02+3DWfSy7VHV9V_EGPRc+ zDfk*1>>j%QAs5COW}+XQE+hA&eEZ(;uq+0>!-%`vu?wLYuvZj%MInh;Nb=Z@<1kXz zH{V`|jDvXJh5C5`L7eR7qUZWaimVFzxFEQGl5uV+2(F)IoSO;~L>tD3r?I@EFpRyT zr6dfBmiSaCU_-81)p_8O==vE!xwRy^e#T{PEs3tbH%xiY%_SMCN;%Ea>N}(OAU_mH zDooEJcP-0sO`^#4W5jCm?BlWmz{?7#GgIKRE?kkja7FGy&bn|#?!pzhi+0w9D{>dE zV82GZboTp|)g3t>jvc?}_+u}#FMl$uq%YuSFQ+dc`z2%#x%2`u<RzQIp$p-mtK|57xU#$V3vD7-WVupJ+aj%YdmhH+`F1)MuB8LkjZDG+H%KFG81)TkX;6#| zovP3@(liZeYk$f~zi0+NzLQQ#`c=RD>9(b67D*>`g4ARZszR#)X}YDVD#Dsh5rr0k qilIk1_@z|=CKLGUjn)+lxW diff --git a/cpp/.bdm.db b/cpp/.bdm.db deleted file mode 100644 index c91b928366f3b30bfc88e45704c4df8289c98571..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeH~-D(p-6vuay*la5?l@bI?VW5=SQrhiqJ{7@KLMc|QF@6-Uw#g(NlHJ7JNt=2h zM6GvT_#k}=!I$vDH}Dn2+3d`8H&*CHE~K0xVdsC&oR9gURgcL!Dc36KH6ob>R0y>%^_SzgwLH~k)tc&e7$ z?e`)g+)cz0YH1aXD)Jm-g^WfbnOR!WhUsvjzK30r&YxdY#HYs#RXs){uW1d>WiyV7C1uWwH4zTiqc}TZn#;)3>+IryhAHmmhT6 zQQ57xJrQnE!bXD{nbL+#PM6?yRZf! zK^(th=?Mwqj5?OCkYd$3w0&O74vP-8!=8v}g+26KgmSZ9gI&wQ5lNf?kn~0*aRQdo z8CL$ z(F4feg!~eVJb>JIH6K8JJMf2C9Cbt*0%_niqcR(k6c6N`#1{K9xCd7JZ*%t14+97Q z0U!VbfB+Bx0zd!=00AHX1b_e#xWWYd?|(IQrm!CjAOHk_01yBIKmZ5;0U!VbfB+Eq zUkH5KQ|~Er>5Z5gi>dMHL?V$&rPJnUq=OL%(@bmrjQRLaGqrlJ*D!P0LQ&7_W+9g^ zmW*tn{3|}Ub^BXPQ@X6Qi5 zr6nmGF~)deAhl5}EuKDIKCGn&8u(71!?FFWVOalX9YwLnON*yYl@F`wQ5tSC2zIl^ zK=)@ILjK;HW2Iv!P85E)5!kzJC%)bIXFNvVys&a*jYsB-ZbQ2K>gL*|wRP$G`sJ$^ z*0-fs*0#?{p51Yz&9z&b=Oh-5csGh()ZJkr#kEw!*=^U`=f)tyWc{t9UElq7a{6w2 zf7tBS_kDi39k>`e2JWPddfH$krkp&mgP`wAD_5?qz~gSvbaDUu2D$+|o0}ciV=l&b zVIQOn*JB?J{gQ~fX&$)&?K(!}>&FHqup>evUv z9RMHhzXVo7j^q;Fkz7Kayo7fom++3{679)Lct>&x?}*ii-+HWH(S7^Q8w1Cy zJKmrZ9Y6b4yAT%O+vmdqjFm9{85=6Vxbx{Oz&IV^BIcnUGF47#bd(I6TFMNZgCvu1 zGBcwbILqG6rQ+{|+OKMNYtys8oqc__Sp9kRm6?CdJe*mY{&0F9@9ZB;ZB_nL`AKD^ z{MYi^Wm5XM^xe|Zw~?fqRO&PWQ|ysrK=QwnkF3nY+>d2AIr~_1!1Bk zKAvickKSW#=Xp^uY9s^tLaWtxT6hTPdMkrwGX}Rbsu`+o=`vAeQz5EGW9-U5V(d!u zryqXA+MO?m!b_QMU-s%<_W14vD~{b^E3({oq9~Sb5{=5LhRu|9E5@(=2KZ(8JcGu5 z%QsJo!k07gFFI}e!ByL{TM4dCG~KXN)zlP970b}o7h_ z+iKydZ+WlRYbqKsOxZLI*@By8k%M(CjN5@%J(=ZZvRKD?TDi??blH;;+J`Ailz+*YsOt*3A`T{leKR1h`Q z0>Sc8b^JQz^F!wI_0NgIX^vYdrV7uy7aZIj1a8}3-fI#hEFp@GSi8L_3KmBkD&>aLaPiXI_fc>$GOK1}fhud3K{SJ)q77p4#GQBx*yC`M6-1b2 zcb*+paCpTFTt9H@{;J)s588InjnhIQGS!JiDN&IYssVChnydc?ekpc7Ot#k+nDbdW zv^t`kQ6H45I#fc|WNK+d;plT;1v>MZd5=*Zm3E#H1w9j;3EZ_o&>I9BK_7w<)t{^* z;V7{b1W_{;S*Hf~e3^R=+;>^qo@dFYWXN2cAZ&=OKE4k{iA|`i8=4_&iUP4?cMwC* zy%)J1<{rzDhqvb#bQUR*Y&wmLEUsQsh?Lc}Wm4U=WYwgyq3JPr=>vds-H14;jpF2} z5?8xk(~UKWm@3hsOUN08P)kRG$JjNdN#)j(`Ykve7UcFU!_Jca-9(gAi3pEX6Do?r z(V)~pe z(Oh^8cuZI*q92RDVC~L~D10r`^P`F7A}=Sdf7CQB6sclCuP|{Tv@tWqzXKn26ixu2 zph!R2n-+x|neJ!cqg=1MJqK$BP9(I7VyG&l7MVj$nL@v09^b!VB*ZtAgK(r zia|27$63j=Km{Vx1B2)?bQd!Z55d9==Llg4CGmD;l+YYaxkG;I5EzRhe`JlSh6=e& zQs)xDncgNd`)g$u-z>Me5Ijl00q$j4h8h@{HIbLeOi?`#be8PNyuMu;^|*7<@#}qe z*J<3uvL6&I${bZk;us3%HOpf2ro|(>@g+bnF~Hi%d*8W&sls0l4uU2@ z)w7D%RnxLeFw7qs7IuI+)mkDlB;^rn*TzKQ`Ajc2u|yZg3Xh_!wt))$LzzHDVkalw z1R$%z-Z4>vj!vo8SB>;0puz zP^4K{RYIdCmOjz2M8g)C6oudm5|)J&O(QzR5(d`gETU_QLGW=2%l>3UpM`H2vZco7 Xu{8>f;iNaf;2bJrd4*zOTQmL(E=4di diff --git a/cpp/skyweaver/benchmark/.bdm.db b/cpp/skyweaver/benchmark/.bdm.db deleted file mode 100644 index db73c6c42a37555417a366b82c31ab1ae591ec79..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNPm9||6qjUot=+^`LNHwtLZ^g~ZdhE6{#j}XbT^o=bknB0rtQ*0k?pZp*pgjo zoOpXEI3c<9&||M9r-XiqLN7h`T=EV29Xcavw9=Bmo|iXxtoM6w-kR&slj@Ub2=XJd);r5 z&>=rT37L`sMfvFV-EEXC!qAUT+CelM5- zckZaGtGtHCd8lzbTg)qQY z7VOs=ty{O$RkNIB7!*mE__iEOeclxMi=UOo2RClWzZ_D(KPCBe^Ghv{-rL#T?~0ZA zI2fY6FMHhw-2?RKVDI71!4dkRd$f%rKP0Hv{jRr-_-f>0wtAU5^Y*uSF z>mqRl22n;(VXeR*%D6176&OVQY07(Hu3$E+l+&!NA(_+TVo_L8ak>_TYZbHGBuRpZ z;(Sh_zpN@)@T!8%GgBC6U4)`=5sJdaJnJG9g^N%WF4kEWp(tF0B7Yi*sprqHY~Bg^ zaY3R1i56jI-+DTg(+&9ilXL^-Ps047R5oC)yqFs>za8>TES9VZfaVKwQDUcZLN>erd`$4Od-BiTTKu2>~C6=AvQ4f?2cvIriC5HGz`z! z`A^{|`Wic8^CS1@uNT+HThC4m-SW(|JFczsZtAXWyAH;NuIskzS-4|hQ};|el|0Mp kc?ZAITa{I|RFzy+rKE~dk3Iecsvv=-M5Lr7p>0%&%GsUWTCu&0 z?QPOZMJxzz95~R5D*_=Q@lS9pXD$f!%!U7fH@3&Iqpj2fr!u3R^?Yw;zW3fY_DX)c z_0hT;6S+SOeJ7SR;iMpn!rQVe2to~d1v+apFsL#BSFtdD-moStD{nMlc4l@*nE&?p z$+=(WuFXCJz#CV9E5H@t3UCFu0$hQYO@WCtQ)`_&SD)Ou?~?ry3F6Qt5xc&#*;(1{ z$T!zIw`69Mok)&|N4l|`tzN#oBu8W4cftqDZZPZ-x@44&`r*)L>+)6d{yh>BIRuUe zBXU(49ILf1U8+w`(HQP$RD*Ch9+f~bn$koeq|0*RG2=MmNbOZ?t&11ylV;J;I3gjV zqvL6AsQpE~Ag5AmUAR!6G>Tq&Ztetz(|myWMO`Rgec9!J8@rk_MxxKw4)6>X?>FK3rpB^df zH88C9++dG^amq>UgTrCuKFpWzdV6I!?CwQuyA!+MIwSUrpj!mgVhZDX3%#78e*1(b5HJN;FNw>XA`=FFzCrDooBI z-&)3Sn?#Wt#3;+jcMi%1ATJvr&(xqE-NFp{7G}t|Xh*j&L%xL>@-6u27G}t|Fhllh z#8S_GztX%T^68iaT@sA_bbj#*PfQ-budXByVD?LxU8K?jm>FNr2Qa%GvWJ)*bqT6W z(sWiULtKC{ImquMSWIDh4=mDm^IGGP(EO>{ng4tK^ZC~re*ol-E5H@t3UCFu0$c&E z09Sx3z!l&Ma0UKL1#Z^b!j-08nUy4|GFz=y8z<_tH5yy)U=$9!?-Qp-!sYI0)JLkO zX_jhg7$MEHlaDa@@$~Aezlo3Pf-q?RH8*Jg{+RZSnYM7gsinMPk+(o-SSm6sT|-#6 zbfl_C#~Gir3w#nK?0?_>h4$OW+5%#MmI$wpz1V#ZDn++K_#YWU9rux@8;Y%BLj`6F zsfulz8FMQD=Htas`_;CvF*Ve+pgW}1%piW3^jxT%hLJlAZnz@?l_)|TskUkBx`qw# zXPTyB8!UrYaKX=aRFXTDwqQ=pV3jyN{rj7QMQ&s$SjV8Zj&<8qEa;3}`5DO10O<+n zv$TIAwS~8)CP8axclQ%%?sgr|+jXE!mz<|HELf zoD<-~8&`lUz!l&Ma0R#mTmh~CSAZ+P72pbR1^x#LOd|13;Y?GLL`f3UYD}ZqY(YD{ z&`jFG`J^>ZHS4K?eb@DRkHn7a^;AVspj4n(22>5OshGC1@{dG|GPVnYTtOekkiS6DL)!$kVO(2WY- diff --git a/cpp/skyweaver/src/.bdm.db b/cpp/skyweaver/src/.bdm.db deleted file mode 100644 index d6a1ed30f03308fb941481998247ccff02948828..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHMO>Er873QvF?~k-vD~cpGf2I+*jvZH?8P4!0L4ZoK;<~Y8N0y^VMp96_+%=(H z?v~`*mJy^?isY66Ii)_eXmco96hVLjxd-i~?X@Tn1Wnyzfn0lS-jFjCSD=8L4Gwm| zdGqtVH{ZNB1mE6x?TQzINr`v!~4Ih7VJz^=AV7Gy5GoZX?v*>Xy)mF?(eDsz5hFth-gVt^+mphRM zss(UAE3~o#*O)Q#o)bkusI6SNwgMIpqK*gqmm3&Fe6|$(p3hAbyKwGh1kdLdrz07` z^J9zCp^OCDdeQEIu6|bNMg8o^n9$9RLohlv^JwG8R9UT)rKjAVR9TPo& zrx){{4v)#OtC-WothPIhZl{L=Mg{R%q)W$TaFa0f{0NIZ^__d;9RNPw0sBlk$l)cj zBV8gp(k1%gC9)%3B0JJ0^zahdkuH%P;x>|77x!1P@6i3}$n{&UKk6srXW#9W;sAW_ zLL7kNCKQkISOAL7M>7D0JBT2bM?EGINog{wkBc-TM((}TlVCCz6CXIszs(n_uPDu5 zH(#Fl@63lYtnqc@=Z!Pd|D3)*{mj%~ro5?o{nPrb+W%|6tG!nJU-c81T$F-=f`Nj8 zf`Nj8f`Nj8gn`WkRk_e)6SZ==JW-mQoNP>2d1*9mc->(zX#L1_+HSDa8V)-c8yHhU zbwn98Y~41Pt?Yj?xAMdn^?y|1?}D!SRlTeJ?U%e>pI4PrxxiUOWFp(bmd+?62GR-B zsX(k@Ks+W8KL*5)w;op&*3@%Mz38?*_(~5#Z{T0=Ieu)dfsh4|6l2?_2BJ2R(2YL< zbcI8A9;mIu<6+{1&jQS(4XJsLGjKCN=`c z4~GK*Y7oYl$$$XeG>J_xmF%AU8Zb`?dmn~<89wCKMeL0_CIMUn4o){rV4c#Gljr^b!1Gz)?Wv=TULEjElOL_P zPM`mInD~c{P2d6i!#3eesflev5OU#jz*fPNr=jx5uHUMwik)k;#7<)|0XN{vuqnY7 zC6;bk7&93z%YO#&Q^N2#z+bJY%E?^t4M>e7-T|I$La-npniyEZa7~;iSL)cn(nI zWyJmM!RE1}Zu(8v>)wvS>~(}}8)BMS$Tks#iGfmoTlg6OgFnFKnb7>7?a8D3O}s+5 zD{ltyk9Bf{gkqv&2oIZp0%}>N48Vm;hp-=RPaJjCanNg{XgG>)L;?77LI}c?!P^Pz zwn-oi!9^q?PrLxwGhp`=kP*Av8)fbudDrN5Z|Fki%v~M$9j`0ieI{iKm$Iv7{n{7*H_5Fp3bin5{#;l3*PYt&Bp;HcX2^-Y&n7k3t$Go@WtC XbplDx@q!I%K}hwUz3e~05jb%~2oU1RiGP4sc2}p{^au`|pi3!rz4~~+ zdcXHxB~`ih!k!noW`7)voXDicd81S^o-s|sFltaMP<2^?MpX;&Ev4FbYu1cS^kfr8 zmzHlDYwtDAul#rA^72<86s2IGV4z^2V4z^2V4z^&x5mI}d8xMX&_ngpm*4Q*+Y{H1 z0?!TgcW1w|b4@UL>QWWe|Y8wwfTt985QF>ki0?j-O=>DY;=pTP8*Y1Dt z!P2|eBIj`E#^%PKO04~0YkRMwJ@c5?Gk2do=v?mXo6qm>UftTiX0GDkADlHq@2|=9!{KpO9CwdHz1)dBfQ}W9 z(n2>aD8$qzPn;+ULUViXxoz0-IO=;~zcyeTso7c_dA{0WY=v`@5!{Lng zWH=*(RWnZK+v>TK=qPatY$}-fB3YW1Axy&1^CKMFG;&U|9iTkh0d~$i$l?;+ku1?2 z$r8G_M0X@hbVss;EiTa=$r9ZWpGMmB`23pN9lEbiUBB!4)6snV!rzCbSpvTO#4G{h zlQ902GYJ@X{x%aZ4u?32wW~8C(Ui_dX;x$@F>+6mNP@{qoX5ZgbvIvMe#dBi(0X$1 z>$Nx69%%l*d1Lj5)qk!&(fF+4HI`QXwX(hZ4NNIY!9c-4!9c-4!9c-4!9c;lugbu| zdfT|rvMS5va=B8fR;$g`x+=})&Q<5w-SfgI+>B15K4y$aMhRyUBg7d&0vV?t-Lw6` z$Cc0PhEX20zg+3R-~P9%ubpojJ1v@{^|?cJJ%{_u*#C>`^xWXUg+B~~&7;YrFC>=| z)H5b2#X@jOVubR2F+u~h|3_7)r4aJ=y*=&s;W?sr$y4thfEFWkr$=A}=Zq7}V$>o6 z+tQ|38)45OK3yHOKV4Dv)>_;6Q!d2S=`iyC3>Ve?vZcKvQl)g`y593F%aSveLlB9I`&28*bm6#TA9JGTUI z6(Fsl2vhaivdW3P=#3y)Z%rd_7;Y+W3kz{9q~z3M7~06v0ek)&C|!mqQqj~uo|L5dvebk^%|KmRu9{&LUAS*V=~6-T5VV=-j#L_|71sol|6SAq0h_ zAQ%hAwDR)aqVn!i+qlxgIZ2)4g$#;iNx_9J!O`GvN-5J8tj8cKuV*=N>)g*K-Sq(e zcbv&;(8-VusZA^?SiCV~@;;B1AU zq6mIh#00H;2PjMFV@cirORa_hKSe1RC>SUhC>SUhC>SUhC>SUhC>SUhC>SUh_&qRi z8kC+gHd?e?Dwj)@_glt^5BtE6}P#O_jP;R}WG>0xxL-ZrsR1 zluALRMYg`Ls+ZY&;o{wY@r&EDN2m*L-N80IOv2S1Dp!wJP%X+BMi}0gU=SffiICWa tS2852WkE}F^(KMu(l;L^vw4Crxb#cVk}eotT;aJ=!sY~{oO2snKLVuF5oiDa diff --git a/cpp/skyweaver/test/data/.bdm.db b/cpp/skyweaver/test/data/.bdm.db deleted file mode 100644 index a6ad13ad3ab19348c3edb844363b2884bacb0401..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHM-HzKt6t5?OlX4P3y3$#V$G4n9!DhJ#|I?H?gDb*l%^2b#47fd+FyFuUz`|(w+6c z7=Qr`00Y1PFaQhy1Hi!Z#=y#0>-4T)H&upfP6bY8i3P5-E z_0w~)AYa$hkCW3ToSmE&(mkexr51rx6`ZJmXH<$@#+2p-+B^7Yk5xRQ6T$X0J(6kc>K_r!-zR z6)bpD!TPKe?nM`=s9mI@cJVH{NJZ@;6}5|h(M2k17pV|mBiXw6epP)J_^SoZPH?tJ zEBn@WiC%8NAKolCp!gDshtb%8Liv1dK=C?=O)QtX0bNOHW%ZiSS0Kg9`b}b!Ijr7+ zEq*rFt;ZVs!2kw;0bl?a00w{oU;r2Z27m!z02lxUfPrVrz&-t{_J$R-)(xZ8GTQBS z_p-@Hw@YzBzZ~NvJ}Y*PNj7mpCyLxK4Ew(4M1x?^j~s3F%f{ZTzqP*o(PXFnjp@}t zFHb)GW67`T{NL6r^`HG`+bd)T27m!z02lxUfB|3t7yt%<0bl?a00#aS23BYK+gi^G z4BasF);d33cP*>O#^$C~j?LG~v9V#9l|tT4&m6}z9y|Te9}I%X^TRN5Z7+&^d-v}) zKXtd+nY%n~|NaO~x4wVu*nZ@dS2zf4ez`6?u#3CxIJPZhr|*Q3?FO#>Uy>Vz1K0O` dr!OV9AFv+_r4HAQ+(F>kc3^v6H1MLp{RcVRgHr$i diff --git a/cpp/skyweaver/test/data/dada_files/.bdm.db b/cpp/skyweaver/test/data/dada_files/.bdm.db deleted file mode 100644 index c9d5502c5359a6820a0328b4aff41e41d9c146f8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHN&u`;I6n30!leS5{Wff7VR(MsdN|&lTGaip;SRv7(XjN^2rCUKO4wc(Dsnysi ziL=|~fRsh_#(@I|t_X1f{s|;5ocIG)NQg5MH#qVf|EOJUqnyi(GV*+HX1@34n;A=f z%A*gq+`!RW-JWL$x}jWG3I*kDT~`#P47CU~DO1oWB>^=($cmdA_sV zSl?;rA8s~2(i59*_w~NhaSj7L9=&#FRqqcx&+a{4bNz1935T3StyZt=CF9aj`jaE4 z=jc7)czWWzh1%E3_2uQta6XLTaZJ_jbq6Ookc}qSkqPOJKJplITyR91rE>lD?aENg zIvVtyUP2ePU*smVPwEBR#d3XVsWPl(y)@m_2{d2i1I&}UKxdz+<(C&13tv78?1PRI zFE@S^WA>f(jjcwqGq1Z%ee=DY#{I^&{&0Kq-SzD~{Xt`IRrhVr(RUiVJF9xQ8*ww* zy~yl^TMBa-h0z_?KTg0P;tcKW^Jd@uG#!4>InKlG;c-8iZU-*7j)}c2ILw03V^j&soUBZ1Q19CZCi%d|HKUH_mn(mn0YVOZ5j(l8*8@Qb^n3mCr%Kr7Aq*1Sthcp8N=QKS@FgS$K8CVIw&1b7iiuS$sVD9g^ujlU7{-~YR z-l+ao9e`ODsen{KDj*e*3P=T{0#X5~fK=fBrNG141?4r(C{C+tv8a|xrCPNTmRhZ8 z2liUiZrUwk5{n3CnS=^%31Km+48Oav@#@dZx>8Y;m(=#c^J?qcU;hm2-pqn>S2HhR z+&?@T_{aNJ#Ec2HIHuI1h;u}_g$a+Ti~okyh37y$n^I3tKuuz512NMO1|ghU$RbE! zF^;-20X2%Lv4BtoCJ33rFa@EOHIBMG0X0seHnCwa!A-*u)F9L}#!&B0h18erUaYkd zQG*^78J58)H<3j-;$xYwO~A}zW)OpOVPK13K`g`s8x!+hX#zRSSPp_HMu-za7*tFV zLgT256Htqo+7v=yW-$&Y!w>`x`PlQJPC(5QYQ`}!xM5-@EKadyjyoT_g$bml5w*!U z0ktU?oSJYSIURRj!tejW+&>EZWRVI;1*8H}0jYpgKq?>=kP1izqyka_slZiJ;Mr8+ zo^new)q<)Piqj=XYSreoI+RxywfT8X(-v=y%8j~K0hB2fvl6Df_D6q@X<}?KVi}xR z96>e-a$qJ||4R)sUsXs>wZ2*SvGvQ@a(m_TA23E#7^BJ{zY$Yi5b356s~lr$OF67b1SX*e!wLAH!h22}Y1p;FEvA4M4x1Q|k5^dEecbBq80 diff --git a/cpp/skyweaver/test/data/dedispersion/.bdm.db b/cpp/skyweaver/test/data/dedispersion/.bdm.db deleted file mode 100644 index e72a2446636c9637f0144a7c4aa0253343f92d4c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHM&u`;I6tZOMbck z?k@lSh4DWPYtkluxdGm#Re{;x^<&C~_z6AV%5j%{vhG<1m;_OQ5)$lB2MtJ1Aky)>t6YhSgeYZLL0U7Kvt&8}jAY z_$(S`KQD{wRBEkTx9amok!4785*VJv0p{~k;3uCbwHr5Y%3s`%o&Lnl(#2dogIT&JvyV@Q5&bxy-k%(oVK6w3c(@Z&pw1TdieOL#jIprE6DN+t2yO4ay$w4K z;t_@Xc>+PqW^?FKpKUSA!Z|4b>a!i^o&unLvg6!UK)^PMk59S0BACQp(NhBBq9;BD zg|%U?Y1TP$2~s~5DEF2i^;0c#ZwXSrKS^}Y-6afJC7Pz&8oJZ?C_fa~RG6Ga9$Ln5 zn?w=yV>Qdkb56(9K@zy5e4ClBD~FC`CP_9o0OO6dX2j2H6(%x;J5A?8P2f*O}J_3C9v z3NUt0@;eD8Gnn23>+IXSy5dUBubSG*FE9{IC?FIN3J3*+0zv_yfKWgvAQTV^2nB=! z|5t&2wJp8YG%CxAQmrV}YPE5-&RU}}fC|`Tb9ewlbAFPinZRuXhQ8@?4p*lEn{FF&2*y?Du zVQ98(63bF`J7a>MyMai43Pe{%U$?(!y|1*Tt$!y9X7OS+IwXKq($P#}Vyqfwhh!YV zr;J0lipd9#$bsaKNxo$r)Ch7NtDa(d^|H~_XN$@9{P(Uak z6c7ps1%v`Z0il3UKqw#-5DEwdE|vn1>+&noTGLQuMUgAZRVYX_n#~rpXIGm^TYWxh z*KaiIse%8m8}+ne;f4_n{2>%63~X5)(>9^$x`lOZ>raK1DK5dsc2YF>`N<38_0PT| znAm#KP0PTnt13}DQ0HL94b{X%(=gTu6bLLR5r9d@wr-egWD{axn0%(qHZ6$K!JCd? b9UV#;tZq_jzy;L}!`6vu+8Wkvo8Z3z2~^yS diff --git a/cpp/skyweaver/test/src/.bdm.db b/cpp/skyweaver/test/src/.bdm.db deleted file mode 100644 index e2283a31f947f644e618d2b4b00633b33def48a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHM&2QX96!&hn*-f$=Ck-Jhs!B$slvY*ncs!mlZ6%U)DG_a2(zGRNMdR#xw^nw& zi|x$^2T(voNC*jzy>LKC98e|x0WKUkz=boaIKioKK!`hU{Izzvm3pOkl#%`B z9a|SBKcAQyzdwFq>{k%7mNAeqkTH-kkTH-kkTLMIF|eN>EzTZ2TH3$4ZJ9eg)Ak(8 zbff3eYHeYyCS6~y-H@U&$#5mtY@2mY3RlmcIxV^VPRDTe<}JJ1F!_>R@YZy?ooHRW zO4_+&I;P}+j=i3FPHr74&Yn0?+Mnh&Y=^2^PPgAn2}x^G6A6<}Nj_qz;{cG-C=_Rp zA202TNua)KI*~g5J`9GBKPd__MvAk?j+OQ+NtA{agTU)B3>bb=2>GM;^2NC$M{=KD z^^DE78M+%jMndgt3yT+PQDk1S8q)I0TJ1t@Rl2ged}(2IU3#Ooep<4Pjw!9x-dZ~? z@o0pvLG*&*4i72p)l^vBw(PA)=m$Facx&+LTJOfoH``ljv0LAAqwR)g0d%OilN9Pn zfg>i2+%-JUaiztJmlwg}uGh4{|Hy%^$5%&U$FjMJp%=z(La=OZaXga{EZes@o=Ql7 z*7mjrSe>NM_BzR#l+a4fcmpA!8*B}{j)7C6Wd{VsvnkQC11{s)lxW$TZJ+meIwgZw zKBs|M4YTLniGu=0IeskSt!Ww3#C0v(L!nO{V>dkj%F_ejXE2}~+!77MTcUw@3qH6d z8i=<<1MwF1;Ff40-VzOjw^8JJcz*?cyXHH6)2^F#zZ1-#{Ggrl58&OG{R0@@gyAEf zI)LHe(|G{HbO;Y|6zY_yM3e@zI4!CP(KC1BOoGK+407Nk|2Cf*dnkxsh-W7sOzun` zu6$d$R{pE}S$Tfq=Lu(GcKo~X6m^HyakzK^)iW+E=We3E!1w5{_z&C^FJz zl`)-Bjc6fU`4fOEwyF=x%`boajrZ%5RRM`O@MtXQQOg4dZo6B1JEpN6IHFONX+$Q9 ziWCLuib^BJ!d+18>%I2NZWH>=N>!jk-8gTyjlD~TZM4!HQ6?)ylMz-hVT55h(mnPM z=suJ*`c}CroQaH1h2e&2weEQCBkD0`L{l}YU|FUzVTvAUf97M*&Y3R&9l!H_YoaQg z6VZ?$U+T9#>oxeMUp0+J(s~UMU6ysEG9v515XK?lGk=5j@uck=<2;2z!c(>nQs>c_ zLds}}Ayoy-DF!$eAw2s7AVBs!;2Cs}_uFGtVPQBzzuAN=ZJg)XV?)_!kTeidO-GuB zDd+}v5K%%Gs#^e10mX5OZkDRTg`ogY@$oFzv+C}W(XRK~hSyDCAWU@y(qCnWs7Ti_ zi-It>8aOcs!g{eP;Gy}m<`}ly>!z%S6Q=5_EGsg_DnUr6G~(y*`#}T-sy9X-7lF&0 zpmiHu=-RKRBHZgWG1fE&^vi^5I@Jjrg^-|90}z7zyjQ*7gkC?y2^w}y49@T%+Jhh< zhE<3IQ*}xdCR2@45`w(&2Y{q5ih7|cEDw#s03^OBlA&fQoCc~e@Q4yfR-i7*))zyr z;!wXm!q37mPvWz1!+~GFlToIMj49EHs$v;wgkV_>^%qw_e<7Kx8~LhG8+z8F3w5Xq zDkLeDDIrJ&G{!U(BAk5~&@{N2)pE?1AMHi|U?c*8WXLFf9jUrP`GuiI zH+E$n5P~Z-j5h!N&xsQP{ADd;AY&k7AY&k7AY&k7AY&k7AY&k7AY&k7;QzqDzLPs6 z%!oLj%ja_=emSu+EzUxlnG>g{MNyo4-f!ZxSOO{0O1R`Y^#&9rWSuEcw?tG^pqxg@ z!k>9wDxHS!ZeCgR%UmZv{08xlbEqvqhjORi5yps6h7lq#3k5j^Dq2t*(sU>-LP3;5 nVT&>aRt+8VXJw}P>zJZKEe2|+>85~E1|8``mL?#2(SOQ!c*G_j})8pYvI|JA5?oQ>1uN z9Jr|hh3kSS3hye4AP5cIYq-<4f(La9@Ka>Q|1@j}J@u_N)~>XUg=^nky}tU(>iyOW z40z)Va0WO7oB_@NXMi*Csxh#XRvMj~H=E1Pp84o(io!JZQ9_^Z9rSmP`pV<|{-+9+ zDQ==9Xo7~RQq1n%-BFTR5V-N8=ZDb8_GREPPxDq>bv0&h6XHrOXDJB`Buu$oR4_ME`ycHd94jC8k9U;N4Ulrwe{!^=kku%L`Re83fGjDtHFZ3D;)kac z%yOSZK0Y5N{ukx+lgVinM#IyDn!BlwLl=$%77STHA~H$N-8798WpD6s4`&>uo{#IN z1w<*CO<~}NB*mf%cg_GmBpH_)2KZr?aj9Yujy6e8&tnBFn4|%#sem!7NzVb}8|Ru= zT?Vc|KgrT+; zGZYR{I16(9_E!@z+koHP%Qj%~B`h9NWdjz%S91dvuS2nk=~7pqMk&p;unKhsrf6Qi zNq91W`8%*p&gS)NzY6lt@)LO#4|wAYa0WO7oB_@NXMi)n8Q=_X1~>zp0nWgGhk-}+ zuCOJ;T1%2@HL+f=x7V7awcGDU^98xN!WX8V11#KZ&9n?%*Pv>fKv@2`vG@9K;`c9_ zg7Cq(`{$bX$sY^S2eq!SnY)y#E{9R*^`;9?*MMqUrfsN>Vc1YrRkLun{sZ>b8sGZq zjCy+@b%k59K@yZ|EjB?v9HDtHPErp5&4v&_!+@Hl8@2^BmF8z-hirC_#jbGg((FjV zV-ybDMKnu$^I+m>jsq5u-i84mz@7a`Ko|_ym1CN1DpZQ0B3+R zz!~5Sa0WO7oB_@NXW;+9z%mx!7B*x<5+zB@&+Khk?%=j@L(bZbH?k(L%gtOs|C_Km ziiRVgL7SYtp<~-7v{YNQ;qG4&xp26G58n80VDA0?To`YEeWuUoV)B zr30wrs|5fo$FP6}Oh?rWe4784k>;4Trt3PepbGGHf`uIaZN+`wpm8RO_)i#-=L$aH=yJ=G| zgsAn-3m?KuUqbLDyzmWt1i{(t%yhS{P$VFfGbG*lpEKuUezUs}=+^U1>JilJI+p1n zU71!?Re6Arq9{rBjkB*96Ks+Yf?d_n{BP5wvV!lXnRjAxSGo52>h#ptsrAX9OuzsF zKmZ5;0U!VbfWT!VFpNzkv$M0Q;j05m4tvD*97c@ zs+QC3_aY+PO~esuX$g%g@*JlMX^liOJ3pTqX2ON~E^$OUe|}yW-+s|5IWwNj&dsHU z>2Q?>l||U}yc*1W(GvLM+gS4E%#8YJ%QN>ngm0H$;yin;ytY{pCv%%N(8iNmWxY~G z&#D`b%hg@5MhuZ`som1kRie#LyXfQ!XQJ0fxnH!*7^G@*tbhw_laF6cHav8i|;z> z=m9*wGkO5|n~+~(kq3|)FXscuZwLMmi=&Q6O&|@tdQ=)A>5+lFlh|Tk2KT_C|J$7X z=!XFWfB+Bx0zd!=00AHX1b_e#00KY&2wY+U{_lS^{Y7CP7(f6B00AHX1b_e#00KY& z2mk>f@V^Os*ir8(vzdHMjm6aXWFnDBr!yISG%~>mgsx{&{*3wfC#9#zz_fboUpX4N zoPqIbf&I;4L(At&80X8s5;xXveT%0QrMdm%sQLZ)cWd$eHpV4Q)AfRe_0?kDC=~KV rTsUhgo;CgZDjbY)rW|Lku@bppvN12y^BOktMhR<1NiUR&xP<=zBuBc} diff --git a/examples/config_files/.bdm.db b/examples/config_files/.bdm.db deleted file mode 100644 index c6df408cbd73cc91d39da0d324300d6e1e323099..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHN-HO{r6qaOntxe)OrI?0Gp5qwpn@+wD#B;R+3$5 zoZan3!7a^QFZvFpcYTRMU!s)0LEoV>(#)*1BqVo%h&vsTzRnm zF9$Gy0bl?a00w{oU;r4nYz(ZdwO04`ZF}|YQ%X)2BxM;Txw?LQFx(jp(bs#!Z;;ZV zC`UPo$(SLzdi&lM%9lwJWoO$ooll5ZvM8l#Hc!+#y$YQ?AsIm#k8!pjA3C!)THU*M z?bTbNhNm*rESoPEHBhalW~h{O4;2|@j0=g}Nwd}6*sxcfYNBOMG8In9&r2ipubKr% zjaK*09edTTW|>f(gp1G1!PT#t!g==YG78iuyas}&1)^Y)`@G7 zrX@kzT7xt#xzyGgr0G#y@UG1@%)2T$Ez6pa1$&}5g%_0-d!b$HnBOLOPE&?uPf2uI zSMcC<1@E&|xEEcNLc1u1cIjPoQ3~y%6xzkR=%N(bMJeRlNVP8CUuEAp`EE(lF-ez6 zY2W-YHj4-F=l6>TP`(M}#i~7kQh7NaKzTdJhgfZO4Z4cb(%P#-uL2o4)prtKEMR#L zY>KnF+49*^hgl+GSI+hx!b(*i~VdXmiY{3c&7a!Tv9^rmTZ@gGJ}&_mqWiU%Gky z`|FS9|71Wmsen{KDj*e*3P=T{0#{9enKoBzt*q2%pPhK_lOs2XL(h%G{o!tBv)j=> z-tK&&3!m;pdgPAXeynG`*YBUn8VQ!XdT6Iobf?kjclt)X@3sNofXbE~jZ=C3!hfA1Mm1vr#HE@sOgM3j~QemWZ?wGg|uN#_*%x&-fKqI3>M2 zB}8os+SuCfQNzk8p@}J>(4ZItiU_nV17)1;w@(=7E58&ukF^4)dWkY2;GD$RHxV*v zS|-G{X&8{&*oL-g5rinsIF;{yV4SZPDI+ykwpyuNf|4l<%{H<2$$*eyN{JAp(1H}( zEDI*BWq^!P{cVFwtNi+7k(7V`s|$ZC>`ykSfK)&#AQg}bNCl(1QWgXGvPua~c0dUfgk z>fa2&00w{oU;r2Z27m!z;9X;2sjW0xw{GdnZ=MF^Sxh3D1SFOBPy4;SK@UAX=zWEx z4*4lc$&8FB60>*j?4WcJhJNyVH;CqA!l%T!G)d;6oL9_3&z_KkpoGPE9+OY-^u0!F zYfE3Q@fx0rP}5|-h%2C6O~p_u=?=;=iWuh-S>t-6b^EryG|GtYT@I#x+Z6ccAGF5$&70~kkElPK5z(%=)kO5c-u_`vuFQwQ z7#(~*=-u!2(U<*$&-VI9=t1vj2St8J(4hDAU`Dn6A2c1 zOs=1viD~d%F?~2Yslxf_B$e(y4OnUsI4pxv8SspyB2RrvlN9YAezDIgp3_Od`j-vN zDW5H27(~28(S?6n0zt$pE-WPwL|Mg!sf1Xz89h166_&w_hGk0yOv@H}21>D6t$Ei) z;tC9+oS?#5fkBjWSy(GDh=#L__rhGktgDREysR;a>62nnSW!u~7KLjSv&SS&gNT}< zr_evGDp>HUg7ujztn)5XQMgD&;bNb6k&41aDhe0pyo*#6E>a=hMzVGB{>uAK$+rs< zjYzZzbNlv>Gd0_QFFwjPpm-CCn^xI?LiuiPK=C+;O)QtX0xd~tZndk>DM3n3izkUq z<}iN-w)xpyHC`(04+by*3;+Yb05AXy00Y1PFaQhy1Hb?<01R9-14P|W9vE(IRnxSZ zTCdleS9Cs_O*)($)5-8SCQ0^hd-nCoF(oNw-@NUP;&@^@4(@g>yX$n^w%f67&$N`~ zZ`by3{IT`Bt|-^18-HDyyxe%jul)S4DTa2@4MIL(02lxUfB|3t7yt%<0bl?a00w{o zVBjJdSbneGRW=P*Q#DPkt@4w0(=b|Wtgjo{SpOg!Yiov{E99qc>v6(8sZ*4Vi_J97W|;@|1^Pkj?Ro}FFou8a9)V$*GBAJ{Z8 s7Pw=XHuh|{W4=l9+Fi?W95zo#7PeTJ%CKDvJFeqli)HV4UF_li0G>vRWB>pF diff --git a/python/.bdm.db b/python/.bdm.db deleted file mode 100644 index 5b29949d35bde710bb557cfeb875939822c80532..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI0&u`pB6vu7vHtQ4?w?ZodRn$RMRZ>NZf39s15(!vUL_#6m0ud)CJG-&6wikQ0 z-Q|F^Ah>bhUqFZ(2mS?)@E>sC#*sTG#Ek8k*ijKxkL??+*YkaE-g`5ju`Sv6KHQH8 zM&mS#LxL>niX_X@+XzXL)S*X<9=xs5PMZsQmTUe0>UN~Q{zjL2*Vd1u7ruM$%H^Lf z-&y~I3TQw82mk>f00e*l5O~@MtdzA*Z*x;!efl864`(4*)$xrDcwdz#VW^zLguL5>D|1kuC#igd5$xlj`bf^hS?vt ziV?Oty&E^wRkvPc6p13V`=}b!{J16bZ@y4EFI~GPe|wLF$5UJ_w>q_|?8BYi{ebVx zYtabpy*~`@1c&H@!@Yx@!z1)waI}q*Fve&Yd@|fdY&WWIxqGEKW?M>Q2R0o&jgm7i z=4DQ1pPYB|=yNgrczV{9>B(8n!$T6$(pBQPE>7x#Rjf^N5fYN+Xm|haE}d~o#t~gV zUqDLOXfER@VNMtDX}#ir0nWyM~I(59xBNuEau zF{(Ai;i74v$(shc&eC9B4B-Y5!VMzCx){O@B7_@6hLlt~%|RNJqfFm(zpRD^P7&q)p9Y>6DgSPnTmdpY>=8w?!Ja6QMWl6C%}+1iuqKQ4GW z_WmzxnnZulfB+Bx0zd!=00AHX1b_e#00KY&2s|qUR*C$k^s;6tvUpqXYFdxBt39n~ zy;qAy`)X<6|Ei_VmgF=|Muy}1hV5CN@0-5o`1G{%*CqBczeZob$M=6ZMc?hm@W-5Y?`FrES8*o OTQF%^`lUhl^?w10^pYz8 diff --git a/python/skyweaver/.bdm.db b/python/skyweaver/.bdm.db deleted file mode 100644 index 8f3911772db3f250d64458fb92d49b1d5256f6d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHLO>Y}T7~T&zc1q2*s;s0`RR=|(xQfhvjdvG>M1rg+YJifMmI#SxV~^dDy=!-O zo!A_Zj06V|2aa43;)3`S9Lt#t2W}ks56tY&Y<8m{^&HK};oCvMjv^fFwy((pA!VTOpGQ2V|5B^Z%PxrI!A7ooH89kEB&bj z{!M^rf&syRU_dY+7!V8y2Cf+X$cgoDJcpiJp^e+7UM!9vtOw;HaBZat(a&Qqlky2^UKmu`O9W8 ze6`xx*r+Y*#VmcCCn3|zauD&$rqI9nQmL-py(@oznD{4S#Fonjl|}Dv@9cN@&b*2H zVDH0jXSZ_z9v|#|uzhd@9(9hkK2|cZ(Fp}^S`WB zCGrzZFd!HZ3uJ+&X|+ti|7%t)KY#ZP-SpbzYQTn$>A0?=w@q*RUzOetTq8&C!6$z$_{rPS zZ7=I>7wWV_%YlYx(qp@>>upOj-@*exh!zW)x#w+OE*l17&1H8hU9{R Vgef)yI-X&>mg{+@=R%JR{{d;eq+|d9 diff --git a/python/skyweaver/__pycache__/.bdm.db b/python/skyweaver/__pycache__/.bdm.db deleted file mode 100644 index e0ff9839ce4f6d500ed074352837584012b31339..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHLO>Y}T7~WmGu~TrjRb?fjDjF0;ZK`1Q`vX)#jI1gmP)H+61gf>MC(g*;wb@-K zHb@8~!2whxu5jVNl{@GDjY=Fiae?2!JG(oZ-6%-8Hky&gGtc|Z`}NGuO0$nX+95GU zhvU%qVx$Y#1yK~80yef4}-* z`A-16aRxX8oB_@NXMi)n8Mtf=ETpAc^Y-of;?rY-PbN5sLxLkVzPsDm+UuZ?w>ys! zlOZoc5gy@QjI!0Odz&bl`o0&=TO=6wF9(b2SLZ(@6;wvneSGDcwU!#LUN;Kw7_AYp$=?7jiMsG{PZsN9X5-q4JAH zQN2p7xwckcG>TFBBo6}9^J0Mbq9K&;zLIJyw{D3~AI09m2xsMHLnZURx7FV1u${R} z`e^%uz0QNqF8XkH`~9umee_;ue-j0skI`P|lf6wucO#poyO)Z6x}{)i8L)apf*}Lb z)F+h>PNxz1B42(m8kXU>H;kCM7ZV7bIrfX7R|GU-A<3B+$6t8z0w4j+IM);a36hL+RRKY?Q9L}2m2d+QQoS9Du7Y$(98RhIoPU#Hps_$p-n zJ?-@VFUnU0`0>UW;0$mEI0Kvk&H!hCGr$?(3~&ZG1Dt`&&A?(Nz9p>5x+F@HnBL(V zvfPArb45b9Lvvim)wceT=$C^k zz1Xil57@<9R}44pmSZZk8?K?dj^e1At|*G_I)Jpp f(sh;vUoT7x+!ULtfHzyy;p2v8I}Uu|(A@t3dBL`5 diff --git a/python/skyweaver_test_utils/.bdm.db b/python/skyweaver_test_utils/.bdm.db deleted file mode 100644 index bfaf50da17f0f961ca0965e65c4c716a1776eadb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHMPjBNy6t|twxC>eD3Pn_`Rt#FLN>^2}$98P9LZU^{s-i4xH)6LSRJq9{9VK>x zJ!umTD+R>fIB?{O5Vw8_BrbdiBn}+;3@~HQxOP=U&((}NnfH5d-kOHozj0{|37Y0zGyU9=S{)CHiUT9W@uY$#jidrfLxSv^p0{_yIp8)r8j zt^PrQY?1-VfMh^2AQ_MhNCvJO150hC(YkwAUw(0d(diV$B*7>Ze3CvuOpCtEm`DB|QLHMjqorBG|7tTKDhkOQW1HhTqL35WjUz&swvEuFSN!RckZa)KPBPO81d!up~j8D@!8A1w9F;+@3|Piek@Jv{Bn3NrpX|_zCuD%>`o#h! zgv}N(!Z9n6uOgh60FGJ3g{1^=oK;+yN{D6~lhJvus0_v=DqAXGShkRJP>M}!%~xF{ zu0R~;1Qpf_#Bt7LVXZ(MAB{8K3v&h2Rb`y!W%bdN92c8Hi%PP+C|s+UJ|=03V`A|& zMd7@vpuwvOy3SkyFS-at;UW};i+#~WC<+&$C|sP2E<#bb2nBx|iKX-RSHA8PeKkXI z55==6w{LtiRyhvG(!?%7}41W6kFV>%^Ys$J|T@t>J`j~!PPEy2z+JWyt%V~Qy^a8K#nvjQF zOK8X@4LKJf+5KO=`Hw>XWRna?1|$QL0m*=5Kr$d1kPJu$Bme-+m5FC$O{L_Fa>8Xj)zyy1ogaX_`LtT+4%2 n;5fedzmjcQ(4!MAk4u(g*)&SIS*{mYj&J+)J44{R0UiDYG&Z8w diff --git a/python/tests/.bdm.db b/python/tests/.bdm.db deleted file mode 100644 index ae7ee8a9a50558dd59fb68b09f72265a4d0d79fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHK-Hy{n6t{bz4b^Esj=#~rQ-Asa&*a`Mz zcf&=cfXZDj`VLil)t9K$m*_>`ptrs5jP04VS0K155;~(ye7lyv&Nz@BPJex2A*Hq6^iW?fp|Q$G0l-1)P=pS`>M zHv=$$0bl?a00w{oU;r3+-xyeEOO4jmtNP;6VMvaqB&JD7Qt|wJx4YTvq6gdEhe+s9 zkfM}~$pJ;h>W!Q0D4j)7kjyv2c+w|)$uyS+$s`i%XFdu0 zX#1;P_ilF=eY3m$<>u}l`l7qHj^ZF9sMr0jw~qL36k@)6xjN!o%6vVCtsaN*NPt-$ zlk0~kVj4b^%lF5lDx4gQQsExZkfkmHM`dtO20UY_$m4+0Bt=_0UvIIBCv*_9{zU^5 z%2x{*g)uL&=pr~SfiUJ3OG^oaaaOT3l@QA|rlXTwQ5lSBRJK&Wuxz0xpcI?cns+S| zS6~?D1W9WJhH=iNv{qmk?~gOyrMZGxR~e^yS$#64PvoYsqLOScq-zzk*Cb8DnBtF1~32&00Y1PFaQhy1Hb?<01N;FzyL4+47^(g9;%m=TZUU()-xwhGMOt0-&u4``oqw%}9 z#;)Gkt^3cHXNGGpzQ?BH+u8Iwu9;2G@jNz7j7`(z5^iJ9H!at~Z<1^iTWzOf;kJ}) XY_l-sf!(%`osQk{ear8-K70HJ$Jc`7 From 00fa1373005ef21c67d3b78a2d78ef6e8d7e67e2 Mon Sep 17 00:00:00 2001 From: Vishnu Balakrishnan Date: Wed, 6 Nov 2024 19:27:25 +0100 Subject: [PATCH 34/69] typo fixes in include flags, migration from cuh to hpp --- .gitignore | 2 ++ cpp/skyweaver/src/SkyCleaver.cpp | 2 +- cpp/skyweaver/src/skycleaver_cli.cpp | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d230542..01066c7 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,6 @@ eggs/ core* report* CMakeCache.txt +.nextflow +*.bdm.db diff --git a/cpp/skyweaver/src/SkyCleaver.cpp b/cpp/skyweaver/src/SkyCleaver.cpp index 4420950..36bd663 100644 --- a/cpp/skyweaver/src/SkyCleaver.cpp +++ b/cpp/skyweaver/src/SkyCleaver.cpp @@ -7,7 +7,7 @@ #include #include #include "skyweaver/types.cuh" -#include "skyweaver/SkyCleaver.cuh" +#include "skyweaver/SkyCleaver.hpp" namespace fs = std::filesystem; using SkyCleaver = skyweaver::SkyCleaver; diff --git a/cpp/skyweaver/src/skycleaver_cli.cpp b/cpp/skyweaver/src/skycleaver_cli.cpp index 0b56bac..dcdcc1a 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cpp +++ b/cpp/skyweaver/src/skycleaver_cli.cpp @@ -1,4 +1,4 @@ -#include "skyweaver/SkyCleaver.cuh" +#include "skyweaver/SkyCleaver.hpp" #include "skyweaver/SkyCleaverConfig.hpp" #include "skyweaver/logging.hpp" @@ -301,4 +301,4 @@ int main(int argc, char** argv) skyweaver::SkyCleaver skycleaver(config); skycleaver.cleave(); -} \ No newline at end of file +} From e697b31d27b0e6f6a626ace318155d729769cfaf Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Mon, 25 Nov 2024 12:44:25 +0100 Subject: [PATCH 35/69] minor merge changes --- cpp/skyweaver/CMakeLists.txt | 3 +- cpp/skyweaver/DescribedVector.hpp | 10 +- cpp/skyweaver/MultiFileWriter.cuh | 10 +- cpp/skyweaver/SkyCleaver.hpp | 30 +- cpp/skyweaver/SkyCleaverConfig.hpp | 10 +- cpp/skyweaver/detail/MultiFileWriter.cu | 17 +- cpp/skyweaver/{src => detail}/SkyCleaver.cpp | 435 ++++++++++-------- .../detail/file_writer_callbacks.cpp | 2 +- cpp/skyweaver/src/skycleaver_cli.cpp | 93 +++- 9 files changed, 353 insertions(+), 257 deletions(-) rename cpp/skyweaver/{src => detail}/SkyCleaver.cpp (50%) diff --git a/cpp/skyweaver/CMakeLists.txt b/cpp/skyweaver/CMakeLists.txt index d81e740..e2a605a 100644 --- a/cpp/skyweaver/CMakeLists.txt +++ b/cpp/skyweaver/CMakeLists.txt @@ -18,7 +18,6 @@ set(skyweaver_src src/Timer.cpp src/Transposer.cu src/WeightsManager.cu - src/SkyCleaver.cpp src/SigprocHeader.cpp ) @@ -43,6 +42,8 @@ set(skyweaver_inc WeightsManager.cuh MultiFileWriter.cuh SigprocHeader.hpp + SkyCleaver.hpp + SkyCleaverConfig.hpp ) set(SKYWEAVER_LIBRARIES ${CMAKE_PROJECT_NAME} ${DEPENDENCY_LIBRARIES}) diff --git a/cpp/skyweaver/DescribedVector.hpp b/cpp/skyweaver/DescribedVector.hpp index 4950f30..9b11578 100644 --- a/cpp/skyweaver/DescribedVector.hpp +++ b/cpp/skyweaver/DescribedVector.hpp @@ -692,14 +692,16 @@ template using FPAStatsD = DescribedVector, FreqDim, PolnDim, AntennaDim>; -//skycleaver output vectors + +//skycleaver vectors template -using TDBPowersStdH = DescribedVector>, +using TDBPowersStdH = DescribedVector, TimeDim, DispersionDim, - BeamDim>; + BeamDim, + PolnDim>; template -using TFPowersStdH = DescribedVector>, +using TFPowersStdH = DescribedVector, TimeDim, FreqDim>; diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 6533d6a..d107713 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -22,22 +22,24 @@ struct MultiFileWriterConfig { std::string prefix; std::string extension; std::string output_basename; + std::string suffix; MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), - output_dir("default/"), prefix(""), extension("") {}; + output_dir("default/"), prefix(""), extension(""), output_basename(""), suffix("") {}; MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, std::string stokes_mode, std::string output_dir, std::string prefix, - std::string extension) + std::string extension, + std::string suffix) : header_size(header_size), max_file_size(max_file_size), stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), - extension(extension), output_basename("") {}; - + extension(extension), suffix(suffix), output_basename("") {}; + std::string to_string() { return "header_size: " + std::to_string(header_size) + diff --git a/cpp/skyweaver/SkyCleaver.hpp b/cpp/skyweaver/SkyCleaver.hpp index 46449de..8c99caa 100644 --- a/cpp/skyweaver/SkyCleaver.hpp +++ b/cpp/skyweaver/SkyCleaver.hpp @@ -29,18 +29,14 @@ struct BridgeReader { std::string freq; }; // BridgeReader - - +template class SkyCleaver { public: - using InputType = int8_t; - using OutputType = uint8_t; - using InputVectorType = TDBPowersStdH; - using OutputVectorType = TFPowersStdH; using FreqType = std::size_t; // up to the nearest Hz using BeamNumberType = std::size_t; using DMNumberType = std::size_t; + using StokesNumberType = std::size_t; private: SkyCleaverConfig const& _config; @@ -53,17 +49,24 @@ class SkyCleaver ObservationHeader _header; std::vector _beam_filenames; - std::map>>> + std::map< + StokesNumberType, + std::map>>>> _beam_writers; - std::map>> - _beam_data; + + std::map >>> + _beam_data; std::size_t _total_beam_writers; + int _nthreads_read; void init_readers(); void init_writers(); + void read(std::size_t gulp_samples); + void write(); Timer _timer; @@ -71,11 +74,12 @@ class SkyCleaver SkyCleaver(SkyCleaverConfig const& config); SkyCleaver(SkyCleaver const&) = delete; void operator=(SkyCleaver const&) = delete; - void cleave(); }; // class SkyCleaver + } // namespace skyweaver +#include "skyweaver/detail/SkyCleaver.cpp" #endif // SKYWEAVER_SKYCLEAVER_HPP \ No newline at end of file diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index 8b77dcc..6e4f76f 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -25,6 +25,9 @@ class SkyCleaverConfig std::vector _required_beams; std::vector _required_dms; + std::string _out_stokes; + std::vector _stokes_positions; + public: SkyCleaverConfig() @@ -32,7 +35,8 @@ class SkyCleaverConfig _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096), - _start_sample(0), _nsamples_to_read(0), _required_beams({}), _required_dms({}) + _start_sample(0), _nsamples_to_read(0), _required_beams({}), _required_dms({}), + _out_stokes("I"), _stokes_positions({}) { } SkyCleaverConfig(SkyCleaverConfig const&) = delete; @@ -68,6 +72,8 @@ class SkyCleaverConfig } void required_beams(std::vector required_beams) { _required_beams = required_beams; } void required_dms(std::vector required_dms) { _required_dms = required_dms; } + void out_stokes(std::string out_stokes) { _out_stokes = out_stokes; } + void stokes_positions(std::vector stokes_positions) { _stokes_positions = stokes_positions; } @@ -91,6 +97,8 @@ class SkyCleaverConfig std::size_t nsamples_to_read() const { return _nsamples_to_read; } std::vector required_beams() const { return _required_beams; } std::vector required_dms() const { return _required_dms; } + std::string out_stokes() const { return _out_stokes; } + std::vector stokes_positions() const { return _stokes_positions; } diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index cfe8a9e..16b6c05 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -43,13 +43,11 @@ MultiFileWriter::MultiFileWriter( CreateStreamCallBackType create_stream_callback) : _tag(tag), _create_stream_callback(create_stream_callback) { - MultiFileWriterConfig writer_config; - writer_config.header_size = config.dada_header_size(); - writer_config.max_file_size = config.max_output_filesize(); - writer_config.stokes_mode = config.stokes_mode(); - writer_config.base_output_dir = config.output_dir(); + _config.header_size = config.dada_header_size(); + _config.max_file_size = config.max_output_filesize(); + _config.stokes_mode = config.stokes_mode(); + _config.base_output_dir = config.output_dir(); - _config = writer_config; } template @@ -125,7 +123,12 @@ MultiFileWriter::get_basefilename(VectorType const& stream_data, base_filename << get_formatted_time(_header.utc_start) << "_" << stream_idx << "_" << std::fixed << std::setprecision(3) << std::setfill('0') << std::setw(9) - << stream_data.reference_dm(); + << stream_data.reference_dm(); + + if(!_config.suffix.empty()) { + base_filename << "_" << _config.suffix; + } + if(!_tag.empty()) { base_filename << "_" << _tag; } diff --git a/cpp/skyweaver/src/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp similarity index 50% rename from cpp/skyweaver/src/SkyCleaver.cpp rename to cpp/skyweaver/detail/SkyCleaver.cpp index 36bd663..8a4945e 100644 --- a/cpp/skyweaver/src/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -1,23 +1,22 @@ +#include "skyweaver/SkyCleaver.hpp" + +#include "skyweaver/types.cuh" +#include "skyweaver/beamformer_utils.cuh" + #include #include #include #include -#include #include +#include #include -#include "skyweaver/types.cuh" -#include "skyweaver/SkyCleaver.hpp" - -namespace fs = std::filesystem; -using SkyCleaver = skyweaver::SkyCleaver; +namespace fs = std::filesystem; -using FreqType = skyweaver::SkyCleaver::FreqType; using BridgeReader = skyweaver::BridgeReader; using MultiFileReader = skyweaver::MultiFileReader; -using OutputVectorType = skyweaver::SkyCleaver::OutputVectorType; -using InputVectorType = skyweaver::SkyCleaver::InputVectorType; + namespace { @@ -91,14 +90,13 @@ std::vector get_files(std::string directory_path, } } // namespace - -void SkyCleaver::init_readers() +template +void skyweaver::SkyCleaver::init_readers() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_readers") - std::string root_dir = _config.root_dir(); - std::string root_prefix = _config.root_prefix(); - std::size_t stream_id = _config.stream_id(); + std::string root_dir = _config.root_dir(); + std::size_t stream_id = _config.stream_id(); // get the list of directories in root/stream_id(for the nex) std::vector freq_dirs = @@ -108,7 +106,7 @@ void SkyCleaver::init_readers() << "Found " << freq_dirs.size() << " frequency directories in root directory: " << root_dir; - std::map bridge_timestamps; + std::map::FreqType, long double> bridge_timestamps; long double latest_timestamp = 0.0; for(const auto& freq_dir: freq_dirs) { @@ -142,8 +140,8 @@ void SkyCleaver::init_readers() int nbridges = _config.nbridges(); - std::size_t gulp_size = - _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); + // std::size_t gulp_size = + // _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); ObservationHeader const& header = (*_bridge_readers.begin()).second->get_header(); @@ -151,15 +149,13 @@ void SkyCleaver::init_readers() << "Read header from first file: " << header.to_string(); long double obs_centre_freq = header.obs_frequency; - long double obs_bandwidth = header.obs_bandwidth; + long double obs_bandwidth = header.obs_bandwidth; long double start_freq = obs_centre_freq - obs_bandwidth / 2; - - for(int i = 0; i < nbridges; i++) { - - int ifreq = std::lround(std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); + int ifreq = std::lround( + std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) << "Expected frequency [" << i << "]: " << ifreq; @@ -172,13 +168,16 @@ void SkyCleaver::init_readers() _bridge_data[ifreq] = std::make_unique( std::initializer_list{_config.nsamples_per_block(), _config.ndms(), - _config.nbeams()}, + _config.nbeams(), + _config.stokes_mode().size()}, 0); } // print _expected_freqs std::size_t smallest_data_size = std::numeric_limits::max(); + std::size_t dbp_factor = + _config.ndms() * _config.nbeams() * _config.stokes_mode().size(); for(const auto& [freq, reader]: _bridge_readers) { // at this point, all non-existed frequencies have been added with zero @@ -205,15 +204,13 @@ void SkyCleaver::init_readers() << " Number of samples to skip: " << nsamples; BOOST_LOG_TRIVIAL(debug) - << "Seeking " << nsamples * _config.ndms() * _config.nbeams() + << "Seeking " << nsamples * dbp_factor << " bytes in bridge reader for frequency: " << freq; - std::size_t bytes_seeking = (nsamples * _config.ndms() * - _config.nbeams() * - sizeof(InputVectorType::value_type)); + std::size_t bytes_seeking = + (nsamples * dbp_factor * sizeof(typename InputVectorType::value_type)); - _bridge_readers[freq]->seekg(bytes_seeking, - std::ios_base::beg); + _bridge_readers[freq]->seekg(bytes_seeking, std::ios_base::beg); std::size_t data_size = _bridge_readers[freq]->get_total_size() - bytes_seeking; @@ -224,16 +221,16 @@ void SkyCleaver::init_readers() } } - - if(smallest_data_size % (_config.ndms() * _config.nbeams()) != 0) { - std::runtime_error("Data size is not a multiple of ndms * nbeams"); + if(smallest_data_size % dbp_factor != 0) { + std::runtime_error( + "Data size is not a multiple of ndms * nbeams * nstokes"); } - std::size_t smallest_nsamples = - std::floor(smallest_data_size / _config.ndms() / _config.nbeams()); + std::size_t smallest_nsamples = std::floor(smallest_data_size / dbp_factor); - if (smallest_nsamples < _config.start_sample()) { - std::runtime_error("start_sample is greater than the smallest_nsamples in the data."); + if(smallest_nsamples < _config.start_sample()) { + std::runtime_error( + "start_sample is greater than the smallest_nsamples in the data."); } smallest_nsamples = smallest_nsamples - _config.start_sample(); @@ -248,22 +245,18 @@ void SkyCleaver::init_readers() } if(_config.nsamples_to_read() > 0) { - if(smallest_nsamples < _config.nsamples_to_read()) { std::runtime_error( "Smallest data size is less than nsamples_to_read"); } - _nsamples_to_read = _config.nsamples_to_read(); - } - else{ + } else { _nsamples_to_read = smallest_nsamples; } - std::size_t bytes_seeking = - (_config.start_sample() * _config.ndms() * _config.nbeams() * - sizeof(InputVectorType::value_type)); + std::size_t bytes_seeking = (_config.start_sample() * dbp_factor * + sizeof(typename InputVectorType::value_type)); if(bytes_seeking > 0) { BOOST_LOG_TRIVIAL(info) << "Seeking " << bytes_seeking @@ -278,93 +271,102 @@ void SkyCleaver::init_readers() << "Added " << _bridge_data.size() << " bridge readers to SkyCleaver"; _header = _bridge_readers[_available_freqs[0]]->get_header(); - BOOST_LOG_TRIVIAL(info) << "Adding first header to SkyCleaver: " << _header.to_string(); + BOOST_LOG_TRIVIAL(info) + << "Adding first header to SkyCleaver: " << _header.to_string(); _header.nchans = _header.nchans * _config.nbridges(); _header.nbeams = _config.nbeams(); - -} -void SkyCleaver::init_writers() + _nthreads_read = _config.nthreads() > _config.nbridges() + ? _config.nbridges() + : _config.nthreads(); +} +template +void skyweaver::SkyCleaver::init_writers() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_writers") BOOST_LOG_TRIVIAL(debug) << "_config.output_dir(); " << _config.output_dir(); - if(!fs::exists(_config.output_dir())) { fs::create_directories(_config.output_dir()); } - std::string out_prefix = _config.out_prefix().empty() - ? "" - : _config.out_prefix() + "_"; std::string output_dir = _config.output_dir(); - - - for(int idm = 0; idm < _config.ndms(); idm++) { - - std::string prefix = _config.ndms() > 1 ? out_prefix + "idm_" + - to_string_with_padding(_header.dms[idm], 9, 3) + "_": out_prefix; - - if(! _config.required_dms().empty()) { - const auto& required_dms = _config.required_dms(); - if(std::ranges::find(required_dms, _header.dms[idm]) == required_dms.end()) { - BOOST_LOG_TRIVIAL(info) - << "DM " << _header.dms[idm] << " is not required, skipping from writing"; - continue; - } - } - for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - - // skip if beam is not used - if (! _config.required_beams().empty()) { - const auto& required_beams = _config.required_beams(); - std::cerr << "required_beams: " << required_beams.size() << "ibeam: " << ibeam << std::endl; - if(std::ranges::find(required_beams, ibeam) == required_beams.end()) { + for(std::size_t istokes = 0; istokes < _config.stokes_positions().size(); + istokes++) { + for(int idm = 0; idm < _config.ndms(); idm++) { + if(!_config.required_dms().empty()) { + const auto& required_dms = _config.required_dms(); + if(std::ranges::find(required_dms, _header.dms[idm]) == + required_dms.end()) { BOOST_LOG_TRIVIAL(info) - << "Beam " << ibeam << " is not required, skipping from writing"; + << "DM " << _header.dms[idm] + << " is not required, skipping from writing"; continue; } } - MultiFileWriterConfig writer_config; - writer_config.header_size = _config.dada_header_size(); - writer_config.max_file_size = _config.max_output_filesize(); - writer_config.stokes_mode = _config.stokes_mode(); - writer_config.base_output_dir = output_dir; - writer_config.prefix = prefix + "cb_" + to_string_with_padding(ibeam, 5);; - writer_config.extension = ".fil"; - - BOOST_LOG_TRIVIAL(info) - << "Writer config: " << writer_config.to_string(); - - typename MultiFileWriter::CreateStreamCallBackType - create_stream_callback_sigproc = - skyweaver::detail::create_sigproc_file_stream< - OutputVectorType>; - _beam_writers[idm][ibeam] = - std::make_unique>( - writer_config, - "", - create_stream_callback_sigproc); - _header.ibeam = ibeam; - _beam_writers[idm][ibeam]->init(_header); - - _beam_data[idm][ibeam] = std::make_shared( - std::initializer_list{_config.nsamples_per_block(), - _config.nbridges()}, - 0); - - _beam_data[idm][ibeam]->reference_dm(_header.refdm); - - _total_beam_writers++; + for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + // skip if beam is not used + if(!_config.required_beams().empty()) { + const auto& required_beams = _config.required_beams(); + std::cerr << "required_beams: " << required_beams.size() + << "ibeam: " << ibeam << std::endl; + if(std::ranges::find(required_beams, ibeam) == + required_beams.end()) { + BOOST_LOG_TRIVIAL(info) + << "Beam " << ibeam + << " is not required, skipping from writing"; + continue; + } + } + MultiFileWriterConfig writer_config; + writer_config.header_size = _config.dada_header_size(); + writer_config.max_file_size = _config.max_output_filesize(); + writer_config.stokes_mode = _config.out_stokes().at(istokes); + writer_config.base_output_dir = output_dir; + writer_config.prefix = _config.out_prefix(); + std::string suffix = + _config.ndms() > 1 + ? "_idm_" + + to_string_with_padding(_header.dms[idm], 9, 3) + : ""; + writer_config.suffix = suffix + "_cb_" + + to_string_with_padding(ibeam, 5) + "_" + + _config.out_stokes().at(istokes); + writer_config.extension = ".fil"; + + BOOST_LOG_TRIVIAL(info) + << "Writer config: " << writer_config.to_string(); + + typename MultiFileWriter:: + CreateStreamCallBackType create_stream_callback_sigproc = + skyweaver::detail::create_sigproc_file_stream< + OutputVectorType>; + _beam_writers[istokes][idm][ibeam] = + std::make_unique>( + writer_config, + "", + create_stream_callback_sigproc); + _header.ibeam = ibeam; + _beam_writers[istokes][idm][ibeam]->init(_header); + + _beam_data[istokes][idm][ibeam] = + std::make_shared( + std::initializer_list{_config.nsamples_per_block(), + _config.nbridges()}, + 0); + + _beam_data[istokes][idm][ibeam]->reference_dm(_header.refdm); + _total_beam_writers++; + } } } BOOST_LOG_TRIVIAL(info) << "Added " << _total_beam_writers << " beam writers to SkyCleaver"; } - -SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) +template +skyweaver::SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) { _timer.start("skycleaver::init_readers"); init_readers(); @@ -373,13 +375,64 @@ SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) init_writers(); _timer.stop("skycleaver::init_writers"); } +template +void skyweaver::SkyCleaver::read(std::size_t gulp_samples) +{ + std::size_t gulp_size = gulp_samples * _config.ndms() * _config.nbeams() * + _config.stokes_mode().size(); + BOOST_LOG_TRIVIAL(info) << "Reading gulp samples: " << gulp_samples + << " with size: " << gulp_size; + + omp_set_num_threads(_nthreads_read); + + std::vector read_failures( + _available_freqs.size(), + false); // since we cannot throw exceptions in parallel regions +#pragma omp parallel for + for(std::size_t i = 0; i < _available_freqs.size(); i++) { + skyweaver::SkyCleaver::FreqType freq = _available_freqs[i]; + if(_bridge_readers.find(freq) == _bridge_readers.end()) { + read_failures[i] = true; + } + const auto& reader = _bridge_readers[freq]; + if(reader->eof()) { + BOOST_LOG_TRIVIAL(warning) + << "End of file reached for bridge " << freq; + read_failures[i] = true; + } + + std::streamsize read_size = + reader->read(reinterpret_cast(thrust::raw_pointer_cast( + _bridge_data[freq]->data())), + gulp_size); // read a big chunk of data + BOOST_LOG_TRIVIAL(debug) + << "Read " << read_size << " bytes from bridge" << freq; + if(read_size < gulp_size * sizeof(typename InputVectorType::value_type)) { + BOOST_LOG_TRIVIAL(warning) + << "Read less data than expected from bridge " << freq; + read_failures[i] = true; + } + } + bool failed = false; + for(int i = 0; i < read_failures.size(); i++) { + if(read_failures[i]) { + BOOST_LOG_TRIVIAL(error) + << "Reading bridge [" << i << "]: failed " << std::endl; + failed = true; + } + } + if(failed) { + std::runtime_error("Failed to read data from bridge readers."); + } + + BOOST_LOG_TRIVIAL(info) << "Read data from bridge readers"; +} -void SkyCleaver::cleave() +template +void skyweaver::SkyCleaver::cleave() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::cleave") - /** read data **/ - for(std::size_t nsamples_read = 0; nsamples_read < _nsamples_to_read; nsamples_read += _config.nsamples_per_block()) { std::size_t gulp_samples = @@ -390,116 +443,96 @@ void SkyCleaver::cleave() BOOST_LOG_TRIVIAL(info) << "Cleaving samples: " << nsamples_read << " to " << nsamples_read + gulp_samples; - std::size_t gulp_size = - gulp_samples * _config.ndms() * _config.nbeams(); - - int nthreads_read = _config.nthreads() > _config.nbridges() - ? _config.nbridges() - : _config.nthreads(); - - omp_set_num_threads(nthreads_read); - _timer.start("skyweaver::read_data"); - - std::vector read_status( - _available_freqs.size(), - false); // since we cannot throw exceptions in parallel regions -#pragma omp parallel for - for(std::size_t i = 0; i < _available_freqs.size(); i++) { - FreqType freq = _available_freqs[i]; - if(_bridge_readers.find(freq) == _bridge_readers.end()) { - read_status[i] = true; - } - const auto& reader = _bridge_readers[freq]; - if(reader->eof()) { - read_status[i] = true; - } - - std::streamsize read_size = - reader->read(reinterpret_cast(thrust::raw_pointer_cast( - _bridge_data[freq]->data())), - gulp_size); // read a big chunk of data - BOOST_LOG_TRIVIAL(debug) - << "Read " << read_size << " bytes from bridge" << freq; - if(read_size < gulp_size * sizeof(InputVectorType::value_type)) { - BOOST_LOG_TRIVIAL(warning) - << "Read less data than expected from bridge " << freq; - read_status[i] = true; - } - } - - if(std::any_of(read_status.begin(), read_status.end(), [](bool status) { - return status; - })) { - std::runtime_error("Some bridges have had unexpected reads"); - } - - BOOST_LOG_TRIVIAL(info) << "Read data from bridge readers"; - + read(gulp_samples); _timer.stop("skyweaver::read_data"); - _timer.start("skyweaver::process_data"); - - /** Process data **/ + _timer.start("skyweaver::process_data"); omp_set_num_threads(_config.nthreads()); - std::size_t nbridges = _config.nbridges(); - std::size_t nsamples_per_block = _config.nsamples_per_block(); - std::size_t ndms = _config.ndms(); - std::size_t nbeams = _config.nbeams(); - -#pragma omp parallel for schedule(static) collapse(2) - for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { - for(std::size_t idm = 0; idm < ndms; idm++) { // cannot separate loops, so do checks later - - // _beam_data[idm][ibeam] is not found, skip - if(_beam_data.find(idm) == _beam_data.end()) { - continue; - } - if(_beam_data[idm].find(ibeam) == _beam_data[idm].end()) { - continue; - } - + std::size_t nbridges = _config.nbridges(); + std::size_t ndms = _config.ndms(); + std::size_t nbeams = _config.nbeams(); + std::size_t nstokes_out = _config.stokes_positions().size(); + std::size_t nstokes_in = _config.stokes_mode().size(); + +#pragma omp parallel for schedule(static) collapse(3) + for(std::size_t istokes = 0; istokes < nstokes_out; istokes++) { + for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { + for(std::size_t idm = 0; idm < ndms; + idm++) { // cannot separate loops, so do checks later + BOOST_LOG_TRIVIAL(debug) + << "Processing data for stokes: " << istokes + << " beam: " << ibeam << " dm: " << idm; + if(_beam_data.find(istokes) == _beam_data.end()) { + continue; + } + if(_beam_data[istokes].find(idm) == + _beam_data[istokes].end()) { + continue; + } + if(_beam_data[istokes][idm].find(ibeam) == + _beam_data[istokes][idm].end()) { + continue; + } + const int stokes_position = _config.stokes_positions()[istokes]; + #pragma omp simd - for(std::size_t isample = 0; isample < nsamples_per_block; isample++) { - const std::size_t base_index = - isample * ndms * nbeams + idm * nbeams + ibeam; - - std::size_t ifreq = 0; - const std::size_t out_offset = isample * nbridges; - for(const auto& [freq, ifreq_data]: - _bridge_data) { // for each frequency - _beam_data[idm][ibeam]->at(out_offset + nbridges - 1 - - ifreq) = clamp(127 + ifreq_data->at(base_index)); - ++ifreq; + for(std::size_t isample = 0; isample < gulp_samples; + isample++) { + // This is the input, so use nstokes_in + const std::size_t base_index = + isample * ndms * nbeams * nstokes_in + + idm * nbeams * nstokes_in + + ibeam * nstokes_in + + stokes_position; + + std::size_t ifreq = 0; + const std::size_t out_offset = isample * nbridges; + for(const auto& [freq, ifreq_data]: + _bridge_data) { // for each frequency + _beam_data[istokes][idm][ibeam]->at( + out_offset + nbridges - 1 - ifreq) = + clamp(127 + + ifreq_data->at(base_index)); + ++ifreq; + } } } } } BOOST_LOG_TRIVIAL(info) << "Processed data"; _timer.stop("skyweaver::process_data"); - - /** Write data **/ - - _timer.start("skyweaver::write_data"); + write(); + _timer.stop("skyweaver::write_data"); + } -#pragma omp parallel for schedule(static) collapse(2) - for(int idm = 0; idm < _config.ndms(); idm++) { - for(int ibeam = 0; ibeam < _config.nbeams(); ibeam++) { - - if(_beam_data.find(idm) == _beam_data.end()) { + _timer.show_all_timings(); +} +template +void skyweaver::SkyCleaver::write() +{ + omp_set_num_threads(_config.nthreads()); +#pragma omp parallel for schedule(static) collapse(3) + for(std::size_t istokes = 0; istokes < _config.stokes_positions().size(); + istokes++) { + for(std::size_t idm = 0; idm < _config.ndms(); idm++) { + for(std::size_t ibeam = 0; ibeam < _config.nbeams(); ibeam++) { + if(_beam_data.find(istokes) == _beam_data.end()) { + continue; + } + if(_beam_data[istokes].find(idm) == _beam_data[istokes].end()) { continue; } - if(_beam_data[idm].find(ibeam) == _beam_data[idm].end()) { + if(_beam_data[istokes][idm].find(ibeam) == + _beam_data[istokes][idm].end()) { continue; } - _beam_writers[idm][ibeam]->write(*_beam_data[idm][ibeam], - _config.stream_id()); + _beam_writers[istokes][idm][ibeam]->write( + *_beam_data[istokes][idm][ibeam], + _config.stream_id()); } } - - _timer.stop("skyweaver::write_data"); } - _timer.show_all_timings(); } diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index f2b9601..882d91b 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -227,7 +227,7 @@ create_sigproc_file_stream(MultiFileWriterConfig const& config, // uint32_t ibeam = 0; double az = 0.0; double za = 0.0; - uint32_t nifs = header.npol; + uint32_t nifs = 1; header.sigproc_params = true; header.rawfile = std::string("unset"); diff --git a/cpp/skyweaver/src/skycleaver_cli.cpp b/cpp/skyweaver/src/skycleaver_cli.cpp index dcdcc1a..62f995f 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cpp +++ b/cpp/skyweaver/src/skycleaver_cli.cpp @@ -1,3 +1,4 @@ +#include "skyweaver/DescribedVector.hpp" #include "skyweaver/SkyCleaver.hpp" #include "skyweaver/SkyCleaverConfig.hpp" #include "skyweaver/logging.hpp" @@ -7,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -18,10 +21,8 @@ #include #include #include -#include -#include -#include #include +#include #define BOOST_LOG_DYN_LINK 1 @@ -42,7 +43,6 @@ const size_t ERROR_UNHANDLED_EXCEPTION = 2; const char* build_time = __DATE__ " " __TIME__; - template std::vector get_list_from_string(const std::string& value, @@ -99,6 +99,12 @@ std::ostream& operator<<(std::ostream& os, const std::vector& vec) return os; } } // namespace std +template +void run_pipeline(skyweaver::SkyCleaverConfig const& config) +{ + skyweaver::SkyCleaver skycleaver(config); + skycleaver.cleave(); +} int main(int argc, char** argv) { @@ -126,11 +132,6 @@ int main(int argc, char** argv) ->default_value(config.output_dir()) ->notifier([&config](std::string key) { config.output_dir(key); }), "The output directory for all results")( - "root-prefix", - po::value() - ->default_value(config.root_prefix()) - ->notifier([&config](std::string key) { config.root_prefix(key); }), - "The prefix for all output files")( "out-prefix", po::value() ->default_value(config.out_prefix()) @@ -212,21 +213,29 @@ int main(int argc, char** argv) [&config](std::size_t key) { config.nsamples_to_read(key); }), "total number of samples to read from start_sample")( "required_beams", - po::value() - ->default_value("") - ->notifier([&config](std::string key) { + po::value()->default_value("")->notifier( + [&config](std::string key) { config.required_beams(get_list_from_string(key)); }), "Comma separated list of beams to process. Syntax - beam1, beam2, " "beam3:beam4:step, beam5 etc..")( "required_dms", - po::value() - ->default_value("") - ->notifier([&config](std::string key) { + po::value()->default_value("")->notifier( + [&config](std::string key) { config.required_dms(get_list_from_string(key)); }), "Comma separated list of DMs to process. Syntax - dm1, dm2, " - "dm1:dm2:step, etc.."); + "dm1:dm2:step, etc..")( + "out-stokes", + po::value() + ->default_value(config.out_stokes()) + ->notifier([&config](std::string key) { + if(key.find_first_not_of("IQUV") != std::string::npos) { + throw std::runtime_error("Invalid Stokes mode: " + key); + } + config.out_stokes(key); + }), + "The list of stokes needed - these will be separte output files"); po::options_description cmdline_options; cmdline_options.add(generic).add(main_options); @@ -271,7 +280,6 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "Configuration: " << config_file; BOOST_LOG_TRIVIAL(info) << "root_dir: " << config.root_dir(); BOOST_LOG_TRIVIAL(info) << "output_dir: " << config.output_dir(); - BOOST_LOG_TRIVIAL(info) << "root_prefix: " << config.root_prefix(); BOOST_LOG_TRIVIAL(info) << "out_prefix: " << config.out_prefix(); BOOST_LOG_TRIVIAL(info) << "nthreads: " << config.nthreads(); BOOST_LOG_TRIVIAL(info) @@ -283,22 +291,57 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "max_ram_gb: " << config.max_ram_gb(); BOOST_LOG_TRIVIAL(info) << "max_output_filesize: " << config.max_output_filesize(); - BOOST_LOG_TRIVIAL(info) << "dada_header_size: " << config.dada_header_size(); + BOOST_LOG_TRIVIAL(info) + << "dada_header_size: " << config.dada_header_size(); BOOST_LOG_TRIVIAL(info) << "start_sample: " << config.start_sample(); - BOOST_LOG_TRIVIAL(info) << "nsamples_to_read: " << config.nsamples_to_read(); + BOOST_LOG_TRIVIAL(info) + << "nsamples_to_read: " << config.nsamples_to_read(); + BOOST_LOG_TRIVIAL(info) << "out_stokes: " << config.out_stokes(); + if(config.required_beams().size() > 0) { - for (auto beam : config.required_beams()) { - BOOST_LOG_TRIVIAL(info) << "required_beam: " << beam; - } + for(auto beam: config.required_beams()) { + BOOST_LOG_TRIVIAL(info) << "required_beam: " << beam; + } } if(config.required_dms().size() > 0) { - for (auto dm : config.required_dms()) { - BOOST_LOG_TRIVIAL(info) << "required_dm: " << dm; - } + for(auto dm: config.required_dms()) { + BOOST_LOG_TRIVIAL(info) << "required_dm: " << dm; + } + } + std::vector stokes_positions; + // make sure that every character in out_stokes is a valid stokes mode + // if present add the relevant position to the stokes_positions vector + for(auto stokes: config.out_stokes()) { + std::size_t pos = config.stokes_mode().find(stokes); + if(pos == std::string::npos) { + throw std::runtime_error("Invalid Stokes mode: " + stokes); + } + stokes_positions.push_back(pos); } + config.stokes_positions(stokes_positions); + + run_pipeline, + skyweaver::TFPowersStdH>(config); skyweaver::SkyCleaver skycleaver(config); skycleaver.cleave(); + + // if(config.stokes_mode() == "I" || config.stokes_mode() == "Q" || + // config.stokes_mode() == "U" || config.stokes_mode() == "V") { + // run_pipeline>, + // skyweaver::TFPowersStdH>(config); + // } else if(config.stokes_mode() == "IV" || config.stokes_mode() == "QU") { + // run_pipeline, + // skyweaver::TFPowersStdH>(config); + + // } else if(config.stokes_mode() == "IQUV") { + // run_pipeline, + // skyweaver::TFPowersStdH>(config); + // } else { + // throw std::runtime_error("Invalid Stokes mode: " + + // config.stokes_mode()); + // } + } From a1a2c94f30790b03aaca862c444f7e1106507f29 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Tue, 26 Nov 2024 17:32:34 +0100 Subject: [PATCH 36/69] get header values from header instead of user --- cpp/skyweaver/ObservationHeader.hpp | 3 +- cpp/skyweaver/SkyCleaver.hpp | 4 +- cpp/skyweaver/detail/SkyCleaver.cpp | 129 ++++++++++++++++++------ cpp/skyweaver/src/ObservationHeader.cpp | 9 +- cpp/skyweaver/src/skycleaver_cli.cpp | 39 +------ 5 files changed, 114 insertions(+), 70 deletions(-) diff --git a/cpp/skyweaver/ObservationHeader.hpp b/cpp/skyweaver/ObservationHeader.hpp index d09564d..6e2fbb9 100644 --- a/cpp/skyweaver/ObservationHeader.hpp +++ b/cpp/skyweaver/ObservationHeader.hpp @@ -41,8 +41,9 @@ struct ObservationHeader { std::string telescope; // Telescope name std::string instrument; // Name of the recording instrument std::string order; // Order of the dimensions in the data - std::string ndms; // Number of DMs + std::size_t ndms; // Number of DMs std::vector dms; // DMs + std::string stokes_mode; // Stokes mode std::string to_string() const; // Convert the header to a string diff --git a/cpp/skyweaver/SkyCleaver.hpp b/cpp/skyweaver/SkyCleaver.hpp index 8c99caa..6ff993f 100644 --- a/cpp/skyweaver/SkyCleaver.hpp +++ b/cpp/skyweaver/SkyCleaver.hpp @@ -39,7 +39,7 @@ class SkyCleaver using StokesNumberType = std::size_t; private: - SkyCleaverConfig const& _config; + SkyCleaverConfig& _config; std::map> _bridge_readers; std::map> _bridge_data; @@ -71,7 +71,7 @@ class SkyCleaver Timer _timer; public: - SkyCleaver(SkyCleaverConfig const& config); + SkyCleaver(SkyCleaverConfig& config); SkyCleaver(SkyCleaver const&) = delete; void operator=(SkyCleaver const&) = delete; void cleave(); diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index 8a4945e..a17b669 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -1,8 +1,8 @@ #include "skyweaver/SkyCleaver.hpp" -#include "skyweaver/types.cuh" #include "skyweaver/beamformer_utils.cuh" +#include "skyweaver/types.cuh" #include #include @@ -12,11 +12,10 @@ #include #include -namespace fs = std::filesystem; - -using BridgeReader = skyweaver::BridgeReader; -using MultiFileReader = skyweaver::MultiFileReader; +namespace fs = std::filesystem; +using BridgeReader = skyweaver::BridgeReader; +using MultiFileReader = skyweaver::MultiFileReader; namespace { @@ -90,6 +89,45 @@ std::vector get_files(std::string directory_path, } } // namespace + +void compare_bridge_headers(const skyweaver::ObservationHeader& first, + const skyweaver::ObservationHeader& second) +{ + if(first.nchans != second.nchans) { + throw std::runtime_error("Number of channels in bridge readers do not " + "match. Expected: " + + std::to_string(first.nchans) + + " Found: " + std::to_string(second.nchans)); + } + if(first.nbeams != second.nbeams) { + throw std::runtime_error( + "Number of beams in bridge readers do not match. " + "Expected: " + + std::to_string(first.nbeams) + + " Found: " + std::to_string(second.nbeams)); + } + if(first.nbits != second.nbits) { + throw std::runtime_error( + "Number of bits in bridge readers do not match. " + "Expected: " + + std::to_string(first.nbits) + + " Found: " + std::to_string(second.nbits)); + } + if(first.tsamp != second.tsamp) { + throw std::runtime_error( + "Sampling time in bridge readers do not match. " + "Expected: " + + std::to_string(first.tsamp) + + " Found: " + std::to_string(second.tsamp)); + } + if(first.stokes_mode != second.stokes_mode) { + throw std::runtime_error("Stokes mode in bridge readers do not match. " + "Expected: " + + first.stokes_mode + + " Found: " + second.stokes_mode); + } +} + template void skyweaver::SkyCleaver::init_readers() { @@ -106,7 +144,9 @@ void skyweaver::SkyCleaver::init_readers() << "Found " << freq_dirs.size() << " frequency directories in root directory: " << root_dir; - std::map::FreqType, long double> bridge_timestamps; + std::map::FreqType, + long double> + bridge_timestamps; long double latest_timestamp = 0.0; for(const auto& freq_dir: freq_dirs) { @@ -140,22 +180,49 @@ void skyweaver::SkyCleaver::init_readers() int nbridges = _config.nbridges(); - // std::size_t gulp_size = - // _config.nsamples_per_block() * _config.ndms() * _config.nbeams(); + _header = _bridge_readers[_available_freqs[0]]->get_header(); + for(const auto& [freq, reader]: _bridge_readers) { + compare_bridge_headers(_header, reader->get_header()); + } + BOOST_LOG_TRIVIAL(info) + << "Number of beams: " << _header.nbeams + << " Number of DMS: " << _header.ndms + << " Stokes mode: " << _header.stokes_mode + << " Number of channels: " << _header.nchans; + + + _config.nbeams(_header.nbeams); + _config.ndms(_header.ndms); + _config.stokes_mode(_header.stokes_mode); + _config.nchans(_header.nchans); - ObservationHeader const& header = - (*_bridge_readers.begin()).second->get_header(); BOOST_LOG_TRIVIAL(info) - << "Read header from first file: " << header.to_string(); + << "Number of beams: " << _config.nbeams() + << " Number of DMS: " << _config.ndms() + << " Stokes mode: " << _config.stokes_mode() + << " Number of channels: " << _config.nchans(); + + std::vector stokes_positions; + // make sure that every character in out_stokes is a valid stokes mode + // if present add the relevant position to the stokes_positions vector + for(auto stokes: _config.out_stokes()) { + std::size_t pos = _config.stokes_mode().find(stokes); + if(pos == std::string::npos) { + throw std::runtime_error("Invalid Stokes mode: " + stokes); + } + stokes_positions.push_back(pos); + } + + _config.stokes_positions(stokes_positions); - long double obs_centre_freq = header.obs_frequency; - long double obs_bandwidth = header.obs_bandwidth; + long double obs_centre_freq = _header.obs_frequency; + long double obs_bandwidth = _header.obs_bandwidth; long double start_freq = obs_centre_freq - obs_bandwidth / 2; - for(int i = 0; i < nbridges; i++) { - int ifreq = std::lround( - std::floor(start_freq + (i + 0.5) * obs_bandwidth / nbridges)); + for(int i = 0; i < _config.nbridges(); i++) { + int ifreq = std::lround(std::floor( + start_freq + (i + 0.5) * obs_bandwidth / _config.nbridges())); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) << "Expected frequency [" << i << "]: " << ifreq; @@ -168,13 +235,11 @@ void skyweaver::SkyCleaver::init_readers() _bridge_data[ifreq] = std::make_unique( std::initializer_list{_config.nsamples_per_block(), _config.ndms(), - _config.nbeams(), + _config.nbeams(), _config.stokes_mode().size()}, 0); } - // print _expected_freqs - std::size_t smallest_data_size = std::numeric_limits::max(); std::size_t dbp_factor = _config.ndms() * _config.nbeams() * _config.stokes_mode().size(); @@ -208,7 +273,8 @@ void skyweaver::SkyCleaver::init_readers() << " bytes in bridge reader for frequency: " << freq; std::size_t bytes_seeking = - (nsamples * dbp_factor * sizeof(typename InputVectorType::value_type)); + (nsamples * dbp_factor * + sizeof(typename InputVectorType::value_type)); _bridge_readers[freq]->seekg(bytes_seeking, std::ios_base::beg); @@ -366,7 +432,9 @@ void skyweaver::SkyCleaver::init_writers() << "Added " << _total_beam_writers << " beam writers to SkyCleaver"; } template -skyweaver::SkyCleaver::SkyCleaver(SkyCleaverConfig const& config): _config(config) +skyweaver::SkyCleaver::SkyCleaver( + SkyCleaverConfig& config) + : _config(config) { _timer.start("skycleaver::init_readers"); init_readers(); @@ -376,7 +444,8 @@ skyweaver::SkyCleaver::SkyCleaver(SkyCleaverC _timer.stop("skycleaver::init_writers"); } template -void skyweaver::SkyCleaver::read(std::size_t gulp_samples) +void skyweaver::SkyCleaver::read( + std::size_t gulp_samples) { std::size_t gulp_size = gulp_samples * _config.ndms() * _config.nbeams() * _config.stokes_mode().size(); @@ -390,7 +459,8 @@ void skyweaver::SkyCleaver::read(std::size_t false); // since we cannot throw exceptions in parallel regions #pragma omp parallel for for(std::size_t i = 0; i < _available_freqs.size(); i++) { - skyweaver::SkyCleaver::FreqType freq = _available_freqs[i]; + skyweaver::SkyCleaver::FreqType + freq = _available_freqs[i]; if(_bridge_readers.find(freq) == _bridge_readers.end()) { read_failures[i] = true; } @@ -407,7 +477,8 @@ void skyweaver::SkyCleaver::read(std::size_t gulp_size); // read a big chunk of data BOOST_LOG_TRIVIAL(debug) << "Read " << read_size << " bytes from bridge" << freq; - if(read_size < gulp_size * sizeof(typename InputVectorType::value_type)) { + if(read_size < + gulp_size * sizeof(typename InputVectorType::value_type)) { BOOST_LOG_TRIVIAL(warning) << "Read less data than expected from bridge " << freq; read_failures[i] = true; @@ -475,17 +546,17 @@ void skyweaver::SkyCleaver::cleave() _beam_data[istokes][idm].end()) { continue; } - const int stokes_position = _config.stokes_positions()[istokes]; - + const int stokes_position = + _config.stokes_positions()[istokes]; + #pragma omp simd for(std::size_t isample = 0; isample < gulp_samples; isample++) { // This is the input, so use nstokes_in const std::size_t base_index = isample * ndms * nbeams * nstokes_in + - idm * nbeams * nstokes_in + - ibeam * nstokes_in + - stokes_position; + idm * nbeams * nstokes_in + ibeam * nstokes_in + + stokes_position; std::size_t ifreq = 0; const std::size_t out_offset = isample * nbridges; diff --git a/cpp/skyweaver/src/ObservationHeader.cpp b/cpp/skyweaver/src/ObservationHeader.cpp index 7040a4e..e17c282 100644 --- a/cpp/skyweaver/src/ObservationHeader.cpp +++ b/cpp/skyweaver/src/ObservationHeader.cpp @@ -69,8 +69,11 @@ void read_dada_header(psrdada_cpp::RawBytes& raw_header, "OBS_FREQ", header.frequency)); - header.ndms = parser.get_or_default("NDMS", "0"); - if(header.ndms != "0") { + header.ndms = parser.get_or_default("NDMS", 0); + header.nbeams = parser.get_or_default("NBEAM", 1); + header.stokes_mode = + parser.get_or_default("STOKES_MODE", "I"); + if(header.ndms != 0) { header.dms = parse_float_list(parser.get("DMS")); } } @@ -152,7 +155,7 @@ std::string ObservationHeader::to_string() const << " instrument: " << instrument << "\n" << " chan0_idx: " << chan0_idx << "\n" << " obs_offset: " << obs_offset << "\n"; - if(ndms != "0") { + if(ndms != 0) { oss << " ndms: " << ndms << "\n"; oss << " dms: "; for(auto dm: dms) { oss << dm << " "; } diff --git a/cpp/skyweaver/src/skycleaver_cli.cpp b/cpp/skyweaver/src/skycleaver_cli.cpp index 62f995f..9fec4af 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cpp +++ b/cpp/skyweaver/src/skycleaver_cli.cpp @@ -100,7 +100,7 @@ std::ostream& operator<<(std::ostream& os, const std::vector& vec) } } // namespace std template -void run_pipeline(skyweaver::SkyCleaverConfig const& config) +void run_pipeline(skyweaver::SkyCleaverConfig& config) { skyweaver::SkyCleaver skycleaver(config); skycleaver.cleave(); @@ -148,31 +148,11 @@ int main(int argc, char** argv) ->notifier( [&config](std::size_t key) { config.nsamples_per_block(key); }), "The number of samples per block")( - "nchans", - po::value() - ->default_value(config.nchans()) - ->notifier([&config](std::size_t key) { config.nchans(key); }), - "The number of channels")( "nbridges", po::value() ->default_value(config.nbridges()) ->notifier([&config](std::size_t key) { config.nbridges(key); }), "The number of bridges")( - "nbeams", - po::value() - ->default_value(config.nbeams()) - ->notifier([&config](std::size_t key) { config.nbeams(key); }), - "The number of beams")( - "ndms", - po::value() - ->default_value(config.ndms()) - ->notifier([&config](std::size_t key) { config.ndms(key); }), - "The number of DMs")( - "stokes-mode", - po::value() - ->default_value(config.stokes_mode()) - ->notifier([&config](std::string key) { config.stokes_mode(key); }), - "The stokes mode")( "stream-id", po::value() ->default_value(config.stream_id()) @@ -308,25 +288,14 @@ int main(int argc, char** argv) BOOST_LOG_TRIVIAL(info) << "required_dm: " << dm; } } - std::vector stokes_positions; - // make sure that every character in out_stokes is a valid stokes mode - // if present add the relevant position to the stokes_positions vector - for(auto stokes: config.out_stokes()) { - std::size_t pos = config.stokes_mode().find(stokes); - if(pos == std::string::npos) { - throw std::runtime_error("Invalid Stokes mode: " + stokes); - } - stokes_positions.push_back(pos); - } - - config.stokes_positions(stokes_positions); + run_pipeline, skyweaver::TFPowersStdH>(config); - skyweaver::SkyCleaver skycleaver(config); - skycleaver.cleave(); + // skyweaver::SkyCleaver skycleaver(config); + // skycleaver.cleave(); // if(config.stokes_mode() == "I" || config.stokes_mode() == "Q" || // config.stokes_mode() == "U" || config.stokes_mode() == "V") { From cbaf03bca758f85ba14e07600d1a3ac12e036f41 Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 28 Nov 2024 11:48:32 +0100 Subject: [PATCH 37/69] dirty pass on scaling change --- cpp/skyweaver/CoherentBeamformer.cuh | 6 +- cpp/skyweaver/IncoherentBeamformer.cuh | 10 +- cpp/skyweaver/StatisticsCalculator.cuh | 6 +- cpp/skyweaver/beamformer_utils.cuh | 34 +++-- cpp/skyweaver/src/CoherentBeamformer.cu | 10 +- cpp/skyweaver/src/IncoherentBeamformer.cu | 14 +- cpp/skyweaver/src/StatisticsCalculator.cu | 166 ++++++++++++---------- 7 files changed, 137 insertions(+), 109 deletions(-) diff --git a/cpp/skyweaver/CoherentBeamformer.cuh b/cpp/skyweaver/CoherentBeamformer.cuh index dc47b9c..d10cc35 100644 --- a/cpp/skyweaver/CoherentBeamformer.cuh +++ b/cpp/skyweaver/CoherentBeamformer.cuh @@ -33,8 +33,8 @@ __global__ void bf_ftpa_general_k( int2 const* __restrict__ ftpa_voltages, int2 const* __restrict__ fbpa_weights, typename BfTraits::QuantisedPowerType* __restrict__ btf_powers, - float const* __restrict__ output_scale, - float const* __restrict__ output_offset, + float4 const* __restrict__ output_scale, + float4 const* __restrict__ output_offset, int const* __restrict__ beamset_mapping, typename BfTraits::RawPowerType const* __restrict__ ib_powers, int nsamples); @@ -55,7 +55,7 @@ class CoherentBeamformer typedef BTFPowersD RawPowerVectorTypeD; // FBA order (assuming equal weight per polarisation) typedef thrust::device_vector WeightsVectorTypeD; - typedef thrust::device_vector ScalingVectorTypeD; + typedef thrust::device_vector ScalingVectorTypeD; typedef thrust::device_vector MappingVectorTypeD; public: diff --git a/cpp/skyweaver/IncoherentBeamformer.cuh b/cpp/skyweaver/IncoherentBeamformer.cuh index e40d73b..ef18b6d 100644 --- a/cpp/skyweaver/IncoherentBeamformer.cuh +++ b/cpp/skyweaver/IncoherentBeamformer.cuh @@ -34,8 +34,8 @@ __global__ void icbf_ftpa_general_k( char2 const* __restrict__ ftpa_voltages, typename BfTraits::RawPowerType* __restrict__ tf_powers_raw, typename BfTraits::QuantisedPowerType* __restrict__ tf_powers, - float const* __restrict__ output_scale, - float const* __restrict__ output_offset, + float4 const* __restrict__ output_scale, + float4 const* __restrict__ output_offset, float const* __restrict__ antenna_weights, int nsamples, int nbeamsets); @@ -52,7 +52,9 @@ class IncoherentBeamformer // TF order typedef BTFPowersD RawPowerVectorTypeD; // TF order - typedef thrust::device_vector ScalingVectorTypeD; + typedef thrust::device_vector ScalingVectorTypeD; // Always IQUV scalings + // BsA order + typedef thrust::device_vector BeamsetWeightsVectorTypeD; public: /** @@ -86,7 +88,7 @@ class IncoherentBeamformer PowerVectorTypeD& output, ScalingVectorTypeD const& output_scale, ScalingVectorTypeD const& output_offset, - ScalingVectorTypeD const& antenna_weights, + BeamsetWeightsVectorTypeD const& antenna_weights, int nbeamsets, cudaStream_t stream); diff --git a/cpp/skyweaver/StatisticsCalculator.cuh b/cpp/skyweaver/StatisticsCalculator.cuh index 204f82b..5767a78 100644 --- a/cpp/skyweaver/StatisticsCalculator.cuh +++ b/cpp/skyweaver/StatisticsCalculator.cuh @@ -39,9 +39,11 @@ struct StatisticsFileHeader { class StatisticsCalculator { public: - typedef float ScalingType; + typedef float4 ScalingType; // Always caclulate for full stokes typedef thrust::device_vector ScalingVectorTypeD; typedef thrust::host_vector ScalingVectorTypeH; + typedef thrust::device_vector BeamsetWeightsVectorTypeD; + typedef thrust::host_vector BeamsetWeightsVectorTypeH; typedef FPAStatsD StatisticsVectorTypeD; typedef FPAStatsH StatisticsVectorTypeH; @@ -93,7 +95,7 @@ class StatisticsCalculator * @brief Update the scaling arrays based on the last statistics * calculation. */ - void update_scalings(ScalingVectorTypeH const& beamset_weights, + void update_scalings(BeamsetWeightsVectorTypeH const& beamset_weights, int nbeamsets); private: diff --git a/cpp/skyweaver/beamformer_utils.cuh b/cpp/skyweaver/beamformer_utils.cuh index 74d4873..ae165e8 100644 --- a/cpp/skyweaver/beamformer_utils.cuh +++ b/cpp/skyweaver/beamformer_utils.cuh @@ -270,10 +270,20 @@ struct IncoherentBeamSubtract { static inline __host__ __device__ void apply(T const& power, T const& ib_power, float const& ib_mutliplier, // 127^2 as default - float const& scale_factor, + float4 const& scale_factor, T& result) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor); + if constexpr(S == StokesParameter::I) { + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.x); + } constexpr(S == StokesParameter::Q) { + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.y); + } constexpr(S == StokesParameter::U) { + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.z); + } constexpr(S == StokesParameter::V) { + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.w); + } else { + static_no_match(); + } } }; @@ -285,10 +295,16 @@ struct Rescale { T& result) { if constexpr(S == StokesParameter::I) { - AT(result, I) = rintf((AT(power, I) - offset) / scale_factor); + AT(result, I) = rintf((AT(power, I) - offset.x) / scale_factor.x); + } constexpr(S == StokesParameter::Q) { + AT(result, I) = rintf((AT(power, I) - offset.y) / scale_factor.y); + } constexpr(S == StokesParameter::U) { + AT(result, I) = rintf((AT(power, I) - offset.z) / scale_factor.z); + } constexpr(S == StokesParameter::V) { + AT(result, I) = rintf((AT(power, I) - offset.w) / scale_factor.w); } else { - AT(result, I) = rintf(AT(power, I) / scale_factor); - } + static_no_match(); + } } }; @@ -335,16 +351,16 @@ struct StokesTraits ib_subtract(RawPowerType const& power, RawPowerType const& ib_power, float const& ib_mutliplier, - float const& scale_factor) { + float4 const& scale_factor) { RawPowerType result{}; - Iterate<0, Stokes...>::template apply(power, ib_power, ib_mutliplier, scale_factor, result); + Iterate<0, Stokes...>::template apply(power, ib_power, ib_mutliplier, scale_factor, result); return result; } static inline __host__ __device__ RawPowerType rescale(RawPowerType const& power, - float const& offset, - float const& scale_factor) { + float4 const& offset, + float4 const& scale_factor) { RawPowerType result{}; Iterate<0, Stokes...>::template apply(power, offset, scale_factor, result); return result; diff --git a/cpp/skyweaver/src/CoherentBeamformer.cu b/cpp/skyweaver/src/CoherentBeamformer.cu index 43a8688..ff1e563 100644 --- a/cpp/skyweaver/src/CoherentBeamformer.cu +++ b/cpp/skyweaver/src/CoherentBeamformer.cu @@ -14,8 +14,8 @@ __global__ void bf_ftpa_general_k( int2 const* __restrict__ ftpa_voltages, int2 const* __restrict__ fbpa_weights, typename BfTraits::QuantisedPowerType* __restrict__ tfb_powers, - float const* __restrict__ output_scale, - float const* __restrict__ output_offset, + float4 const* __restrict__ output_scale, + float4 const* __restrict__ output_offset, int const* __restrict__ beamset_mapping, typename BfTraits::RawPowerType const* __restrict__ ib_powers, int nsamples) @@ -156,7 +156,7 @@ __global__ void bf_ftpa_general_k( int const ib_power_idx = beamset_idx * nsamps_out * gridDim.y + output_sample_idx * gridDim.y + blockIdx.y; int const scloff_idx = beamset_idx * gridDim.y + blockIdx.y; - float scale = output_scale[scloff_idx]; + float4 scale = output_scale[scloff_idx]; typename BfTraits::RawPowerType ib_power = ib_powers[ib_power_idx]; #if SKYWEAVER_IB_SUBTRACTION /* @@ -246,9 +246,9 @@ void CoherentBeamformer::beamform( char2 const* fbpa_weights_ptr = thrust::raw_pointer_cast(weights.data()); typename BfTraits::QuantisedPowerType* tfb_powers_ptr = thrust::raw_pointer_cast(output.data()); - float const* power_scaling_ptr = + float4 const* power_scaling_ptr = thrust::raw_pointer_cast(output_scale.data()); - float const* power_offset_ptr = + float4 const* power_offset_ptr = thrust::raw_pointer_cast(output_offset.data()); typename BfTraits::RawPowerType const* ib_powers_ptr = thrust::raw_pointer_cast(ib_powers.data()); diff --git a/cpp/skyweaver/src/IncoherentBeamformer.cu b/cpp/skyweaver/src/IncoherentBeamformer.cu index c91618e..ee4a80e 100644 --- a/cpp/skyweaver/src/IncoherentBeamformer.cu +++ b/cpp/skyweaver/src/IncoherentBeamformer.cu @@ -17,8 +17,8 @@ __global__ void icbf_ftpa_general_k( char2 const* __restrict__ ftpa_voltages, typename BfTraits::RawPowerType* __restrict__ tf_powers_raw, typename BfTraits::QuantisedPowerType* __restrict__ tf_powers, - float const* __restrict__ output_scale, - float const* __restrict__ output_offset, + float4 const* __restrict__ output_scale, + float4 const* __restrict__ output_offset, float const* __restrict__ antenna_weights, int nsamples, int nbeamsets) @@ -79,9 +79,9 @@ __global__ void icbf_ftpa_general_k( const typename BfTraits::RawPowerType power_f32 = acc_buffer[0]; const int output_idx = beamset_idx * gridDim.x * gridDim.y + output_t_idx * gridDim.y + output_f_idx; - const float scale = + const float4 scale = output_scale[beamset_idx * gridDim.y + output_f_idx]; - const float offset = + const float4 offset = output_offset[beamset_idx * gridDim.y + output_f_idx]; tf_powers_raw[output_idx] = power_f32; tf_powers[output_idx] = @@ -112,7 +112,7 @@ void IncoherentBeamformer::beamform( PowerVectorTypeD& output, ScalingVectorTypeD const& output_scale, ScalingVectorTypeD const& output_offset, - ScalingVectorTypeD const& antenna_weights, + BeamsetWeightsVectorTypeD const& antenna_weights, int nbeamsets, cudaStream_t stream) { @@ -156,9 +156,9 @@ void IncoherentBeamformer::beamform( dim3 grid(nsamples / _config.ib_tscrunch(), _config.nchans() / _config.ib_fscrunch()); char2 const* ftpa_voltages_ptr = thrust::raw_pointer_cast(input.data()); - float const* output_scale_ptr = + float4 const* output_scale_ptr = thrust::raw_pointer_cast(output_scale.data()); - float const* output_offset_ptr = + float4 const* output_offset_ptr = thrust::raw_pointer_cast(output_offset.data()); float const* antenna_weights_ptr = thrust::raw_pointer_cast(antenna_weights.data()); diff --git a/cpp/skyweaver/src/StatisticsCalculator.cu b/cpp/skyweaver/src/StatisticsCalculator.cu index 4de451a..a2cf68c 100644 --- a/cpp/skyweaver/src/StatisticsCalculator.cu +++ b/cpp/skyweaver/src/StatisticsCalculator.cu @@ -130,7 +130,7 @@ void StatisticsCalculator::calculate_statistics( } void StatisticsCalculator::update_scalings( - ScalingVectorTypeH const& beamset_weights, + BeamsetWeightsVectorTypeH const& beamset_weights, int nbeamsets) { // At this stage we have the standard deviations of each channel @@ -139,99 +139,107 @@ void StatisticsCalculator::update_scalings( // disk. const float weights_amp = 127.0f; - std::size_t reduced_nchans_ib = _config.nchans() / _config.ib_fscrunch(); - std::size_t reduced_nchans_cb = _config.nchans() / _config.cb_fscrunch(); + if ((_config.ib_fscrunch() != _config.cb_fscrunch()) || + (_config.ib_tscrunch() != _config.cb_tscrunch())) { + throw std::invalid_argument("IB and CB must share same F and T scrunch"); + } + const std::size_t output_nchans = _config.nchans() / _config.cb_fscrunch(); + const std::size_t fscrunch = _config.cb_fscrunch(); + const std::size_t tscrunch = _config.cb_tscrunch(); // Offsets for the coherent beams - _cb_offsets_d.resize(reduced_nchans_cb * nbeamsets); - _cb_offsets_h.resize(reduced_nchans_cb * nbeamsets); + _cb_offsets_d.resize(output_nchans * nbeamsets); + _cb_offsets_h.resize(output_nchans * nbeamsets); // Scalings for the coherent beams - _cb_scaling_d.resize(reduced_nchans_cb * nbeamsets); - _cb_scaling_h.resize(reduced_nchans_cb * nbeamsets); + _cb_scaling_d.resize(output_nchans * nbeamsets); + _cb_scaling_h.resize(output_nchans * nbeamsets); // Offsets for the incoherent beam - _ib_offsets_d.resize(reduced_nchans_ib * nbeamsets); - _ib_offsets_h.resize(reduced_nchans_ib * nbeamsets); + _ib_offsets_d.resize(output_nchans * nbeamsets); + _ib_offsets_h.resize(output_nchans * nbeamsets); // Scalings for the incoherent beam - _ib_scaling_d.resize(reduced_nchans_ib * nbeamsets); - _ib_scaling_h.resize(reduced_nchans_ib * nbeamsets); + _ib_scaling_d.resize(output_nchans * nbeamsets); + _ib_scaling_h.resize(output_nchans * nbeamsets); const std::uint32_t pa = _config.npol() * _config.nantennas(); const std::uint32_t a = _config.nantennas(); for(std::uint32_t beamset_idx = 0; beamset_idx < nbeamsets; ++beamset_idx) { - // For each frequency channel we average the std estimates then - // calculate the offsets and scalings. - for(int f_idx = 0; f_idx < _config.nchans(); ++f_idx) { - float sum = 0.0f; - float count = 0.0f; - for(int p_idx = 0; p_idx < _config.npol(); ++p_idx) { - for(int a_idx = 0; a_idx < _config.nantennas(); ++a_idx) { - const float weight = - beamset_weights[_config.nantennas() * beamset_idx + - a_idx]; - sum += - weight * _stats_h[f_idx * pa + p_idx * a + a_idx].std; - count += weight; - } - } - const float avg_std = sum / count; - BOOST_LOG_TRIVIAL(debug) << "Channel " << f_idx; - BOOST_LOG_TRIVIAL(debug) - << "Averaged standard deviation = " << avg_std; - float const effective_nantennas = count / _config.npol(); - - // CB OFFSET - { - float scale = std::pow(weights_amp * avg_std * - std::sqrt(effective_nantennas), - 2); - float dof = 2 * _config.cb_tscrunch() * _config.npol(); - _cb_offsets_h[f_idx] = scale * dof; - BOOST_LOG_TRIVIAL(debug) - << "CB offset = " << _cb_offsets_h[f_idx]; - } - // CB SCALE - { - float scale = std::pow(weights_amp * avg_std * - std::sqrt(effective_nantennas), - 2); - float dof = 2 * _config.cb_tscrunch() * _config.npol(); - _cb_scaling_h[f_idx] = - scale * std::sqrt(2 * dof) / _config.output_level(); - BOOST_LOG_TRIVIAL(debug) - << "CB scaling = " << _cb_scaling_h[f_idx]; - } - - // IB OFFSET - { - float scale = std::pow(avg_std, 2); - float dof = 2 * _config.ib_tscrunch() * effective_nantennas * - _config.npol(); - _ib_offsets_h[f_idx] = scale * dof; - BOOST_LOG_TRIVIAL(debug) - << "IB offset = " << _ib_offsets_h[f_idx]; + // Here we compute the offsets and scaling factors for I, Q, U and V for each beamset + // Statistics are in FPA order + + for(int fo_idx = 0; fo_idx < output_nchans; ++fo_idx) { + + const int output_idx = beamset_idx * nbeamsets + fo_idx; + + // reset main accumulators + float meanI = 0.0f; + float meanQ = 0.0f; + float varIQc = 0.0f; + float varUVc = 0.0f; + float varIQi = 0.0f; + float varUVi = 0.0f; + + for(int fs_idx = 0; fs_idx < fscrunch; ++fs_idx) { + const int f_idx = fo_idx * fscrunch + fs_idx; + + // Per antenna/pol accumulators + float var_p0_sum = 0.0f; // sum(sigma_p0^2) + float var_p1_sum = 0.0f; // sum(sigma_p1^2) + float quad_sum = 0.0f; // sum(sigma_p0^4 + sigma_p1^4) + float sum_mul = 0.0f; // sum(sigma_p0^2 * sigma_p1^2) + for (int a_idx = 0; a_idx < _cofig.npol(); ++a_idx) { + // stats are in FPA order + // weights are in FPBA order + const int input_idx = f_idx * pa + a_idx; + Statistics p0 = _stats_h[input_idx]; + Statistics p1 = _stats_h[input_idx + nantennas]; + const float weight = beamset_weights[beamset_idx * a + a_idx]; + const float std_p0 = p0.std * weight; + const float std_p1 = p1.std * weight; + const float var_p0 = std_p0 * std_p0; + const float var_p1 = std_p1 * std_p1; + meanI += var_p0 + var_p1; + meanQ += var_p0 - var_p1; + var_p0_sum += var_p0; + var_p1_sum += var_p1; + quad_sum += (var_p0 * var_p0) + (var_p1 * var_p1); + sum_mul += var_p0 * var_p1; + } + varIQc += (var_p0_sum * var_p0_sum) + (var_p1_sum * var_p1_sum); + varIQi += quad_sum; + varUVc += var_p0_sum * var_p1_sum; + varUVi += sum_mul; } - // IB SCALE - { - float scale = std::pow(avg_std, 2); - float dof = 2 * _config.ib_tscrunch() * effective_nantennas * - _config.npol(); - _ib_scaling_h[f_idx] = - scale * std::sqrt(2 * dof) / _config.output_level(); - BOOST_LOG_TRIVIAL(debug) - << "IB scaling = " << _ib_scaling_h[f_idx]; - } + // Coherent offsets + _cb_offsets_h[output_idx].x = 2 * tscrunch * meanI; // I + _cb_offsets_h[output_idx].y = 2 * tscrunch * meanQ; // Q + _cb_offsets_h[output_idx].z = 0.0f; // U + _cb_offsets_h[output_idx].w = 0.0f; // V + + // Coherent scales + _cb_scaling_h[output_idx].x = 4 * tscrunch * varIQc / _config.output_level(); // I + _cb_scaling_h[output_idx].y = 4 * tscrunch * varIQc / _config.output_level(); // Q + _cb_scaling_h[output_idx].z = 8 * tscrunch * varUVc / _config.output_level(); // U + _cb_scaling_h[output_idx].w = 8 * tscrunch * varUVc / _config.output_level(); // V + + // Incoherent offsets + _ib_offsets_h[output_idx].x = 2 * tscrunch * meanI; // I + _ib_offsets_h[output_idx].y = 2 * tscrunch * meanQ; // Q + _ib_offsets_h[output_idx].z = 0.0f; // U + _ib_offsets_h[output_idx].w = 0.0f; // V + + // Incoherent scales + _ib_scaling_h[output_idx].x = 4 * tscrunch * varIQi / _config.output_level(); // I + _ib_scaling_h[output_idx].y = 4 * tscrunch * varIQi / _config.output_level(); // Q + _ib_scaling_h[output_idx].z = 8 * tscrunch * varUVi / _config.output_level(); // U + _ib_scaling_h[output_idx].w = 8 * tscrunch * varUVi / _config.output_level(); // V } } - // At this stage, all scaling vectors are available on the host and - // could be written to disk. - - // Copying these back to the device _cb_offsets_d = _cb_offsets_h; _cb_scaling_d = _cb_scaling_h; _ib_offsets_d = _ib_offsets_h; @@ -256,7 +264,7 @@ void StatisticsCalculator::dump_scalings( std::string const& timestamp, std::string const& tag, std::string const& path, - thrust::host_vector const& ar) const + ScalingVectorTypeH const& ar) const { std::ofstream writer; std::string filename = path + "/" + timestamp + "_" + tag + "_" + @@ -271,7 +279,7 @@ void StatisticsCalculator::dump_scalings( BOOST_LOG_TRIVIAL(error) << error_message.str(); return; } - writer.write((char*)ar.data(), ar.size() * sizeof(float)); + writer.write(static_cast(ar.data()), ar.size() * sizeof(ScalingType)); writer.close(); } @@ -305,4 +313,4 @@ StatisticsCalculator::ib_scaling() const return _ib_scaling_d; } -} // namespace skyweaver +} // namespace skyweaver \ No newline at end of file From 820efb994e341feea8119be5ec12b90143905b90 Mon Sep 17 00:00:00 2001 From: Ewan Barr Date: Thu, 28 Nov 2024 15:25:23 +0100 Subject: [PATCH 38/69] compiling now and beamformer tests running --- cpp/skyweaver/StatisticsCalculator.cuh | 2 +- cpp/skyweaver/beamformer_utils.cuh | 16 ++--- cpp/skyweaver/benchmark/beamformer_bench.cu | 2 +- cpp/skyweaver/src/StatisticsCalculator.cu | 10 +-- .../test/CoherentBeamformerTester.cuh | 17 ++++- .../test/IncoherentBeamformerTester.cuh | 7 +- .../test/src/CoherentBeamformerTester.cu | 71 +++++++++++-------- .../test/src/IncoherentBeamformerTester.cu | 34 +++++---- 8 files changed, 95 insertions(+), 64 deletions(-) diff --git a/cpp/skyweaver/StatisticsCalculator.cuh b/cpp/skyweaver/StatisticsCalculator.cuh index 5767a78..1b9dee3 100644 --- a/cpp/skyweaver/StatisticsCalculator.cuh +++ b/cpp/skyweaver/StatisticsCalculator.cuh @@ -104,7 +104,7 @@ class StatisticsCalculator void dump_scalings(std::string const& timestamp, std::string const& tag, std::string const& path, - thrust::host_vector const& ar) const; + ScalingVectorTypeH const& ar) const; private: PipelineConfig const& _config; diff --git a/cpp/skyweaver/beamformer_utils.cuh b/cpp/skyweaver/beamformer_utils.cuh index ae165e8..15ec773 100644 --- a/cpp/skyweaver/beamformer_utils.cuh +++ b/cpp/skyweaver/beamformer_utils.cuh @@ -275,11 +275,11 @@ struct IncoherentBeamSubtract { { if constexpr(S == StokesParameter::I) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.x); - } constexpr(S == StokesParameter::Q) { + } else if constexpr(S == StokesParameter::Q) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.y); - } constexpr(S == StokesParameter::U) { + } else if constexpr(S == StokesParameter::U) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.z); - } constexpr(S == StokesParameter::V) { + } else if constexpr(S == StokesParameter::V) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.w); } else { static_no_match(); @@ -290,17 +290,17 @@ struct IncoherentBeamSubtract { struct Rescale { template static inline __host__ __device__ void apply(T const& power, - float const& offset, - float const& scale_factor, + float4 const& offset, + float4 const& scale_factor, T& result) { if constexpr(S == StokesParameter::I) { AT(result, I) = rintf((AT(power, I) - offset.x) / scale_factor.x); - } constexpr(S == StokesParameter::Q) { + } else if constexpr(S == StokesParameter::Q) { AT(result, I) = rintf((AT(power, I) - offset.y) / scale_factor.y); - } constexpr(S == StokesParameter::U) { + } else if constexpr(S == StokesParameter::U) { AT(result, I) = rintf((AT(power, I) - offset.z) / scale_factor.z); - } constexpr(S == StokesParameter::V) { + } else if constexpr(S == StokesParameter::V) { AT(result, I) = rintf((AT(power, I) - offset.w) / scale_factor.w); } else { static_no_match(); diff --git a/cpp/skyweaver/benchmark/beamformer_bench.cu b/cpp/skyweaver/benchmark/beamformer_bench.cu index a179741..663cb3b 100644 --- a/cpp/skyweaver/benchmark/beamformer_bench.cu +++ b/cpp/skyweaver/benchmark/beamformer_bench.cu @@ -70,7 +70,7 @@ class BeamformerBencher: public benchmark::Fixture typename CoherentBeamformer::PowerVectorTypeD btf_powers_gpu; typename CoherentBeamformer::ScalingVectorTypeD scales; typename CoherentBeamformer::ScalingVectorTypeD offsets; - typename CoherentBeamformer::ScalingVectorTypeD _antenna_weights; + typename IncoherentBeamformer::BeamsetWeightsVectorTypeD _antenna_weights; typename IncoherentBeamformer::PowerVectorTypeD tf_powers_gpu; typename IncoherentBeamformer::RawPowerVectorTypeD tf_powers_raw_gpu; typename CoherentBeamformer::MappingVectorTypeD _beamset_mapping; diff --git a/cpp/skyweaver/src/StatisticsCalculator.cu b/cpp/skyweaver/src/StatisticsCalculator.cu index a2cf68c..a8dcb33 100644 --- a/cpp/skyweaver/src/StatisticsCalculator.cu +++ b/cpp/skyweaver/src/StatisticsCalculator.cu @@ -191,15 +191,15 @@ void StatisticsCalculator::update_scalings( float var_p1_sum = 0.0f; // sum(sigma_p1^2) float quad_sum = 0.0f; // sum(sigma_p0^4 + sigma_p1^4) float sum_mul = 0.0f; // sum(sigma_p0^2 * sigma_p1^2) - for (int a_idx = 0; a_idx < _cofig.npol(); ++a_idx) { + for (int a_idx = 0; a_idx < _config.npol(); ++a_idx) { // stats are in FPA order // weights are in FPBA order const int input_idx = f_idx * pa + a_idx; Statistics p0 = _stats_h[input_idx]; - Statistics p1 = _stats_h[input_idx + nantennas]; + Statistics p1 = _stats_h[input_idx + a]; const float weight = beamset_weights[beamset_idx * a + a_idx]; - const float std_p0 = p0.std * weight; - const float std_p1 = p1.std * weight; + const float std_p0 = p0.std * weight * weights_amp; + const float std_p1 = p1.std * weight * weights_amp; const float var_p0 = std_p0 * std_p0; const float var_p1 = std_p1 * std_p1; meanI += var_p0 + var_p1; @@ -279,7 +279,7 @@ void StatisticsCalculator::dump_scalings( BOOST_LOG_TRIVIAL(error) << error_message.str(); return; } - writer.write(static_cast(ar.data()), ar.size() * sizeof(ScalingType)); + writer.write(reinterpret_cast(thrust::raw_pointer_cast(ar.data())), ar.size() * sizeof(ScalingType)); writer.close(); } diff --git a/cpp/skyweaver/test/CoherentBeamformerTester.cuh b/cpp/skyweaver/test/CoherentBeamformerTester.cuh index fd60262..e956de1 100644 --- a/cpp/skyweaver/test/CoherentBeamformerTester.cuh +++ b/cpp/skyweaver/test/CoherentBeamformerTester.cuh @@ -21,26 +21,37 @@ class CoherentBeamformerTester: public ::testing::Test typedef CoherentBeamformer CoherentBeamformer; typedef IncoherentBeamformer IncoherentBeamformer; typedef CoherentBeamformer::VoltageVectorTypeD VoltageVectorTypeD; + typedef FTPAVoltagesH VoltageVectorTypeH; + typedef CoherentBeamformer::PowerVectorTypeD PowerVectorTypeD; typedef TFBPowersH HostPowerVectorType; + typedef IncoherentBeamformer::PowerVectorTypeD IBPowerVectorTypeD; typedef BTFPowersH HostIBPowerVectorType; + typedef IncoherentBeamformer::RawPowerVectorTypeD RawIBPowerVectorTypeD; typedef BTFPowersH HostRawIBPowerVectorType; + typedef CoherentBeamformer::WeightsVectorTypeD WeightsVectorTypeD; typedef thrust::host_vector WeightsVectorTypeH; + typedef CoherentBeamformer::ScalingVectorTypeD ScalingVectorTypeD; typedef thrust::host_vector HostScalingVectorType; + typedef CoherentBeamformer::MappingVectorTypeD MappingVectorTypeD; typedef thrust::host_vector MappingVectorTypeH; + + typedef IncoherentBeamformer::BeamsetWeightsVectorTypeD BeamsetWeightsVectorTypeD; + typedef thrust::host_vector + BeamsetWeightsVectorTypeH; protected: void SetUp() override; @@ -61,8 +72,8 @@ class CoherentBeamformerTester: public ::testing::Test int nbeams, int nantennas, int npol, - float const* scales, - float const* offsets, + float4 const* scales, + float4 const* offsets, float const* antenna_weights, int const* beamset_mapping); @@ -70,7 +81,7 @@ class CoherentBeamformerTester: public ::testing::Test WeightsVectorTypeD const& fbpa_weights_gpu, ScalingVectorTypeD const& scales_gpu, ScalingVectorTypeD const& offsets_gpu, - ScalingVectorTypeD const& antenna_weights, + BeamsetWeightsVectorTypeD const& antenna_weights, MappingVectorTypeD const& beamset_mapping, PowerVectorTypeD& btf_powers_gpu, int nsamples); diff --git a/cpp/skyweaver/test/IncoherentBeamformerTester.cuh b/cpp/skyweaver/test/IncoherentBeamformerTester.cuh index 37da649..5983321 100644 --- a/cpp/skyweaver/test/IncoherentBeamformerTester.cuh +++ b/cpp/skyweaver/test/IncoherentBeamformerTester.cuh @@ -31,6 +31,9 @@ class IncoherentBeamformerTester: public ::testing::Test typedef IncoherentBeamformer::ScalingVectorTypeD ScalingVectorTypeD; typedef thrust::host_vector HostScalingVectorType; + typedef IncoherentBeamformer::BeamsetWeightsVectorTypeD BeamsetWeightsVectorTypeD; + typedef thrust::host_vector + HostBeamsetWeightsVectorType; protected: void SetUp() override; @@ -51,7 +54,7 @@ class IncoherentBeamformerTester: public ::testing::Test int nantennas, HostScalingVectorType const& scale, HostScalingVectorType const& offset, - HostScalingVectorType const& beamset_weights, + HostBeamsetWeightsVectorType const& beamset_weights, int nbeamsets); void compare_against_host(VoltageVectorTypeD const& ftpa_voltages_gpu, @@ -59,7 +62,7 @@ class IncoherentBeamformerTester: public ::testing::Test DevicePowerVectorType& tf_powers_gpu, ScalingVectorTypeD const& scaling_vector, ScalingVectorTypeD const& offset_vector, - ScalingVectorTypeD const& beamset_weights, + BeamsetWeightsVectorTypeD const& beamset_weights, int ntimestamps, int nbeamsets); diff --git a/cpp/skyweaver/test/src/CoherentBeamformerTester.cu b/cpp/skyweaver/test/src/CoherentBeamformerTester.cu index a369015..9a98df5 100644 --- a/cpp/skyweaver/test/src/CoherentBeamformerTester.cu +++ b/cpp/skyweaver/test/src/CoherentBeamformerTester.cu @@ -51,8 +51,8 @@ void CoherentBeamformerTester::beamformer_c_reference( int nbeams, int nantennas, int npol, - float const* scales, - float const* offsets, + float4 const* scales, + float4 const* offsets, float const* antenna_weights, int const* beamset_mapping) { @@ -157,7 +157,7 @@ void CoherentBeamformerTester::compare_against_host( WeightsVectorTypeD const& fbpa_weights_gpu, ScalingVectorTypeD const& scales_gpu, ScalingVectorTypeD const& offsets_gpu, - ScalingVectorTypeD const& antenna_weights, + BeamsetWeightsVectorTypeD const& antenna_weights, MappingVectorTypeD const& beamset_mapping, PowerVectorTypeD& btf_powers_gpu, int nsamples) @@ -170,7 +170,7 @@ void CoherentBeamformerTester::compare_against_host( HostScalingVectorType scales = scales_gpu; HostScalingVectorType offsets = offsets_gpu; - HostScalingVectorType antenna_weights_host = antenna_weights; + thrust::host_vector antenna_weights_host = antenna_weights; MappingVectorTypeH beamset_mapping_host = beamset_mapping; beamformer_c_reference( @@ -227,29 +227,36 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) const float input_level = 32.0f; const double pi = std::acos(-1); config.output_level(input_level); - float scale = - std::pow(127.0f * input_level * - std::sqrt(static_cast(config.nantennas())), - 2); - float dof = 2 * config.cb_tscrunch() * config.cb_fscrunch() * config.npol(); - float offset_val = (scale * dof); - float scale_val = (scale * std::sqrt(2 * dof) / config.output_level()); + + float4 cb_offset_val{ + static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(127.0f * input_level, 2)) / config.output_level()), + 0.0f, + 0.0f, + 0.0f + }; + float cb_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * std::pow(config.nantennas() * std::pow(127.0f * input_level, 2), 2) ) / config.output_level()); + float4 cb_scale_val{ + cb_scaling, + cb_scaling, + cb_scaling, + cb_scaling + }; // Set constant scales and offsets for all channels and beamsets typename CBT::ScalingVectorTypeD cb_scales( config.nchans() / config.cb_fscrunch() * nbeamsets, - scale_val); + cb_scale_val); typename CBT::ScalingVectorTypeD cb_offsets( config.nchans() / config.cb_fscrunch() * nbeamsets, - offset_val); + cb_offset_val); // Map all beams to the first beamset by default typename CBT::MappingVectorTypeD beamset_mapping(config.nbeams(), 0); // Enable all antennas in all beamsets - typename CBT::ScalingVectorTypeD beamset_weights(config.nantennas() * + typename thrust::device_vector beamset_weights(config.nantennas() * nbeamsets, - 1.0f); + 1.0f); /** This currently causes tests to fail even though the weights are the same for @@ -260,9 +267,6 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) */ beamset_mapping[0] = 1; - BOOST_LOG_TRIVIAL(info) << "CB scaling: " << scale_val; - BOOST_LOG_TRIVIAL(info) << "CB offset: " << offset_val; - std::default_random_engine generator; std::normal_distribution normal_dist(0.0, input_level); std::uniform_real_distribution uniform_dist(0.0, 2 * pi); @@ -280,9 +284,6 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) std::size_t input_size = (ntimestamps * config.nantennas() * config.nchans() * config.nsamples_per_heap() * config.npol()); - BOOST_LOG_TRIVIAL(info) << "FTPA input dims: " << config.nchans() << ", " - << ntimestamps * config.nsamples_per_heap() << ", " - << config.npol() << ", " << config.nantennas(); int nsamples = config.nsamples_per_heap() * ntimestamps; std::size_t weights_size = @@ -314,20 +315,28 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) typename CBT::WeightsVectorTypeD fbpa_weights_gpu = fbpa_weights_host; typename CBT::PowerVectorTypeD tfb_powers_gpu; - // Note that below even though this is for the IB we have to use the - // CB scrunching parameters to make sure we get the right data out. - float ib_scale = std::pow(input_level, 2); - float ib_dof = 2 * config.cb_tscrunch() * config.cb_fscrunch() * - config.nantennas() * config.npol(); - float ib_power_offset = ib_scale * ib_dof; - float ib_power_scaling = - ib_scale * std::sqrt(2 * ib_dof) / config.output_level(); + float4 ib_offset_val{ + static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 2)) / config.output_level()), + 0.0f, + 0.0f, + 0.0f + }; + + float ib_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 4)) / config.output_level()); + float4 ib_scale_val{ + ib_scaling, + ib_scaling, + ib_scaling, + ib_scaling + }; + typename CBT::ScalingVectorTypeD ib_scales( config.nchans() / config.cb_fscrunch() * nbeamsets, - ib_power_scaling); + ib_scale_val); typename CBT::ScalingVectorTypeD ib_offset( config.nchans() / config.cb_fscrunch() * nbeamsets, - ib_power_offset); + ib_offset_val); + typename CBT::IBPowerVectorTypeD tf_powers_gpu; typename CBT::RawIBPowerVectorTypeD tf_powers_raw_gpu; diff --git a/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu b/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu index a52f675..d7af1c5 100644 --- a/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu +++ b/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu @@ -49,7 +49,7 @@ void IncoherentBeamformerTester::beamformer_c_reference( int nantennas, HostScalingVectorType const& scale, HostScalingVectorType const& offset, - HostScalingVectorType const& beamset_weights, + HostBeamsetWeightsVectorType const& beamset_weights, int nbeamsets) { static_assert(SKYWEAVER_NPOL == 2, "Tests only work for dual poln data."); @@ -109,7 +109,7 @@ void IncoherentBeamformerTester::compare_against_host( DevicePowerVectorType& tf_powers_gpu, ScalingVectorTypeD const& scaling_vector, ScalingVectorTypeD const& offset_vector, - ScalingVectorTypeD const& beamset_weights, + BeamsetWeightsVectorTypeD const& beamset_weights, int ntimestamps, int nbeamsets) { @@ -118,7 +118,7 @@ void IncoherentBeamformerTester::compare_against_host( HostRawPowerVectorType tf_powers_raw_cuda = tf_powers_raw_gpu; HostScalingVectorType h_scaling_vector = scaling_vector; HostScalingVectorType h_offset_vector = offset_vector; - HostScalingVectorType h_beamset_weights = beamset_weights; + HostBeamsetWeightsVectorType h_beamset_weights = beamset_weights; HostRawPowerVectorType tf_powers_raw_host; tf_powers_raw_host.like(tf_powers_raw_gpu); HostPowerVectorType tf_powers_host; @@ -175,23 +175,31 @@ TYPED_TEST(IncoherentBeamformerTester, ib_representative_noise_test) static_cast(std::lround(normal_dist(generator))); } - float ib_scale = std::pow(input_level, 2); - float ib_dof = 2 * config.ib_tscrunch() * config.ib_fscrunch() * - config.nantennas() * config.npol(); - float ib_power_offset = ib_scale * ib_dof; - float ib_power_scaling = - ib_scale * std::sqrt(2 * ib_dof) / config.output_level(); + float4 ib_offset_val{ + static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 2)) / config.output_level()), + 0.0f, + 0.0f, + 0.0f + }; + + float ib_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 4)) / config.output_level()); + float4 ib_scale_val{ + ib_scaling, + ib_scaling, + ib_scaling, + ib_scaling + }; for(int nbeamsets = 1; nbeamsets < 5; ++nbeamsets) { typename IBT::ScalingVectorTypeD scales( config.nchans() / config.ib_fscrunch() * nbeamsets, - ib_power_scaling); + ib_scale_val); typename IBT::ScalingVectorTypeD offset( config.nchans() / config.ib_fscrunch() * nbeamsets, - ib_power_offset); - typename IBT::ScalingVectorTypeD beamset_weights(config.nantennas() * + ib_offset_val); + typename IBT::BeamsetWeightsVectorTypeD beamset_weights(config.nantennas() * nbeamsets, - 1.0f); + 1.0f); typename IBT::VoltageVectorTypeD ftpa_voltages_gpu = ftpa_voltages_host; typename IBT::DevicePowerVectorType tf_powers_gpu; From 49c966a6a1d606162409632a9d7e0a4469bf8008 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 12 Dec 2024 17:26:08 +0100 Subject: [PATCH 39/69] linear pol output and actual beam positions in skycleaver --- cpp/skyweaver/SkyCleaver.hpp | 8 + cpp/skyweaver/SkyCleaverConfig.hpp | 20 +- cpp/skyweaver/detail/SkyCleaver.cpp | 202 +++++++++++++++--- .../detail/file_writer_callbacks.cpp | 6 +- cpp/skyweaver/src/skycleaver_cli.cpp | 57 +---- 5 files changed, 203 insertions(+), 90 deletions(-) diff --git a/cpp/skyweaver/SkyCleaver.hpp b/cpp/skyweaver/SkyCleaver.hpp index 6ff993f..fad34cc 100644 --- a/cpp/skyweaver/SkyCleaver.hpp +++ b/cpp/skyweaver/SkyCleaver.hpp @@ -29,6 +29,13 @@ struct BridgeReader { std::string freq; }; // BridgeReader + +struct BeamInfo { + std::string beam_name; + std::string beam_ra; + std::string beam_dec; +}; + template class SkyCleaver { @@ -47,6 +54,7 @@ class SkyCleaver std::vector _available_freqs; std::size_t _nsamples_to_read; ObservationHeader _header; + std::vector _beam_infos; std::vector _beam_filenames; std::map< diff --git a/cpp/skyweaver/SkyCleaverConfig.hpp b/cpp/skyweaver/SkyCleaverConfig.hpp index 6e4f76f..924d0d2 100644 --- a/cpp/skyweaver/SkyCleaverConfig.hpp +++ b/cpp/skyweaver/SkyCleaverConfig.hpp @@ -24,19 +24,20 @@ class SkyCleaverConfig std::size_t _nsamples_to_read; std::vector _required_beams; std::vector _required_dms; + std::string _targets; std::string _out_stokes; - std::vector _stokes_positions; + std::vector> _stokes_positions; public: SkyCleaverConfig() : _output_dir(""), _root_dir(""), _root_prefix(""), _out_prefix(""), - _nthreads(0), _nsamples_per_block(0), _nchans(0), _nbeams(0), + _nthreads(0), _nsamples_per_block(32768), _nchans(0), _nbeams(0), _max_ram_gb(0), _max_output_filesize(2147483647), _stream_id(0), _nbridges(64), _ndms(0), _stokes_mode("I"), _dada_header_size(4096), _start_sample(0), _nsamples_to_read(0), _required_beams({}), _required_dms({}), - _out_stokes("I"), _stokes_positions({}) + _targets(""), _out_stokes("I"), _stokes_positions({}) { } SkyCleaverConfig(SkyCleaverConfig const&) = delete; @@ -73,7 +74,11 @@ class SkyCleaverConfig void required_beams(std::vector required_beams) { _required_beams = required_beams; } void required_dms(std::vector required_dms) { _required_dms = required_dms; } void out_stokes(std::string out_stokes) { _out_stokes = out_stokes; } - void stokes_positions(std::vector stokes_positions) { _stokes_positions = stokes_positions; } + void stokes_positions(std::vector> stokes_positions) + { + _stokes_positions = stokes_positions; + } + void targets_file(std::string targets) { _targets = targets; } @@ -98,8 +103,11 @@ class SkyCleaverConfig std::vector required_beams() const { return _required_beams; } std::vector required_dms() const { return _required_dms; } std::string out_stokes() const { return _out_stokes; } - std::vector stokes_positions() const { return _stokes_positions; } - + std::vector> stokes_positions() const + { + return _stokes_positions; + } + std::string targets_file() const { return _targets; } }; diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index a17b669..b84cff7 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -3,6 +3,8 @@ #include "skyweaver/beamformer_utils.cuh" #include "skyweaver/types.cuh" +#include "skyweaver/skycleaver_utils.hpp" + #include #include @@ -16,9 +18,20 @@ namespace fs = std::filesystem; using BridgeReader = skyweaver::BridgeReader; using MultiFileReader = skyweaver::MultiFileReader; +using BeamInfo = skyweaver::BeamInfo; namespace { + +std::string trim(const std::string& str) { + auto start = str.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) { + return ""; // String is all whitespace + } + auto end = str.find_last_not_of(" \t\n\r"); + return str.substr(start, end - start + 1); +} + template std::string to_string_with_padding(T num, int width, int precision = -1) { @@ -71,6 +84,10 @@ std::vector get_files(std::string directory_path, if(fs::is_regular_file(entry.status())) { std::string file_name = entry.path().string(); if(file_name.find(extension) != std::string::npos) { + //check if .tmp not in filename + if(file_name.find(".tmp") != std::string::npos) { + continue; + } files.push_back(file_name); } } @@ -88,8 +105,62 @@ std::vector get_files(std::string directory_path, return files; } + } // namespace +void parse_target_file(std::string file_name, std::vector& beam_infos){ + std::ifstream targets_file(file_name); + if(!targets_file.is_open()) { + std::runtime_error("Error opening target file: " + file_name); + } + // the file is in csv format. First read the header to know the positions of name, ra and dec + std::string header; + do{ + std::getline(targets_file, header); + } while(header.empty() || header.find("#") != std::string::npos); + + std::vector header_tokens; + std::stringstream header_stream(header); + std::string token; + while(std::getline(header_stream, token, ',')) { + header_tokens.push_back(token); + } + std::size_t name_pos = std::distance(header_tokens.begin(), std::find(header_tokens.begin(), header_tokens.end(), "name")); + std::size_t ra_pos = std::distance(header_tokens.begin(), std::find(header_tokens.begin(), header_tokens.end(), "ra")); + std::size_t dec_pos = std::distance(header_tokens.begin(), std::find(header_tokens.begin(), header_tokens.end(), "dec")); + + if(name_pos == header_tokens.size() || ra_pos == header_tokens.size() || dec_pos == header_tokens.size()) { + std::runtime_error("Invalid header in target file: " + file_name); + } + + std::string line; + while(std::getline(targets_file, line)) { + + line = trim(line); + + //if empty line or # anywhere in line, continue + if(line.empty() || line.find("#") != std::string::npos) { + BOOST_LOG_TRIVIAL(debug) << "Skipping line: " << line; + continue; + } + + std::vector tokens; + std::stringstream line_stream(line); + std::string token; + while(std::getline(line_stream, token, ',')) { + tokens.push_back(token); + } + if(tokens.size() != header_tokens.size()) { + std::runtime_error("Invalid number of columns in target file: " + file_name); + } + BeamInfo beam_info; + beam_info.beam_name = tokens[name_pos]; + beam_info.beam_ra = tokens[ra_pos]; + beam_info.beam_dec = tokens[dec_pos]; + beam_infos.push_back(beam_info); + } +} + void compare_bridge_headers(const skyweaver::ObservationHeader& first, const skyweaver::ObservationHeader& second) { @@ -202,19 +273,30 @@ void skyweaver::SkyCleaver::init_readers() << " Stokes mode: " << _config.stokes_mode() << " Number of channels: " << _config.nchans(); - std::vector stokes_positions; - // make sure that every character in out_stokes is a valid stokes mode - // if present add the relevant position to the stokes_positions vector - for(auto stokes: _config.out_stokes()) { + std::vector> stokes_positions; + for(const auto stokes: _config.out_stokes()) { std::size_t pos = _config.stokes_mode().find(stokes); if(pos == std::string::npos) { - throw std::runtime_error("Invalid Stokes mode: " + stokes); + + if(stokes == 'L') { + std::size_t pos1 = _config.stokes_mode().find("Q"); + std::size_t pos2 = _config.stokes_mode().find("U"); + if(pos1 == std::string::npos || pos2 == std::string::npos) { + throw std::runtime_error("Asked for L, but beamformed data does not have Q and/or U"); + } + stokes_positions.push_back({pos1, pos2}); + continue; + } + else { + throw std::runtime_error("Requested stokes not found in beamformed data: " + stokes); + } } - stokes_positions.push_back(pos); + stokes_positions.push_back({pos}); } _config.stokes_positions(stokes_positions); + long double obs_centre_freq = _header.obs_frequency; long double obs_bandwidth = _header.obs_bandwidth; @@ -350,15 +432,16 @@ template void skyweaver::SkyCleaver::init_writers() { BOOST_LOG_NAMED_SCOPE("SkyCleaver::init_writers") + BOOST_LOG_TRIVIAL(debug) << "_config.output_dir(); " << _config.output_dir(); + std::string output_dir = _config.output_dir(); - if(!fs::exists(_config.output_dir())) { - fs::create_directories(_config.output_dir()); + if(!fs::exists(output_dir)) { + fs::create_directories(output_dir); } - std::string output_dir = _config.output_dir(); - for(std::size_t istokes = 0; istokes < _config.stokes_positions().size(); + for(std::size_t istokes = 0; istokes < _config.out_stokes().size(); istokes++) { for(int idm = 0; idm < _config.ndms(); idm++) { if(!_config.required_dms().empty()) { @@ -385,19 +468,18 @@ void skyweaver::SkyCleaver::init_writers() continue; } } + BeamInfo beam_info = _beam_infos[ibeam]; MultiFileWriterConfig writer_config; writer_config.header_size = _config.dada_header_size(); writer_config.max_file_size = _config.max_output_filesize(); writer_config.stokes_mode = _config.out_stokes().at(istokes); writer_config.base_output_dir = output_dir; writer_config.prefix = _config.out_prefix(); - std::string suffix = - _config.ndms() > 1 - ? "_idm_" + - to_string_with_padding(_header.dms[idm], 9, 3) - : ""; - writer_config.suffix = suffix + "_cb_" + - to_string_with_padding(ibeam, 5) + "_" + + std::string suffix = "idm_" + + to_string_with_padding(_header.dms[idm], 9, 3); + + writer_config.suffix = suffix + "_" + + beam_info.beam_name + "_" + _config.out_stokes().at(istokes); writer_config.extension = ".fil"; @@ -414,6 +496,9 @@ void skyweaver::SkyCleaver::init_writers() "", create_stream_callback_sigproc); _header.ibeam = ibeam; + + _header.ra = beam_info.beam_ra; + _header.dec = beam_info.beam_dec; _beam_writers[istokes][idm][ibeam]->init(_header); _beam_data[istokes][idm][ibeam] = @@ -436,9 +521,37 @@ skyweaver::SkyCleaver::SkyCleaver( SkyCleaverConfig& config) : _config(config) { + BOOST_LOG_TRIVIAL(info) << "Reading and initialising beam details from file: " + << _config.targets_file(); + + parse_target_file(_config.targets_file(), _beam_infos); + + BOOST_LOG_TRIVIAL(info) << "Number of beams in target file: " << _beam_infos.size(); + _timer.start("skycleaver::init_readers"); init_readers(); _timer.stop("skycleaver::init_readers"); + + + if(_beam_infos.size() < _config.nbeams()){ // there are some null beams with zeros, do not create filterbanks for them. + std::string required_beams = "0:" + std::to_string(_beam_infos.size()-1); + std::vector required_beam_numbers = skyweaver::get_list_from_string(required_beams); + + if(_config.required_beams().empty()) { // if nothing given, set the valid beams to required beams + BOOST_LOG_TRIVIAL(warning) << "Number of beams in target file is less than the number of beams in the header. " + << "Setting required beams to: " << required_beams; + _config.required_beams(skyweaver::get_list_from_string(required_beams)); + } + else{ + for(auto beam_num: _config.required_beams()){ // if given, check if all requested beams are valid beams + if(std::find(required_beam_numbers.begin(), required_beam_numbers.end(), beam_num) == required_beam_numbers.end()){ + std::runtime_error("Beam number " + std::to_string(beam_num) + " not found in target file."); + } + } + } + } + + _timer.start("skycleaver::init_writers"); init_writers(); _timer.stop("skycleaver::init_writers"); @@ -524,7 +637,7 @@ void skyweaver::SkyCleaver::cleave() std::size_t nbridges = _config.nbridges(); std::size_t ndms = _config.ndms(); std::size_t nbeams = _config.nbeams(); - std::size_t nstokes_out = _config.stokes_positions().size(); + std::size_t nstokes_out = _config.out_stokes().size(); std::size_t nstokes_in = _config.stokes_mode().size(); #pragma omp parallel for schedule(static) collapse(3) @@ -532,9 +645,7 @@ void skyweaver::SkyCleaver::cleave() for(std::size_t ibeam = 0; ibeam < nbeams; ibeam++) { for(std::size_t idm = 0; idm < ndms; idm++) { // cannot separate loops, so do checks later - BOOST_LOG_TRIVIAL(debug) - << "Processing data for stokes: " << istokes - << " beam: " << ibeam << " dm: " << idm; + if(_beam_data.find(istokes) == _beam_data.end()) { continue; } @@ -546,29 +657,55 @@ void skyweaver::SkyCleaver::cleave() _beam_data[istokes][idm].end()) { continue; } - const int stokes_position = - _config.stokes_positions()[istokes]; + + const std::vector stokes_positions = + _config.stokes_positions()[istokes]; + #pragma omp simd for(std::size_t isample = 0; isample < gulp_samples; isample++) { - // This is the input, so use nstokes_in - const std::size_t base_index = + const std::size_t out_offset = isample * nbridges; + + // This is stupid but preferred over a more elegant solution that is not fast, this can be easily vectorised + if(stokes_positions.size() == 1) { + const std::size_t base_index = isample * ndms * nbeams * nstokes_in + idm * nbeams * nstokes_in + ibeam * nstokes_in + - stokes_position; - - std::size_t ifreq = 0; - const std::size_t out_offset = isample * nbridges; - for(const auto& [freq, ifreq_data]: + stokes_positions[0]; + + std::size_t ifreq = 0; + for(const auto& [freq, ifreq_data]: _bridge_data) { // for each frequency _beam_data[istokes][idm][ibeam]->at( out_offset + nbridges - 1 - ifreq) = clamp(127 + ifreq_data->at(base_index)); ++ifreq; + } + } + else{ + std::size_t ifreq = 0; + for(const auto& [freq, ifreq_data]: _bridge_data) { + float value = 0; + for(int stokes_position=0; stokes_positionat(base_index) * ifreq_data->at(base_index)); + } + // for each frequency + _beam_data[istokes][idm][ibeam]->at( + out_offset + nbridges - 1 - ifreq) = + clamp(127 + + sqrt(value)); + ++ifreq; + } } + } + } } } @@ -578,15 +715,16 @@ void skyweaver::SkyCleaver::cleave() write(); _timer.stop("skyweaver::write_data"); } - _timer.show_all_timings(); + } + template void skyweaver::SkyCleaver::write() { omp_set_num_threads(_config.nthreads()); #pragma omp parallel for schedule(static) collapse(3) - for(std::size_t istokes = 0; istokes < _config.stokes_positions().size(); + for(std::size_t istokes = 0; istokes < _config.out_stokes().size(); istokes++) { for(std::size_t idm = 0; idm < _config.ndms(); idm++) { for(std::size_t ibeam = 0; ibeam < _config.nbeams(); ibeam++) { diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index 882d91b..e1c59dc 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -38,13 +38,10 @@ HEADER DADA HDR_VERSION 1.0 HDR_SIZE 4096 DADA_VERSION 1.0 - FILE_SIZE 100000000000 FILE_NUMBER 0 - UTC_START 1708082229.000020336 MJD_START 60356.47024305579093 - SOURCE J1644-4559 RA 16:44:49.27 DEC -45:59:09.7 @@ -54,7 +51,6 @@ RECEIVER L-band FREQ 1284000000.000000 BW 856000000.000000 TSAMP 4.7850467290 - NBIT 8 NDIM 1 NPOL 1 @@ -62,7 +58,7 @@ NCHAN 64 NBEAM 800 ORDER TFB CHAN0_IDX 2688 - )"; +)"; #define MJD_UNIX_EPOCH 40587.0 } // namespace diff --git a/cpp/skyweaver/src/skycleaver_cli.cpp b/cpp/skyweaver/src/skycleaver_cli.cpp index 9fec4af..88b4e55 100644 --- a/cpp/skyweaver/src/skycleaver_cli.cpp +++ b/cpp/skyweaver/src/skycleaver_cli.cpp @@ -2,6 +2,7 @@ #include "skyweaver/SkyCleaver.hpp" #include "skyweaver/SkyCleaverConfig.hpp" #include "skyweaver/logging.hpp" +#include "skyweaver/skycleaver_utils.hpp" #include #include @@ -43,51 +44,7 @@ const size_t ERROR_UNHANDLED_EXCEPTION = 2; const char* build_time = __DATE__ " " __TIME__; -template -std::vector -get_list_from_string(const std::string& value, - T epsilon = std::numeric_limits::epsilon()) -{ - std::vector output; - std::vector comma_chunks; - - // Split the input string by commas - std::stringstream ss(value); - std::string token; - while(std::getline(ss, token, ',')) { comma_chunks.push_back(token); } - - for(const auto& comma_chunk: comma_chunks) { - // Check if the chunk contains a colon (indicating a range) - if(comma_chunk.find(':') == std::string::npos) { - output.push_back(static_cast(std::atof(comma_chunk.c_str()))); - continue; - } - - // Split the range chunk by colons - std::stringstream ss_chunk(comma_chunk); - std::vector colon_chunks; - std::string colon_token; - while(std::getline(ss_chunk, colon_token, ':')) { - colon_chunks.push_back( - static_cast(std::atof(colon_token.c_str()))); - } - // Determine the step size - T step = colon_chunks.size() == 3 ? colon_chunks[2] : static_cast(1); - T start = colon_chunks[0]; - T stop = colon_chunks[1]; - - // Loop and add values to the output vector - if constexpr(std::is_floating_point::value) { - for(T k = start; k <= stop + epsilon; k += step) { - output.push_back(k); - } - } else { - for(T k = start; k <= stop; k += step) { output.push_back(k); } - } - } - return output; -} } // namespace @@ -192,17 +149,23 @@ int main(int argc, char** argv) ->notifier( [&config](std::size_t key) { config.nsamples_to_read(key); }), "total number of samples to read from start_sample")( + "targets_file", + po::value()->default_value("")->notifier( + [&config](std::string key) { + config.targets_file(key); + }), "update beam names and positions from this file" + )( "required_beams", po::value()->default_value("")->notifier( [&config](std::string key) { - config.required_beams(get_list_from_string(key)); + config.required_beams(skyweaver::get_list_from_string(key)); }), "Comma separated list of beams to process. Syntax - beam1, beam2, " "beam3:beam4:step, beam5 etc..")( "required_dms", po::value()->default_value("")->notifier( [&config](std::string key) { - config.required_dms(get_list_from_string(key)); + config.required_dms(skyweaver::get_list_from_string(key)); }), "Comma separated list of DMs to process. Syntax - dm1, dm2, " "dm1:dm2:step, etc..")( @@ -210,7 +173,7 @@ int main(int argc, char** argv) po::value() ->default_value(config.out_stokes()) ->notifier([&config](std::string key) { - if(key.find_first_not_of("IQUV") != std::string::npos) { + if(key.find_first_not_of("IQUVL") != std::string::npos) { throw std::runtime_error("Invalid Stokes mode: " + key); } config.out_stokes(key); From ff71497be6db64406ffc0b95a5fa5255c43b768a Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 12 Dec 2024 17:58:03 +0100 Subject: [PATCH 40/69] fix filename convention --- cpp/skyweaver/detail/MultiFileWriter.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 16b6c05..439f1ab 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -121,7 +121,7 @@ MultiFileWriter::get_basefilename(VectorType const& stream_data, base_filename << _config.prefix << "_"; } base_filename << get_formatted_time(_header.utc_start) << "_" << stream_idx - << "_" << std::fixed << std::setprecision(3) + << "_cdm_" << std::fixed << std::setprecision(3) << std::setfill('0') << std::setw(9) << stream_data.reference_dm(); From 45964ca84d7fc2bae5f461237938902b9820abf0 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Tue, 17 Dec 2024 13:31:26 +0100 Subject: [PATCH 41/69] draft documentation --- README.md | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/README.md b/README.md index ffc3e94..038f3bd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,177 @@ # skyweaver Implementation of an offline FBFUSE beamformer for the MeerKAT telescope + +# Installation + +It is easiest to use the software inside a Docker container. Two dockerfiles is included part of this repository: To compile the c++ and python parts respectively. + +# Usage + +## Step 1: Get delays for the beamformer + +### Start with the skyweaver python CLI + +```bash +alias sw="python /path/to/python/skyweaver/cli.py" +sw --help +``` +This will print: +```console +usage: skyweaver [-h] {metadata,delays} ... + +positional arguments: + {metadata,delays} sub-command help + metadata Tools for observation metadata files + delays Tools for delay files + +optional arguments: + -h, --help show this help message and exit +``` + +### Get the metadata for the corresponding observation + +This is done outside this repository + +### Obtain metadata information + +```bash +sw metadata show +``` + +This will produce an output like the following: +```console +sw metadata show bvrmetadata_2024-02-16T10\:50\:46_72275.hdf5 +---------------Array configuration---------------- +Nantennas: 57 +Subarray: m000,m002,m003,m004,m005,m007,m008,m009,m010,m011,m012,m014,m015,m016, + m017,m018,m019,m020,m021,m022,m023,m024,m025,m026,m027,m029,m030,m031, + m032,m033,m034,m035,m036,m037,m038,m039,m040,m041,m042,m043,m044,m045, + m046,m048,m049,m050,m051,m053,m054,m056,m057,m058,m059,m060,m061,m062, + m063 +Centre frequency: 1284000000.0 Hz +Bandwidth: 856000000.0 Hz +Nchannels: 4096 +Sync epoch (UNIX): 1708039531.0 +Project ID: - +Schedule block ID: - +CBF version: cbf_dev +--------------------Pointings--------------------- +#0 J1644-4559 2024-02-16T11:16:08.957000000 until 2024-02-16T11:21:04.865000000 (UTC) + 1708082168.957 until 1708082464.865 (UNIX) + 72996182384029 until 73502776880016 (SAMPLE CLOCK) +#1 J1644-4559_Offset1 2024-02-16T11:21:22.092000000 until 2024-02-16T11:26:15.682000000 (UTC) + 1708082482.092 until 1708082775.682 (UNIX) + 73532269504013 until 74034895583866 (SAMPLE CLOCK) +#2 J1644-4559_Offset2 2024-02-16T11:26:34.536000000 until 2024-02-16T11:31:25.723000000 (UTC) + 1708082794.536 until 1708083085.723 (UNIX) + 74067173632022 until 74565685776084 (SAMPLE CLOCK) +#3 M28 2024-02-16T11:31:52.503000000 until 2024-02-16T12:01:51.651000000 (UTC) + 1708083112.503 until 1708084911.651 (UNIX) + 74611533136035 until 77691674512039 (SAMPLE CLOCK) +#4 J0437-4715 2024-02-16T12:03:06.117000000 until 2024-02-16T12:08:04.364000000 (UTC) + 1708084986.117 until 1708085284.364 (UNIX) + 77819160304176 until 78329759168140 (SAMPLE CLOCK) +``` + +### Create a config file in .yml format +Here is an example - there are comments to explain each parameter. + +```.yml +created_by: Vivek +beamformer_config: + # The total number of beams to be produced (must be a multiple of 32). This needs to be <= the number that SKYWEAVER is compiled for. + total_nbeams: 800 + # The number of time samples that will be accumulated after detection, inside the beamformer + tscrunch: 4 + # The number of frequency channels that will be accumulated after detection, inside the beamformer + # Will be coerced to 1 if coherent dedispersion is specified. + fscrunch: 1 + # The Stokes product to be calculated in the beamformer (I=0, Q=1, U=2, V=3) + stokes_mode: 0 + # Enable CB-IB subtraction in the beamformer + subtract_ib: True + + # Dispersion measure for coherent / incoherent dedispersion in pc cm^-3 + # A dispersion plan definition string " + # "(::::) or " + # "(:) " + # "or ()") +# Each DD plan is a "Stream" with zero indexed stream-ids + +ddplan: + - "478.6:478.6:478.6:1:1" #stream-id=0 + - "0.00:478.6:478.6:1:1" #stream-id=1 + +# every beamset can contain arbitrary set of antennas, corresponding targeted beams, and tiled beams +# total number of beams across all beamsets should be <= the number of beams that SKYWEAVER is compiled for. +beam_sets: + + - antenna_set: ['m000','m002','m003','m004','m005','m007','m008','m009','m010','m011', + 'm012','m014','m015','m016','m017','m018','m019','m020','m021','m022', + 'm023','m024','m025','m026','m027','m029','m030','m031','m032','m033', + 'm034','m035','m036','m037','m038','m039','m040','m041','m042','m043', + 'm044','m045','m046','m048','m049','m050','m051','m053','m054','m056', + 'm057','m058','m059','m060','m061','m062','m063'] + beams: [] + tilings: + - nbeams: 32 + reference_frequency: null + target: "J1644-4559,radec,16:44:49.273,-45:59:09.71" + overlap: 0.9 +``` + + + +### Create delay file for the corresponding pointing + +```bash +sw delays create --pointing-idx 0 --outfile J1644-4559_pointing_0.delays --step 4 bvrmetadata_2024-02-16T10\:50\:46_72275.hdf5 J1644-4559_boresight.yaml +``` + +This produces a `.delays` file used for beamforming, and a `.targets` file that contains beam metadata. There are also other files produced here for reproducibility and for visualisation. + +## Step 2: Initialise input and compile skyweaver + +### Create a list of dada files that correspond to the pointing + +```console +ls /b/u/vivek/00_DADA_FILES/J1644-4559/2024-02-16-11\:16\:08/L/48/*dada -1 > /bscratch/vivek/skyweaver_tests/J1644-4559_boresight_dadafiles.list +``` + +### Compile skyweaver + +This is done inside the dockerfile too. Either edit that to produce a docker image that has the software precompiled, or compile separately. + +```bash + cmake -S . -B $cmake_tmp_dir -DENABLE_TESTING=0 -DCMAKE_INSTALL_PREFIX=$install_dir -DARCH=native -DPSRDADA_INCLUDE_DIR=/usr/local/include/psrdada -DPSRDADACPP_INCLUDE_DIR=/usr/local/include/psrdada_cpp -DSKYWEAVER_NANTENNAS=64 -DSKYWEAVER_NBEAMS=${nbeams} -DSKYWEAVER_NCHANS=64 -DSKYWEAVER_IB_SUBTRACTION=1 -DCMAKE_BUILD_TYPE=RELEASE -DSKYWEAVER_CB_TSCRUNCH=${tscrunch} -DSKYWEAVER_IB_TSCRUNCH=${tscrunch}; + cd $cmake_tmp_dir + make -j 16 +``` +This compilation produces two binaries: `skyweavercpp` and `skycleaver` +## Step 3: Run the beamformer + +```bash + +/path/to/skyweavercpp --input-file J1644-4559_boresight_dadafiles.list --delay-file J1644-4559_pointing_0.delays --output-dir=/bscratch/vivek/skyweaver_out --gulp-size=32768 --log-level=warning --output-level=12 --stokes-mode I +``` + +Change the output level to 7 for bright pulsars like J1644-4559. + +This will produce `.tdb` files for the corresponding bridge. Run Step 3 for ALL 64 bridges with their corresponding dada file lists. These are DADA format files with the dimensions of TIME, INCOHERENT DM and BEAM as the order. For stokes I mode, The datatype is `int8_t`. For IQUV it is `char4`. + +## Steo 4: Cleave all bridges to form Filterbanks + +Here we cleave the 64 TDB[I/Q/U/V/IV/QU/IQUV] files to produce `NDM*NBEAMS*NSTOKES` number of T(F=64) files. + +to run this, do + +```bash + +/path/to/skycleaver -r /bscratch/vivek/skyweaver_out --output-dir /bscratch/vivek/skycleaver_out --nsamples-per-block 65536 --nthreads 32 --stream-id 0 --targets_file/bscratch/vivek/skyweaver_out/swdlays_J1644-4559.targets --out-stokes I --required_beams 0 + +``` + +This will produce a standard sigproc format `.fil` file that can be used for traditional processing. + + + From 8610cab48b4eb0eefefcf144d1daa7c81e8dfc2e Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Sun, 22 Dec 2024 14:35:08 +0100 Subject: [PATCH 42/69] Revert "compiling now and beamformer tests running" This reverts commit 820efb994e341feea8119be5ec12b90143905b90. --- cpp/skyweaver/StatisticsCalculator.cuh | 2 +- cpp/skyweaver/beamformer_utils.cuh | 16 ++--- cpp/skyweaver/benchmark/beamformer_bench.cu | 2 +- cpp/skyweaver/src/StatisticsCalculator.cu | 10 +-- .../test/CoherentBeamformerTester.cuh | 17 +---- .../test/IncoherentBeamformerTester.cuh | 7 +- .../test/src/CoherentBeamformerTester.cu | 71 ++++++++----------- .../test/src/IncoherentBeamformerTester.cu | 34 ++++----- 8 files changed, 64 insertions(+), 95 deletions(-) diff --git a/cpp/skyweaver/StatisticsCalculator.cuh b/cpp/skyweaver/StatisticsCalculator.cuh index 1b9dee3..5767a78 100644 --- a/cpp/skyweaver/StatisticsCalculator.cuh +++ b/cpp/skyweaver/StatisticsCalculator.cuh @@ -104,7 +104,7 @@ class StatisticsCalculator void dump_scalings(std::string const& timestamp, std::string const& tag, std::string const& path, - ScalingVectorTypeH const& ar) const; + thrust::host_vector const& ar) const; private: PipelineConfig const& _config; diff --git a/cpp/skyweaver/beamformer_utils.cuh b/cpp/skyweaver/beamformer_utils.cuh index 15ec773..ae165e8 100644 --- a/cpp/skyweaver/beamformer_utils.cuh +++ b/cpp/skyweaver/beamformer_utils.cuh @@ -275,11 +275,11 @@ struct IncoherentBeamSubtract { { if constexpr(S == StokesParameter::I) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.x); - } else if constexpr(S == StokesParameter::Q) { + } constexpr(S == StokesParameter::Q) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.y); - } else if constexpr(S == StokesParameter::U) { + } constexpr(S == StokesParameter::U) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.z); - } else if constexpr(S == StokesParameter::V) { + } constexpr(S == StokesParameter::V) { AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.w); } else { static_no_match(); @@ -290,17 +290,17 @@ struct IncoherentBeamSubtract { struct Rescale { template static inline __host__ __device__ void apply(T const& power, - float4 const& offset, - float4 const& scale_factor, + float const& offset, + float const& scale_factor, T& result) { if constexpr(S == StokesParameter::I) { AT(result, I) = rintf((AT(power, I) - offset.x) / scale_factor.x); - } else if constexpr(S == StokesParameter::Q) { + } constexpr(S == StokesParameter::Q) { AT(result, I) = rintf((AT(power, I) - offset.y) / scale_factor.y); - } else if constexpr(S == StokesParameter::U) { + } constexpr(S == StokesParameter::U) { AT(result, I) = rintf((AT(power, I) - offset.z) / scale_factor.z); - } else if constexpr(S == StokesParameter::V) { + } constexpr(S == StokesParameter::V) { AT(result, I) = rintf((AT(power, I) - offset.w) / scale_factor.w); } else { static_no_match(); diff --git a/cpp/skyweaver/benchmark/beamformer_bench.cu b/cpp/skyweaver/benchmark/beamformer_bench.cu index 663cb3b..a179741 100644 --- a/cpp/skyweaver/benchmark/beamformer_bench.cu +++ b/cpp/skyweaver/benchmark/beamformer_bench.cu @@ -70,7 +70,7 @@ class BeamformerBencher: public benchmark::Fixture typename CoherentBeamformer::PowerVectorTypeD btf_powers_gpu; typename CoherentBeamformer::ScalingVectorTypeD scales; typename CoherentBeamformer::ScalingVectorTypeD offsets; - typename IncoherentBeamformer::BeamsetWeightsVectorTypeD _antenna_weights; + typename CoherentBeamformer::ScalingVectorTypeD _antenna_weights; typename IncoherentBeamformer::PowerVectorTypeD tf_powers_gpu; typename IncoherentBeamformer::RawPowerVectorTypeD tf_powers_raw_gpu; typename CoherentBeamformer::MappingVectorTypeD _beamset_mapping; diff --git a/cpp/skyweaver/src/StatisticsCalculator.cu b/cpp/skyweaver/src/StatisticsCalculator.cu index a8dcb33..a2cf68c 100644 --- a/cpp/skyweaver/src/StatisticsCalculator.cu +++ b/cpp/skyweaver/src/StatisticsCalculator.cu @@ -191,15 +191,15 @@ void StatisticsCalculator::update_scalings( float var_p1_sum = 0.0f; // sum(sigma_p1^2) float quad_sum = 0.0f; // sum(sigma_p0^4 + sigma_p1^4) float sum_mul = 0.0f; // sum(sigma_p0^2 * sigma_p1^2) - for (int a_idx = 0; a_idx < _config.npol(); ++a_idx) { + for (int a_idx = 0; a_idx < _cofig.npol(); ++a_idx) { // stats are in FPA order // weights are in FPBA order const int input_idx = f_idx * pa + a_idx; Statistics p0 = _stats_h[input_idx]; - Statistics p1 = _stats_h[input_idx + a]; + Statistics p1 = _stats_h[input_idx + nantennas]; const float weight = beamset_weights[beamset_idx * a + a_idx]; - const float std_p0 = p0.std * weight * weights_amp; - const float std_p1 = p1.std * weight * weights_amp; + const float std_p0 = p0.std * weight; + const float std_p1 = p1.std * weight; const float var_p0 = std_p0 * std_p0; const float var_p1 = std_p1 * std_p1; meanI += var_p0 + var_p1; @@ -279,7 +279,7 @@ void StatisticsCalculator::dump_scalings( BOOST_LOG_TRIVIAL(error) << error_message.str(); return; } - writer.write(reinterpret_cast(thrust::raw_pointer_cast(ar.data())), ar.size() * sizeof(ScalingType)); + writer.write(static_cast(ar.data()), ar.size() * sizeof(ScalingType)); writer.close(); } diff --git a/cpp/skyweaver/test/CoherentBeamformerTester.cuh b/cpp/skyweaver/test/CoherentBeamformerTester.cuh index e956de1..fd60262 100644 --- a/cpp/skyweaver/test/CoherentBeamformerTester.cuh +++ b/cpp/skyweaver/test/CoherentBeamformerTester.cuh @@ -21,37 +21,26 @@ class CoherentBeamformerTester: public ::testing::Test typedef CoherentBeamformer CoherentBeamformer; typedef IncoherentBeamformer IncoherentBeamformer; typedef CoherentBeamformer::VoltageVectorTypeD VoltageVectorTypeD; - typedef FTPAVoltagesH VoltageVectorTypeH; - typedef CoherentBeamformer::PowerVectorTypeD PowerVectorTypeD; typedef TFBPowersH HostPowerVectorType; - typedef IncoherentBeamformer::PowerVectorTypeD IBPowerVectorTypeD; typedef BTFPowersH HostIBPowerVectorType; - typedef IncoherentBeamformer::RawPowerVectorTypeD RawIBPowerVectorTypeD; typedef BTFPowersH HostRawIBPowerVectorType; - typedef CoherentBeamformer::WeightsVectorTypeD WeightsVectorTypeD; typedef thrust::host_vector WeightsVectorTypeH; - typedef CoherentBeamformer::ScalingVectorTypeD ScalingVectorTypeD; typedef thrust::host_vector HostScalingVectorType; - typedef CoherentBeamformer::MappingVectorTypeD MappingVectorTypeD; typedef thrust::host_vector MappingVectorTypeH; - - typedef IncoherentBeamformer::BeamsetWeightsVectorTypeD BeamsetWeightsVectorTypeD; - typedef thrust::host_vector - BeamsetWeightsVectorTypeH; protected: void SetUp() override; @@ -72,8 +61,8 @@ class CoherentBeamformerTester: public ::testing::Test int nbeams, int nantennas, int npol, - float4 const* scales, - float4 const* offsets, + float const* scales, + float const* offsets, float const* antenna_weights, int const* beamset_mapping); @@ -81,7 +70,7 @@ class CoherentBeamformerTester: public ::testing::Test WeightsVectorTypeD const& fbpa_weights_gpu, ScalingVectorTypeD const& scales_gpu, ScalingVectorTypeD const& offsets_gpu, - BeamsetWeightsVectorTypeD const& antenna_weights, + ScalingVectorTypeD const& antenna_weights, MappingVectorTypeD const& beamset_mapping, PowerVectorTypeD& btf_powers_gpu, int nsamples); diff --git a/cpp/skyweaver/test/IncoherentBeamformerTester.cuh b/cpp/skyweaver/test/IncoherentBeamformerTester.cuh index 5983321..37da649 100644 --- a/cpp/skyweaver/test/IncoherentBeamformerTester.cuh +++ b/cpp/skyweaver/test/IncoherentBeamformerTester.cuh @@ -31,9 +31,6 @@ class IncoherentBeamformerTester: public ::testing::Test typedef IncoherentBeamformer::ScalingVectorTypeD ScalingVectorTypeD; typedef thrust::host_vector HostScalingVectorType; - typedef IncoherentBeamformer::BeamsetWeightsVectorTypeD BeamsetWeightsVectorTypeD; - typedef thrust::host_vector - HostBeamsetWeightsVectorType; protected: void SetUp() override; @@ -54,7 +51,7 @@ class IncoherentBeamformerTester: public ::testing::Test int nantennas, HostScalingVectorType const& scale, HostScalingVectorType const& offset, - HostBeamsetWeightsVectorType const& beamset_weights, + HostScalingVectorType const& beamset_weights, int nbeamsets); void compare_against_host(VoltageVectorTypeD const& ftpa_voltages_gpu, @@ -62,7 +59,7 @@ class IncoherentBeamformerTester: public ::testing::Test DevicePowerVectorType& tf_powers_gpu, ScalingVectorTypeD const& scaling_vector, ScalingVectorTypeD const& offset_vector, - BeamsetWeightsVectorTypeD const& beamset_weights, + ScalingVectorTypeD const& beamset_weights, int ntimestamps, int nbeamsets); diff --git a/cpp/skyweaver/test/src/CoherentBeamformerTester.cu b/cpp/skyweaver/test/src/CoherentBeamformerTester.cu index 9a98df5..a369015 100644 --- a/cpp/skyweaver/test/src/CoherentBeamformerTester.cu +++ b/cpp/skyweaver/test/src/CoherentBeamformerTester.cu @@ -51,8 +51,8 @@ void CoherentBeamformerTester::beamformer_c_reference( int nbeams, int nantennas, int npol, - float4 const* scales, - float4 const* offsets, + float const* scales, + float const* offsets, float const* antenna_weights, int const* beamset_mapping) { @@ -157,7 +157,7 @@ void CoherentBeamformerTester::compare_against_host( WeightsVectorTypeD const& fbpa_weights_gpu, ScalingVectorTypeD const& scales_gpu, ScalingVectorTypeD const& offsets_gpu, - BeamsetWeightsVectorTypeD const& antenna_weights, + ScalingVectorTypeD const& antenna_weights, MappingVectorTypeD const& beamset_mapping, PowerVectorTypeD& btf_powers_gpu, int nsamples) @@ -170,7 +170,7 @@ void CoherentBeamformerTester::compare_against_host( HostScalingVectorType scales = scales_gpu; HostScalingVectorType offsets = offsets_gpu; - thrust::host_vector antenna_weights_host = antenna_weights; + HostScalingVectorType antenna_weights_host = antenna_weights; MappingVectorTypeH beamset_mapping_host = beamset_mapping; beamformer_c_reference( @@ -227,36 +227,29 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) const float input_level = 32.0f; const double pi = std::acos(-1); config.output_level(input_level); - - float4 cb_offset_val{ - static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(127.0f * input_level, 2)) / config.output_level()), - 0.0f, - 0.0f, - 0.0f - }; - float cb_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * std::pow(config.nantennas() * std::pow(127.0f * input_level, 2), 2) ) / config.output_level()); - float4 cb_scale_val{ - cb_scaling, - cb_scaling, - cb_scaling, - cb_scaling - }; + float scale = + std::pow(127.0f * input_level * + std::sqrt(static_cast(config.nantennas())), + 2); + float dof = 2 * config.cb_tscrunch() * config.cb_fscrunch() * config.npol(); + float offset_val = (scale * dof); + float scale_val = (scale * std::sqrt(2 * dof) / config.output_level()); // Set constant scales and offsets for all channels and beamsets typename CBT::ScalingVectorTypeD cb_scales( config.nchans() / config.cb_fscrunch() * nbeamsets, - cb_scale_val); + scale_val); typename CBT::ScalingVectorTypeD cb_offsets( config.nchans() / config.cb_fscrunch() * nbeamsets, - cb_offset_val); + offset_val); // Map all beams to the first beamset by default typename CBT::MappingVectorTypeD beamset_mapping(config.nbeams(), 0); // Enable all antennas in all beamsets - typename thrust::device_vector beamset_weights(config.nantennas() * + typename CBT::ScalingVectorTypeD beamset_weights(config.nantennas() * nbeamsets, - 1.0f); + 1.0f); /** This currently causes tests to fail even though the weights are the same for @@ -267,6 +260,9 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) */ beamset_mapping[0] = 1; + BOOST_LOG_TRIVIAL(info) << "CB scaling: " << scale_val; + BOOST_LOG_TRIVIAL(info) << "CB offset: " << offset_val; + std::default_random_engine generator; std::normal_distribution normal_dist(0.0, input_level); std::uniform_real_distribution uniform_dist(0.0, 2 * pi); @@ -284,6 +280,9 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) std::size_t input_size = (ntimestamps * config.nantennas() * config.nchans() * config.nsamples_per_heap() * config.npol()); + BOOST_LOG_TRIVIAL(info) << "FTPA input dims: " << config.nchans() << ", " + << ntimestamps * config.nsamples_per_heap() << ", " + << config.npol() << ", " << config.nantennas(); int nsamples = config.nsamples_per_heap() * ntimestamps; std::size_t weights_size = @@ -315,28 +314,20 @@ TYPED_TEST(CoherentBeamformerTester, representative_noise_test) typename CBT::WeightsVectorTypeD fbpa_weights_gpu = fbpa_weights_host; typename CBT::PowerVectorTypeD tfb_powers_gpu; - float4 ib_offset_val{ - static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 2)) / config.output_level()), - 0.0f, - 0.0f, - 0.0f - }; - - float ib_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 4)) / config.output_level()); - float4 ib_scale_val{ - ib_scaling, - ib_scaling, - ib_scaling, - ib_scaling - }; - + // Note that below even though this is for the IB we have to use the + // CB scrunching parameters to make sure we get the right data out. + float ib_scale = std::pow(input_level, 2); + float ib_dof = 2 * config.cb_tscrunch() * config.cb_fscrunch() * + config.nantennas() * config.npol(); + float ib_power_offset = ib_scale * ib_dof; + float ib_power_scaling = + ib_scale * std::sqrt(2 * ib_dof) / config.output_level(); typename CBT::ScalingVectorTypeD ib_scales( config.nchans() / config.cb_fscrunch() * nbeamsets, - ib_scale_val); + ib_power_scaling); typename CBT::ScalingVectorTypeD ib_offset( config.nchans() / config.cb_fscrunch() * nbeamsets, - ib_offset_val); - + ib_power_offset); typename CBT::IBPowerVectorTypeD tf_powers_gpu; typename CBT::RawIBPowerVectorTypeD tf_powers_raw_gpu; diff --git a/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu b/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu index d7af1c5..a52f675 100644 --- a/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu +++ b/cpp/skyweaver/test/src/IncoherentBeamformerTester.cu @@ -49,7 +49,7 @@ void IncoherentBeamformerTester::beamformer_c_reference( int nantennas, HostScalingVectorType const& scale, HostScalingVectorType const& offset, - HostBeamsetWeightsVectorType const& beamset_weights, + HostScalingVectorType const& beamset_weights, int nbeamsets) { static_assert(SKYWEAVER_NPOL == 2, "Tests only work for dual poln data."); @@ -109,7 +109,7 @@ void IncoherentBeamformerTester::compare_against_host( DevicePowerVectorType& tf_powers_gpu, ScalingVectorTypeD const& scaling_vector, ScalingVectorTypeD const& offset_vector, - BeamsetWeightsVectorTypeD const& beamset_weights, + ScalingVectorTypeD const& beamset_weights, int ntimestamps, int nbeamsets) { @@ -118,7 +118,7 @@ void IncoherentBeamformerTester::compare_against_host( HostRawPowerVectorType tf_powers_raw_cuda = tf_powers_raw_gpu; HostScalingVectorType h_scaling_vector = scaling_vector; HostScalingVectorType h_offset_vector = offset_vector; - HostBeamsetWeightsVectorType h_beamset_weights = beamset_weights; + HostScalingVectorType h_beamset_weights = beamset_weights; HostRawPowerVectorType tf_powers_raw_host; tf_powers_raw_host.like(tf_powers_raw_gpu); HostPowerVectorType tf_powers_host; @@ -175,31 +175,23 @@ TYPED_TEST(IncoherentBeamformerTester, ib_representative_noise_test) static_cast(std::lround(normal_dist(generator))); } - float4 ib_offset_val{ - static_cast(std::sqrt(4 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 2)) / config.output_level()), - 0.0f, - 0.0f, - 0.0f - }; - - float ib_scaling = static_cast(std::sqrt(8 * config.cb_tscrunch() * config.cb_fscrunch() * config.nantennas() * std::pow(input_level, 4)) / config.output_level()); - float4 ib_scale_val{ - ib_scaling, - ib_scaling, - ib_scaling, - ib_scaling - }; + float ib_scale = std::pow(input_level, 2); + float ib_dof = 2 * config.ib_tscrunch() * config.ib_fscrunch() * + config.nantennas() * config.npol(); + float ib_power_offset = ib_scale * ib_dof; + float ib_power_scaling = + ib_scale * std::sqrt(2 * ib_dof) / config.output_level(); for(int nbeamsets = 1; nbeamsets < 5; ++nbeamsets) { typename IBT::ScalingVectorTypeD scales( config.nchans() / config.ib_fscrunch() * nbeamsets, - ib_scale_val); + ib_power_scaling); typename IBT::ScalingVectorTypeD offset( config.nchans() / config.ib_fscrunch() * nbeamsets, - ib_offset_val); - typename IBT::BeamsetWeightsVectorTypeD beamset_weights(config.nantennas() * + ib_power_offset); + typename IBT::ScalingVectorTypeD beamset_weights(config.nantennas() * nbeamsets, - 1.0f); + 1.0f); typename IBT::VoltageVectorTypeD ftpa_voltages_gpu = ftpa_voltages_host; typename IBT::DevicePowerVectorType tf_powers_gpu; From 112a857d41062342d1aff003e416b2f180e98172 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Sun, 22 Dec 2024 14:35:32 +0100 Subject: [PATCH 43/69] Revert "dirty pass on scaling change" This reverts commit cbaf03bca758f85ba14e07600d1a3ac12e036f41. --- cpp/skyweaver/CoherentBeamformer.cuh | 6 +- cpp/skyweaver/IncoherentBeamformer.cuh | 10 +- cpp/skyweaver/StatisticsCalculator.cuh | 6 +- cpp/skyweaver/beamformer_utils.cuh | 34 ++--- cpp/skyweaver/src/CoherentBeamformer.cu | 10 +- cpp/skyweaver/src/IncoherentBeamformer.cu | 14 +- cpp/skyweaver/src/StatisticsCalculator.cu | 166 ++++++++++------------ 7 files changed, 109 insertions(+), 137 deletions(-) diff --git a/cpp/skyweaver/CoherentBeamformer.cuh b/cpp/skyweaver/CoherentBeamformer.cuh index d10cc35..dc47b9c 100644 --- a/cpp/skyweaver/CoherentBeamformer.cuh +++ b/cpp/skyweaver/CoherentBeamformer.cuh @@ -33,8 +33,8 @@ __global__ void bf_ftpa_general_k( int2 const* __restrict__ ftpa_voltages, int2 const* __restrict__ fbpa_weights, typename BfTraits::QuantisedPowerType* __restrict__ btf_powers, - float4 const* __restrict__ output_scale, - float4 const* __restrict__ output_offset, + float const* __restrict__ output_scale, + float const* __restrict__ output_offset, int const* __restrict__ beamset_mapping, typename BfTraits::RawPowerType const* __restrict__ ib_powers, int nsamples); @@ -55,7 +55,7 @@ class CoherentBeamformer typedef BTFPowersD RawPowerVectorTypeD; // FBA order (assuming equal weight per polarisation) typedef thrust::device_vector WeightsVectorTypeD; - typedef thrust::device_vector ScalingVectorTypeD; + typedef thrust::device_vector ScalingVectorTypeD; typedef thrust::device_vector MappingVectorTypeD; public: diff --git a/cpp/skyweaver/IncoherentBeamformer.cuh b/cpp/skyweaver/IncoherentBeamformer.cuh index ef18b6d..e40d73b 100644 --- a/cpp/skyweaver/IncoherentBeamformer.cuh +++ b/cpp/skyweaver/IncoherentBeamformer.cuh @@ -34,8 +34,8 @@ __global__ void icbf_ftpa_general_k( char2 const* __restrict__ ftpa_voltages, typename BfTraits::RawPowerType* __restrict__ tf_powers_raw, typename BfTraits::QuantisedPowerType* __restrict__ tf_powers, - float4 const* __restrict__ output_scale, - float4 const* __restrict__ output_offset, + float const* __restrict__ output_scale, + float const* __restrict__ output_offset, float const* __restrict__ antenna_weights, int nsamples, int nbeamsets); @@ -52,9 +52,7 @@ class IncoherentBeamformer // TF order typedef BTFPowersD RawPowerVectorTypeD; // TF order - typedef thrust::device_vector ScalingVectorTypeD; // Always IQUV scalings - // BsA order - typedef thrust::device_vector BeamsetWeightsVectorTypeD; + typedef thrust::device_vector ScalingVectorTypeD; public: /** @@ -88,7 +86,7 @@ class IncoherentBeamformer PowerVectorTypeD& output, ScalingVectorTypeD const& output_scale, ScalingVectorTypeD const& output_offset, - BeamsetWeightsVectorTypeD const& antenna_weights, + ScalingVectorTypeD const& antenna_weights, int nbeamsets, cudaStream_t stream); diff --git a/cpp/skyweaver/StatisticsCalculator.cuh b/cpp/skyweaver/StatisticsCalculator.cuh index 5767a78..204f82b 100644 --- a/cpp/skyweaver/StatisticsCalculator.cuh +++ b/cpp/skyweaver/StatisticsCalculator.cuh @@ -39,11 +39,9 @@ struct StatisticsFileHeader { class StatisticsCalculator { public: - typedef float4 ScalingType; // Always caclulate for full stokes + typedef float ScalingType; typedef thrust::device_vector ScalingVectorTypeD; typedef thrust::host_vector ScalingVectorTypeH; - typedef thrust::device_vector BeamsetWeightsVectorTypeD; - typedef thrust::host_vector BeamsetWeightsVectorTypeH; typedef FPAStatsD StatisticsVectorTypeD; typedef FPAStatsH StatisticsVectorTypeH; @@ -95,7 +93,7 @@ class StatisticsCalculator * @brief Update the scaling arrays based on the last statistics * calculation. */ - void update_scalings(BeamsetWeightsVectorTypeH const& beamset_weights, + void update_scalings(ScalingVectorTypeH const& beamset_weights, int nbeamsets); private: diff --git a/cpp/skyweaver/beamformer_utils.cuh b/cpp/skyweaver/beamformer_utils.cuh index ae165e8..74d4873 100644 --- a/cpp/skyweaver/beamformer_utils.cuh +++ b/cpp/skyweaver/beamformer_utils.cuh @@ -270,20 +270,10 @@ struct IncoherentBeamSubtract { static inline __host__ __device__ void apply(T const& power, T const& ib_power, float const& ib_mutliplier, // 127^2 as default - float4 const& scale_factor, + float const& scale_factor, T& result) { - if constexpr(S == StokesParameter::I) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.x); - } constexpr(S == StokesParameter::Q) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.y); - } constexpr(S == StokesParameter::U) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.z); - } constexpr(S == StokesParameter::V) { - AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor.w); - } else { - static_no_match(); - } + AT(result, I) = rintf((AT(power, I) - AT(ib_power, I) * ib_mutliplier) / scale_factor); } }; @@ -295,16 +285,10 @@ struct Rescale { T& result) { if constexpr(S == StokesParameter::I) { - AT(result, I) = rintf((AT(power, I) - offset.x) / scale_factor.x); - } constexpr(S == StokesParameter::Q) { - AT(result, I) = rintf((AT(power, I) - offset.y) / scale_factor.y); - } constexpr(S == StokesParameter::U) { - AT(result, I) = rintf((AT(power, I) - offset.z) / scale_factor.z); - } constexpr(S == StokesParameter::V) { - AT(result, I) = rintf((AT(power, I) - offset.w) / scale_factor.w); + AT(result, I) = rintf((AT(power, I) - offset) / scale_factor); } else { - static_no_match(); - } + AT(result, I) = rintf(AT(power, I) / scale_factor); + } } }; @@ -351,16 +335,16 @@ struct StokesTraits ib_subtract(RawPowerType const& power, RawPowerType const& ib_power, float const& ib_mutliplier, - float4 const& scale_factor) { + float const& scale_factor) { RawPowerType result{}; - Iterate<0, Stokes...>::template apply(power, ib_power, ib_mutliplier, scale_factor, result); + Iterate<0, Stokes...>::template apply(power, ib_power, ib_mutliplier, scale_factor, result); return result; } static inline __host__ __device__ RawPowerType rescale(RawPowerType const& power, - float4 const& offset, - float4 const& scale_factor) { + float const& offset, + float const& scale_factor) { RawPowerType result{}; Iterate<0, Stokes...>::template apply(power, offset, scale_factor, result); return result; diff --git a/cpp/skyweaver/src/CoherentBeamformer.cu b/cpp/skyweaver/src/CoherentBeamformer.cu index ff1e563..43a8688 100644 --- a/cpp/skyweaver/src/CoherentBeamformer.cu +++ b/cpp/skyweaver/src/CoherentBeamformer.cu @@ -14,8 +14,8 @@ __global__ void bf_ftpa_general_k( int2 const* __restrict__ ftpa_voltages, int2 const* __restrict__ fbpa_weights, typename BfTraits::QuantisedPowerType* __restrict__ tfb_powers, - float4 const* __restrict__ output_scale, - float4 const* __restrict__ output_offset, + float const* __restrict__ output_scale, + float const* __restrict__ output_offset, int const* __restrict__ beamset_mapping, typename BfTraits::RawPowerType const* __restrict__ ib_powers, int nsamples) @@ -156,7 +156,7 @@ __global__ void bf_ftpa_general_k( int const ib_power_idx = beamset_idx * nsamps_out * gridDim.y + output_sample_idx * gridDim.y + blockIdx.y; int const scloff_idx = beamset_idx * gridDim.y + blockIdx.y; - float4 scale = output_scale[scloff_idx]; + float scale = output_scale[scloff_idx]; typename BfTraits::RawPowerType ib_power = ib_powers[ib_power_idx]; #if SKYWEAVER_IB_SUBTRACTION /* @@ -246,9 +246,9 @@ void CoherentBeamformer::beamform( char2 const* fbpa_weights_ptr = thrust::raw_pointer_cast(weights.data()); typename BfTraits::QuantisedPowerType* tfb_powers_ptr = thrust::raw_pointer_cast(output.data()); - float4 const* power_scaling_ptr = + float const* power_scaling_ptr = thrust::raw_pointer_cast(output_scale.data()); - float4 const* power_offset_ptr = + float const* power_offset_ptr = thrust::raw_pointer_cast(output_offset.data()); typename BfTraits::RawPowerType const* ib_powers_ptr = thrust::raw_pointer_cast(ib_powers.data()); diff --git a/cpp/skyweaver/src/IncoherentBeamformer.cu b/cpp/skyweaver/src/IncoherentBeamformer.cu index ee4a80e..c91618e 100644 --- a/cpp/skyweaver/src/IncoherentBeamformer.cu +++ b/cpp/skyweaver/src/IncoherentBeamformer.cu @@ -17,8 +17,8 @@ __global__ void icbf_ftpa_general_k( char2 const* __restrict__ ftpa_voltages, typename BfTraits::RawPowerType* __restrict__ tf_powers_raw, typename BfTraits::QuantisedPowerType* __restrict__ tf_powers, - float4 const* __restrict__ output_scale, - float4 const* __restrict__ output_offset, + float const* __restrict__ output_scale, + float const* __restrict__ output_offset, float const* __restrict__ antenna_weights, int nsamples, int nbeamsets) @@ -79,9 +79,9 @@ __global__ void icbf_ftpa_general_k( const typename BfTraits::RawPowerType power_f32 = acc_buffer[0]; const int output_idx = beamset_idx * gridDim.x * gridDim.y + output_t_idx * gridDim.y + output_f_idx; - const float4 scale = + const float scale = output_scale[beamset_idx * gridDim.y + output_f_idx]; - const float4 offset = + const float offset = output_offset[beamset_idx * gridDim.y + output_f_idx]; tf_powers_raw[output_idx] = power_f32; tf_powers[output_idx] = @@ -112,7 +112,7 @@ void IncoherentBeamformer::beamform( PowerVectorTypeD& output, ScalingVectorTypeD const& output_scale, ScalingVectorTypeD const& output_offset, - BeamsetWeightsVectorTypeD const& antenna_weights, + ScalingVectorTypeD const& antenna_weights, int nbeamsets, cudaStream_t stream) { @@ -156,9 +156,9 @@ void IncoherentBeamformer::beamform( dim3 grid(nsamples / _config.ib_tscrunch(), _config.nchans() / _config.ib_fscrunch()); char2 const* ftpa_voltages_ptr = thrust::raw_pointer_cast(input.data()); - float4 const* output_scale_ptr = + float const* output_scale_ptr = thrust::raw_pointer_cast(output_scale.data()); - float4 const* output_offset_ptr = + float const* output_offset_ptr = thrust::raw_pointer_cast(output_offset.data()); float const* antenna_weights_ptr = thrust::raw_pointer_cast(antenna_weights.data()); diff --git a/cpp/skyweaver/src/StatisticsCalculator.cu b/cpp/skyweaver/src/StatisticsCalculator.cu index a2cf68c..4de451a 100644 --- a/cpp/skyweaver/src/StatisticsCalculator.cu +++ b/cpp/skyweaver/src/StatisticsCalculator.cu @@ -130,7 +130,7 @@ void StatisticsCalculator::calculate_statistics( } void StatisticsCalculator::update_scalings( - BeamsetWeightsVectorTypeH const& beamset_weights, + ScalingVectorTypeH const& beamset_weights, int nbeamsets) { // At this stage we have the standard deviations of each channel @@ -139,107 +139,99 @@ void StatisticsCalculator::update_scalings( // disk. const float weights_amp = 127.0f; - if ((_config.ib_fscrunch() != _config.cb_fscrunch()) || - (_config.ib_tscrunch() != _config.cb_tscrunch())) { - throw std::invalid_argument("IB and CB must share same F and T scrunch"); - } - const std::size_t output_nchans = _config.nchans() / _config.cb_fscrunch(); - const std::size_t fscrunch = _config.cb_fscrunch(); - const std::size_t tscrunch = _config.cb_tscrunch(); + std::size_t reduced_nchans_ib = _config.nchans() / _config.ib_fscrunch(); + std::size_t reduced_nchans_cb = _config.nchans() / _config.cb_fscrunch(); // Offsets for the coherent beams - _cb_offsets_d.resize(output_nchans * nbeamsets); - _cb_offsets_h.resize(output_nchans * nbeamsets); + _cb_offsets_d.resize(reduced_nchans_cb * nbeamsets); + _cb_offsets_h.resize(reduced_nchans_cb * nbeamsets); // Scalings for the coherent beams - _cb_scaling_d.resize(output_nchans * nbeamsets); - _cb_scaling_h.resize(output_nchans * nbeamsets); + _cb_scaling_d.resize(reduced_nchans_cb * nbeamsets); + _cb_scaling_h.resize(reduced_nchans_cb * nbeamsets); // Offsets for the incoherent beam - _ib_offsets_d.resize(output_nchans * nbeamsets); - _ib_offsets_h.resize(output_nchans * nbeamsets); + _ib_offsets_d.resize(reduced_nchans_ib * nbeamsets); + _ib_offsets_h.resize(reduced_nchans_ib * nbeamsets); // Scalings for the incoherent beam - _ib_scaling_d.resize(output_nchans * nbeamsets); - _ib_scaling_h.resize(output_nchans * nbeamsets); + _ib_scaling_d.resize(reduced_nchans_ib * nbeamsets); + _ib_scaling_h.resize(reduced_nchans_ib * nbeamsets); const std::uint32_t pa = _config.npol() * _config.nantennas(); const std::uint32_t a = _config.nantennas(); for(std::uint32_t beamset_idx = 0; beamset_idx < nbeamsets; ++beamset_idx) { - - // Here we compute the offsets and scaling factors for I, Q, U and V for each beamset - // Statistics are in FPA order - - for(int fo_idx = 0; fo_idx < output_nchans; ++fo_idx) { - - const int output_idx = beamset_idx * nbeamsets + fo_idx; - - // reset main accumulators - float meanI = 0.0f; - float meanQ = 0.0f; - float varIQc = 0.0f; - float varUVc = 0.0f; - float varIQi = 0.0f; - float varUVi = 0.0f; - - for(int fs_idx = 0; fs_idx < fscrunch; ++fs_idx) { - const int f_idx = fo_idx * fscrunch + fs_idx; - - // Per antenna/pol accumulators - float var_p0_sum = 0.0f; // sum(sigma_p0^2) - float var_p1_sum = 0.0f; // sum(sigma_p1^2) - float quad_sum = 0.0f; // sum(sigma_p0^4 + sigma_p1^4) - float sum_mul = 0.0f; // sum(sigma_p0^2 * sigma_p1^2) - for (int a_idx = 0; a_idx < _cofig.npol(); ++a_idx) { - // stats are in FPA order - // weights are in FPBA order - const int input_idx = f_idx * pa + a_idx; - Statistics p0 = _stats_h[input_idx]; - Statistics p1 = _stats_h[input_idx + nantennas]; - const float weight = beamset_weights[beamset_idx * a + a_idx]; - const float std_p0 = p0.std * weight; - const float std_p1 = p1.std * weight; - const float var_p0 = std_p0 * std_p0; - const float var_p1 = std_p1 * std_p1; - meanI += var_p0 + var_p1; - meanQ += var_p0 - var_p1; - var_p0_sum += var_p0; - var_p1_sum += var_p1; - quad_sum += (var_p0 * var_p0) + (var_p1 * var_p1); - sum_mul += var_p0 * var_p1; + // For each frequency channel we average the std estimates then + // calculate the offsets and scalings. + for(int f_idx = 0; f_idx < _config.nchans(); ++f_idx) { + float sum = 0.0f; + float count = 0.0f; + for(int p_idx = 0; p_idx < _config.npol(); ++p_idx) { + for(int a_idx = 0; a_idx < _config.nantennas(); ++a_idx) { + const float weight = + beamset_weights[_config.nantennas() * beamset_idx + + a_idx]; + sum += + weight * _stats_h[f_idx * pa + p_idx * a + a_idx].std; + count += weight; } - varIQc += (var_p0_sum * var_p0_sum) + (var_p1_sum * var_p1_sum); - varIQi += quad_sum; - varUVc += var_p0_sum * var_p1_sum; - varUVi += sum_mul; + } + const float avg_std = sum / count; + BOOST_LOG_TRIVIAL(debug) << "Channel " << f_idx; + BOOST_LOG_TRIVIAL(debug) + << "Averaged standard deviation = " << avg_std; + float const effective_nantennas = count / _config.npol(); + + // CB OFFSET + { + float scale = std::pow(weights_amp * avg_std * + std::sqrt(effective_nantennas), + 2); + float dof = 2 * _config.cb_tscrunch() * _config.npol(); + _cb_offsets_h[f_idx] = scale * dof; + BOOST_LOG_TRIVIAL(debug) + << "CB offset = " << _cb_offsets_h[f_idx]; + } + + // CB SCALE + { + float scale = std::pow(weights_amp * avg_std * + std::sqrt(effective_nantennas), + 2); + float dof = 2 * _config.cb_tscrunch() * _config.npol(); + _cb_scaling_h[f_idx] = + scale * std::sqrt(2 * dof) / _config.output_level(); + BOOST_LOG_TRIVIAL(debug) + << "CB scaling = " << _cb_scaling_h[f_idx]; } - // Coherent offsets - _cb_offsets_h[output_idx].x = 2 * tscrunch * meanI; // I - _cb_offsets_h[output_idx].y = 2 * tscrunch * meanQ; // Q - _cb_offsets_h[output_idx].z = 0.0f; // U - _cb_offsets_h[output_idx].w = 0.0f; // V - - // Coherent scales - _cb_scaling_h[output_idx].x = 4 * tscrunch * varIQc / _config.output_level(); // I - _cb_scaling_h[output_idx].y = 4 * tscrunch * varIQc / _config.output_level(); // Q - _cb_scaling_h[output_idx].z = 8 * tscrunch * varUVc / _config.output_level(); // U - _cb_scaling_h[output_idx].w = 8 * tscrunch * varUVc / _config.output_level(); // V - - // Incoherent offsets - _ib_offsets_h[output_idx].x = 2 * tscrunch * meanI; // I - _ib_offsets_h[output_idx].y = 2 * tscrunch * meanQ; // Q - _ib_offsets_h[output_idx].z = 0.0f; // U - _ib_offsets_h[output_idx].w = 0.0f; // V - - // Incoherent scales - _ib_scaling_h[output_idx].x = 4 * tscrunch * varIQi / _config.output_level(); // I - _ib_scaling_h[output_idx].y = 4 * tscrunch * varIQi / _config.output_level(); // Q - _ib_scaling_h[output_idx].z = 8 * tscrunch * varUVi / _config.output_level(); // U - _ib_scaling_h[output_idx].w = 8 * tscrunch * varUVi / _config.output_level(); // V + // IB OFFSET + { + float scale = std::pow(avg_std, 2); + float dof = 2 * _config.ib_tscrunch() * effective_nantennas * + _config.npol(); + _ib_offsets_h[f_idx] = scale * dof; + BOOST_LOG_TRIVIAL(debug) + << "IB offset = " << _ib_offsets_h[f_idx]; + } + + // IB SCALE + { + float scale = std::pow(avg_std, 2); + float dof = 2 * _config.ib_tscrunch() * effective_nantennas * + _config.npol(); + _ib_scaling_h[f_idx] = + scale * std::sqrt(2 * dof) / _config.output_level(); + BOOST_LOG_TRIVIAL(debug) + << "IB scaling = " << _ib_scaling_h[f_idx]; + } } } + // At this stage, all scaling vectors are available on the host and + // could be written to disk. + + // Copying these back to the device _cb_offsets_d = _cb_offsets_h; _cb_scaling_d = _cb_scaling_h; _ib_offsets_d = _ib_offsets_h; @@ -264,7 +256,7 @@ void StatisticsCalculator::dump_scalings( std::string const& timestamp, std::string const& tag, std::string const& path, - ScalingVectorTypeH const& ar) const + thrust::host_vector const& ar) const { std::ofstream writer; std::string filename = path + "/" + timestamp + "_" + tag + "_" + @@ -279,7 +271,7 @@ void StatisticsCalculator::dump_scalings( BOOST_LOG_TRIVIAL(error) << error_message.str(); return; } - writer.write(static_cast(ar.data()), ar.size() * sizeof(ScalingType)); + writer.write((char*)ar.data(), ar.size() * sizeof(float)); writer.close(); } @@ -313,4 +305,4 @@ StatisticsCalculator::ib_scaling() const return _ib_scaling_d; } -} // namespace skyweaver \ No newline at end of file +} // namespace skyweaver From 13047bb4b22a998b2227562d3c4e38e3688b8be1 Mon Sep 17 00:00:00 2001 From: Vivek Venkatraman Krishnan Date: Thu, 2 Jan 2025 13:18:54 +0100 Subject: [PATCH 44/69] segregate filterbanks in beam directories --- cpp/skyweaver/MultiFileWriter.cuh | 6 ++- cpp/skyweaver/detail/MultiFileWriter.cu | 5 +++ cpp/skyweaver/detail/SkyCleaver.cpp | 1 + cpp/skyweaver/skycleaver_utils.hpp | 53 +++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 cpp/skyweaver/skycleaver_utils.hpp diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index d107713..3d5fb45 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -19,6 +19,7 @@ struct MultiFileWriterConfig { std::string stokes_mode; std::string output_dir; std::string base_output_dir; + std::string inner_dir; std::string prefix; std::string extension; std::string output_basename; @@ -26,7 +27,7 @@ struct MultiFileWriterConfig { MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), - output_dir("default/"), prefix(""), extension(""), output_basename(""), suffix("") {}; + output_dir("default/"), base_output_dir("default_base/"), inner_dir(""), prefix(""), extension(""), output_basename(""), suffix("") {}; MultiFileWriterConfig(std::size_t header_size, std::size_t max_file_size, @@ -47,7 +48,8 @@ struct MultiFileWriterConfig { ", stokes_mode: " + stokes_mode + ", output_dir: " + output_dir + ", prefix: " + prefix + ", extension: " + extension + ", output_basename: " + output_basename + - ", base_output_dir: " + base_output_dir; + ", base_output_dir: " + base_output_dir + + ", inner_dir: " + inner_dir + ", suffix: " + suffix; } }; /** diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 439f1ab..5c2dbd8 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -47,6 +47,7 @@ MultiFileWriter::MultiFileWriter( _config.max_file_size = config.max_output_filesize(); _config.stokes_mode = config.stokes_mode(); _config.base_output_dir = config.output_dir(); + _config.inner_dir = ""; } @@ -106,6 +107,10 @@ MultiFileWriter::get_output_dir(VectorType const& stream_data, output_dir << _config.base_output_dir << "/" << get_formatted_time(_header.utc_start) << "/" << stream_idx; + if(!_config.inner_dir.empty()) { + output_dir << "/" << _config.inner_dir; + } + return output_dir.str(); } diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index b84cff7..8505c94 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -474,6 +474,7 @@ void skyweaver::SkyCleaver::init_writers() writer_config.max_file_size = _config.max_output_filesize(); writer_config.stokes_mode = _config.out_stokes().at(istokes); writer_config.base_output_dir = output_dir; + writer_config.inner_dir = beam_info.beam_name; writer_config.prefix = _config.out_prefix(); std::string suffix = "idm_" + to_string_with_padding(_header.dms[idm], 9, 3); diff --git a/cpp/skyweaver/skycleaver_utils.hpp b/cpp/skyweaver/skycleaver_utils.hpp new file mode 100644 index 0000000..d0ad9e9 --- /dev/null +++ b/cpp/skyweaver/skycleaver_utils.hpp @@ -0,0 +1,53 @@ +#ifndef SKYWEAVER_SKYCLEAVER_UTILS_HPP +#define SKYWEAVER_SKYCLEAVER_UTILS_HPP +namespace skyweaver +{ + +template +std::vector +get_list_from_string(const std::string& value, + T epsilon = std::numeric_limits::epsilon()) +{ + std::vector output; + std::vector comma_chunks; + + // Split the input string by commas + std::stringstream ss(value); + std::string token; + while(std::getline(ss, token, ',')) { comma_chunks.push_back(token); } + + for(const auto& comma_chunk: comma_chunks) { + // Check if the chunk contains a colon (indicating a range) + if(comma_chunk.find(':') == std::string::npos) { + output.push_back(static_cast(std::atof(comma_chunk.c_str()))); + continue; + } + + // Split the range chunk by colons + std::stringstream ss_chunk(comma_chunk); + std::vector colon_chunks; + std::string colon_token; + while(std::getline(ss_chunk, colon_token, ':')) { + colon_chunks.push_back( + static_cast(std::atof(colon_token.c_str()))); + } + + // Determine the step size + T step = colon_chunks.size() == 3 ? colon_chunks[2] : static_cast(1); + T start = colon_chunks[0]; + T stop = colon_chunks[1]; + + // Loop and add values to the output vector + if constexpr(std::is_floating_point::value) { + for(T k = start; k <= stop + epsilon; k += step) { + output.push_back(k); + } + } else { + for(T k = start; k <= stop; k += step) { output.push_back(k); } + } + } + return output; +} + +} +#endif // SKYWEAVER_SKYCLEAVER_UTILS_HPP \ No newline at end of file From 4f15b64e74bc7867a658789a2e1682b66dd1febd Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Wed, 29 Jan 2025 15:29:49 +0000 Subject: [PATCH 45/69] Fix for incorrect S-band frequencies being reported in filenames due to int overflow --- cpp/skyweaver/detail/file_writer_callbacks.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/skyweaver/detail/file_writer_callbacks.cpp b/cpp/skyweaver/detail/file_writer_callbacks.cpp index e1c59dc..ffb2c42 100644 --- a/cpp/skyweaver/detail/file_writer_callbacks.cpp +++ b/cpp/skyweaver/detail/file_writer_callbacks.cpp @@ -87,12 +87,12 @@ create_dada_file_stream(MultiFileWriterConfig const& config, std::stringstream output_dir; output_dir << config.output_dir << "/" << std::fixed << std::setfill('0') - << std::setw(9) << static_cast(header.frequency); + << std::setw(9) << static_cast(header.frequency); std::stringstream output_basename; output_basename << config.output_basename << "_" << std::fixed << std::setfill('0') << std::setw(9) - << static_cast(header.frequency); + << static_cast(header.frequency); std::unique_ptr file_stream = std::make_unique( From 03b92c5e814e997d9402ecc8a3e77471b783298f Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 13 Feb 2025 11:15:24 +0000 Subject: [PATCH 46/69] Fix for integer overflow in SkyCleaver while computing expected bridge frequencies --- cpp/skyweaver/detail/SkyCleaver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/SkyCleaver.cpp b/cpp/skyweaver/detail/SkyCleaver.cpp index 8505c94..9f86c58 100644 --- a/cpp/skyweaver/detail/SkyCleaver.cpp +++ b/cpp/skyweaver/detail/SkyCleaver.cpp @@ -303,7 +303,7 @@ void skyweaver::SkyCleaver::init_readers() long double start_freq = obs_centre_freq - obs_bandwidth / 2; for(int i = 0; i < _config.nbridges(); i++) { - int ifreq = std::lround(std::floor( + std::size_t ifreq = std::lround(std::floor( start_freq + (i + 0.5) * obs_bandwidth / _config.nbridges())); _expected_freqs.push_back(ifreq); BOOST_LOG_TRIVIAL(info) From 4242cf90f49c72239e77a4bff146c75986987e24 Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 20 Feb 2025 09:51:40 +0000 Subject: [PATCH 47/69] Added command-line option for computing/outputing voltage statistics --- cpp/skyweaver/PipelineConfig.hpp | 12 +++++++ cpp/skyweaver/detail/BeamformerPipeline.cu | 39 +++++++++++++--------- cpp/skyweaver/src/PipelineConfig.cpp | 12 ++++++- cpp/skyweaver/src/skyweaver_cli.cu | 11 ++++-- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 1656b0c..807d1bb 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -165,6 +165,17 @@ class PipelineConfig */ bool enable_incoherent_dedispersion() const; + + /** + * @brief Enable/disable calculation and writing of voltage statistics + */ + void output_statistics(bool enable); + + /** + * @brief Check if calculation of voltage statistics is enabled + */ + bool output_statistics() const; + /** * @brief Return the number of time samples to be integrated * in the coherent beamformer. @@ -339,6 +350,7 @@ class PipelineConfig std::size_t _max_output_filesize; std::string _output_file_prefix; bool _enable_incoherent_dedispersion; + bool _output_statistics; double _cfreq; double _bw; mutable bool _channel_frequencies_stale; diff --git a/cpp/skyweaver/detail/BeamformerPipeline.cu b/cpp/skyweaver/detail/BeamformerPipeline.cu index 7c44422..82f7b41 100644 --- a/cpp/skyweaver/detail/BeamformerPipeline.cu +++ b/cpp/skyweaver/detail/BeamformerPipeline.cu @@ -115,7 +115,10 @@ void BeamformerPipeline:: _utc_offset = utc_offset; _cb_handler.init(_header); _ib_handler.init(_header); - _stats_handler.init(_header); + if (_config.output_statistics()) + { + _stats_handler.init(_header); + } NVTX_RANGE_POP(); } @@ -161,20 +164,23 @@ void BeamformerPipeline:: _timer.stop("transpose TAFTP to FTPA"); NVTX_RANGE_POP(); - NVTX_RANGE_PUSH("Calculate statistics"); - BOOST_LOG_TRIVIAL(debug) << "Checking if channel statistics update request"; - _timer.start("calculate statistics"); - _stats_manager->calculate_statistics(_ftpa_post_transpose); - _timer.stop("calculate statistics"); - NVTX_RANGE_POP(); - NVTX_RANGE_PUSH("Update scalings"); - if(_call_count == 0) { + if ((_call_count == 0) || _config.output_statistics()) + { + NVTX_RANGE_PUSH("Calculate statistics"); + BOOST_LOG_TRIVIAL(debug) << "Checking if channel statistics update request"; + _timer.start("calculate statistics"); + _stats_manager->calculate_statistics(_ftpa_post_transpose); + _timer.stop("calculate statistics"); + NVTX_RANGE_POP(); + if(_call_count == 0) { + NVTX_RANGE_PUSH("Update scalings"); _timer.start("update scalings"); _stats_manager->update_scalings(_delay_manager->beamset_weights(), _delay_manager->nbeamsets()); _timer.stop("update scalings"); + NVTX_RANGE_POP(); + } } - NVTX_RANGE_POP(); // BOOST_LOG_TRIVIAL(debug) << "Peeking the statistics"; // peek(_stats_manager->statistics(), 64); @@ -251,11 +257,14 @@ void BeamformerPipeline:: NVTX_RANGE_POP(); } NVTX_RANGE_POP(); - NVTX_RANGE_PUSH("Stats handler"); - _timer.start("statistics handler"); - _stats_handler(_stats_manager->statistics()); - _timer.stop("statistics handler"); - NVTX_RANGE_POP(); + if (_config.output_statistics()) + { + NVTX_RANGE_PUSH("Stats handler"); + _timer.start("statistics handler"); + _stats_handler(_stats_manager->statistics()); + _timer.stop("statistics handler"); + NVTX_RANGE_POP(); + } NVTX_RANGE_POP(); } diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index c2cdf3b..ca99c8d 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -15,7 +15,7 @@ PipelineConfig::PipelineConfig() _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f) + _stokes_mode("I"), _output_level(24.0f), _output_statistics(true) { } @@ -169,6 +169,16 @@ bool PipelineConfig::enable_incoherent_dedispersion() const return _enable_incoherent_dedispersion; } +void PipelineConfig::output_statistics(bool enable) +{ + _output_statistics = enable; +} + +bool PipelineConfig::output_statistics() const +{ + return _output_statistics; +} + std::vector const& PipelineConfig::channel_frequencies() const { if(_channel_frequencies_stale) { diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 8204ca3..342dcaa 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -476,7 +476,14 @@ int main(int argc, char** argv) ("log-level", po::value()->default_value("info")->notifier( [](std::string level) { skyweaver::set_log_level(level); }), - "The logging level to use (debug, info, warning, error)"); + "The logging level to use (debug, info, warning, error)") + + ("statistics", + po::value()->default_value(true)->notifier( + [&config](bool const& enable) { + config.output_statistics(enable); }), + "Turn on/off calculation and output of voltage statistics"); + // set options allowed on command line po::options_description cmdline_options; @@ -617,4 +624,4 @@ int main(int argc, char** argv) return ERROR_UNHANDLED_EXCEPTION; } return SUCCESS; -} \ No newline at end of file +} From 6675ae6e080109c92105d88ae34649dfa12c458b Mon Sep 17 00:00:00 2001 From: Colin Clark Date: Thu, 20 Feb 2025 10:11:47 +0000 Subject: [PATCH 48/69] Added command-line option to disable writing the incoherent beam --- cpp/skyweaver/PipelineConfig.hpp | 12 +++++++++++- cpp/skyweaver/detail/BeamformerPipeline.cu | 18 ++++++++++++------ cpp/skyweaver/src/PipelineConfig.cpp | 12 +++++++++++- cpp/skyweaver/src/skyweaver_cli.cu | 8 +++++++- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 807d1bb..65fa664 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -165,7 +165,6 @@ class PipelineConfig */ bool enable_incoherent_dedispersion() const; - /** * @brief Enable/disable calculation and writing of voltage statistics */ @@ -176,6 +175,16 @@ class PipelineConfig */ bool output_statistics() const; + /** + * @brief Enable/disable writing out the incoherent beam + */ + void output_incoherent_beam(bool enable); + + /** + * @brief Check if outputting incoherent beam is enabled + */ + bool output_incoherent_beam() const; + /** * @brief Return the number of time samples to be integrated * in the coherent beamformer. @@ -351,6 +360,7 @@ class PipelineConfig std::string _output_file_prefix; bool _enable_incoherent_dedispersion; bool _output_statistics; + bool _output_incoherent_beam; double _cfreq; double _bw; mutable bool _channel_frequencies_stale; diff --git a/cpp/skyweaver/detail/BeamformerPipeline.cu b/cpp/skyweaver/detail/BeamformerPipeline.cu index 82f7b41..18617b3 100644 --- a/cpp/skyweaver/detail/BeamformerPipeline.cu +++ b/cpp/skyweaver/detail/BeamformerPipeline.cu @@ -114,7 +114,10 @@ void BeamformerPipeline:: _header = header; _utc_offset = utc_offset; _cb_handler.init(_header); - _ib_handler.init(_header); + if (_config.output_incoherent_beam()) + { + _ib_handler.init(_header); + } if (_config.output_statistics()) { _stats_handler.init(_header); @@ -250,11 +253,14 @@ void BeamformerPipeline:: _timer.stop("coherent beam handler"); NVTX_RANGE_POP(); - NVTX_RANGE_PUSH("Incoherent beamformer handler"); - _timer.start("incoherent beam handler"); - _ib_handler(_tf_ib, dm_idx); - _timer.stop("incoherent beam handler"); - NVTX_RANGE_POP(); + if (_config.output_incoherent_beam()) + { + NVTX_RANGE_PUSH("Incoherent beamformer handler"); + _timer.start("incoherent beam handler"); + _ib_handler(_tf_ib, dm_idx); + _timer.stop("incoherent beam handler"); + NVTX_RANGE_POP(); + } } NVTX_RANGE_POP(); if (_config.output_statistics()) diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index ca99c8d..15605c2 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -15,7 +15,7 @@ PipelineConfig::PipelineConfig() _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f), _output_statistics(true) + _stokes_mode("I"), _output_level(24.0f), _output_statistics(true), _output_incoherent_beam(true) { } @@ -179,6 +179,16 @@ bool PipelineConfig::output_statistics() const return _output_statistics; } +void PipelineConfig::output_incoherent_beam(bool enable) +{ + _output_incoherent_beam = enable; +} + +bool PipelineConfig::output_incoherent_beam() const +{ + return _output_incoherent_beam; +} + std::vector const& PipelineConfig::channel_frequencies() const { if(_channel_frequencies_stale) { diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 342dcaa..7c656b9 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -482,8 +482,14 @@ int main(int argc, char** argv) po::value()->default_value(true)->notifier( [&config](bool const& enable) { config.output_statistics(enable); }), - "Turn on/off calculation and output of voltage statistics"); + "Turn on/off calculation and output of voltage statistics") + ("write-incoherent-beam", + po::value()->default_value(true)->notifier( + [&config](bool const& enable) { + config.output_incoherent_beam(enable); }), + "Turn on/off output of incoherent beam" + "Turning off does not disable incoherent beam subtraction"); // set options allowed on command line po::options_description cmdline_options; From 68e8a2c6a1774c42a992363042db7aa4eef36393 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:06:27 +0000 Subject: [PATCH 49/69] define the pre_write struct, containing possible configuration important for pre write actions --- cpp/skyweaver/PipelineConfig.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 65fa664..9dcb8f8 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -9,6 +9,19 @@ namespace skyweaver { + struct WaitConfig + { + bool is_enabled; + int iterations; + int sleep_time; + std::size_t min_free_space; + }; + + struct PreWriteConfig + { + WaitConfig wait; + }; + /** * @brief Class for wrapping the skyweaver pipeline configuration. From 1c7133145400aeddc70310f5810eb5e11e57229a Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:07:11 +0000 Subject: [PATCH 50/69] create pre write config infrastructure --- cpp/skyweaver/PipelineConfig.hpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 9dcb8f8..b7628ef 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -166,6 +166,11 @@ class PipelineConfig */ DedispersionPlan const& ddplan() const; + /** + * @brief configures wait for filesystem space + */ + void configure_wait(std::string argument); + /** * @brief Enable/disable incoherent dedispersion based fscrunch after * beamforming @@ -242,6 +247,11 @@ class PipelineConfig return SKYWEAVER_CB_NSAMPLES_PER_BLOCK; } + PreWriteConfig pre_write_config() const + { + return _pre_write_config; + } + /** * @brief Return the total number of samples to read from file in each gulp. * @@ -385,6 +395,7 @@ class PipelineConfig float _output_level; DedispersionPlan _ddplan; mutable std::vector _channel_frequencies; + PreWriteConfig _pre_write_config; }; } // namespace skyweaver From 7ef69a51f5546b183bf7a08c197263b4f39f5e80 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:07:42 +0000 Subject: [PATCH 51/69] needed to compute bytes from units like G, M or k --- cpp/skyweaver/PipelineConfig.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index b7628ef..c323a0f 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -366,6 +366,7 @@ class PipelineConfig } private: + std::size_t convertMemorySize(const std::string& str) const; void calculate_channel_frequencies() const; void update_power_offsets_and_scalings(); From 9c18ea258db922e178d02062e37a9b3580605d20 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Thu, 8 May 2025 08:59:28 +0200 Subject: [PATCH 52/69] resolve merge conflict --- cpp/skyweaver/MultiFileWriter.cuh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 3d5fb45..3cb53bb 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -24,7 +24,8 @@ struct MultiFileWriterConfig { std::string extension; std::string output_basename; std::string suffix; - + PreWriteConfig pre_write; + MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("default/"), base_output_dir("default_base/"), inner_dir(""), prefix(""), extension(""), output_basename(""), suffix("") {}; @@ -51,6 +52,7 @@ struct MultiFileWriterConfig { ", base_output_dir: " + base_output_dir + ", inner_dir: " + inner_dir + ", suffix: " + suffix; } + } }; /** * @brief A class for handling writing of DescribedVectors @@ -141,4 +143,4 @@ class MultiFileWriter #include "skyweaver/detail/MultiFileWriter.cu" #include "skyweaver/detail/file_writer_callbacks.cpp" -#endif // SKYWEAVER_MULTIFILEWRITER_CUH \ No newline at end of file +#endif // SKYWEAVER_MULTIFILEWRITER_CUH From dd1acc8e8d8a58f8cc995545070e81b91f248e76 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Thu, 8 May 2025 09:01:22 +0200 Subject: [PATCH 53/69] resolve merge conflict --- cpp/skyweaver/MultiFileWriter.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 3cb53bb..12e6f58 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -68,8 +68,8 @@ class MultiFileWriter ObservationHeader const&, VectorType const&, std::size_t)>; - public: + using PreWriteCallback = std::function; /** * @brief Construct a new Multi File Writer object * From bd188e51d3a90ee1d109d0db45198af4ad7bdff6 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Thu, 8 May 2025 09:04:43 +0200 Subject: [PATCH 54/69] resolve merge conflict --- cpp/skyweaver/MultiFileWriter.cuh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 12e6f58..ba30a76 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -84,6 +84,14 @@ class MultiFileWriter MultiFileWriter(MultiFileWriterConfig config, std::string tag, CreateStreamCallBackType create_stream_callback); + MultiFileWriter(PipelineConfig const& config, + std::string tag, + CreateStreamCallBackType create_stream_callback, + PreWriteCallback pre_write_callback); + MultiFileWriter(MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback, + PreWriteCallback pre_write_callback); MultiFileWriter(MultiFileWriter const&) = delete; /** From da09af62b03d2fbb844132a6162505ee8ee38dfa Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:09:51 +0000 Subject: [PATCH 55/69] define a private call back function --- cpp/skyweaver/MultiFileWriter.cuh | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index ba30a76..d889469 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -139,6 +139,7 @@ class MultiFileWriter std::string get_extension(VectorType const& stream_data); CreateStreamCallBackType _create_stream_callback; MultiFileWriterConfig _config; + PreWriteCallback _pre_write_callback; std::string _tag; ObservationHeader _header; std::map> _file_streams; From 5f5fd61cfa93c4408750394ba0563a5de3d1aba4 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Thu, 8 May 2025 09:19:20 +0200 Subject: [PATCH 56/69] resolve merge conflict --- cpp/skyweaver/detail/MultiFileWriter.cu | 32 ++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 5c2dbd8..247cd9e 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -43,12 +43,29 @@ MultiFileWriter::MultiFileWriter( CreateStreamCallBackType create_stream_callback) : _tag(tag), _create_stream_callback(create_stream_callback) { +<<<<<<< HEAD _config.header_size = config.dada_header_size(); _config.max_file_size = config.max_output_filesize(); _config.stokes_mode = config.stokes_mode(); _config.base_output_dir = config.output_dir(); _config.inner_dir = ""; + _pre_write_callback = nullptr; +} +template +MultiFileWriter::MultiFileWriter( + PipelineConfig const& config, + std::string tag, + CreateStreamCallBackType create_stream_callback, + PreWriteCallback pre_write_callback) + : _tag(tag), _create_stream_callback(create_stream_callback), _pre_write_callback(pre_write_callback) +{ + _config.header_size = config.dada_header_size(); + _config.max_file_size = config.max_output_filesize(); + _config.stokes_mode = config.stokes_mode(); + _config.base_output_dir = config.output_dir(); + _config.inner_dir = ""; + _config.pre_write = writer_config.pre_write; } template @@ -58,6 +75,19 @@ MultiFileWriter::MultiFileWriter( CreateStreamCallBackType create_stream_callback) : _config(config), _tag(tag), _create_stream_callback(create_stream_callback) +{ + _pre_write_callback = nullptr; +} + +template +MultiFileWriter::MultiFileWriter( + MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback, + PreWriteCallback pre_write_callback) + : _config(config), _tag(tag), + _create_stream_callback(create_stream_callback), + _pre_write_callback(pre_write_callback) { } @@ -185,4 +215,4 @@ bool MultiFileWriter::write(VectorType const& stream_data, return this->operator()(stream_data, stream_idx); } -} // namespace skyweaver \ No newline at end of file +} // namespace skyweaver From 92c104043c193a97c65aeed728441e1048372ba8 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:11:53 +0000 Subject: [PATCH 57/69] implement the logic for the pre write callback --- cpp/skyweaver/detail/MultiFileWriter.cu | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 247cd9e..630d1fd 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -188,6 +188,11 @@ template bool MultiFileWriter::operator()(VectorType const& stream_data, std::size_t stream_idx) { + std::size_t const data_size = stream_data.size() * sizeof(typename VectorType::value_type); + if (_pre_write_callback != nullptr && _config.pre_write.wait.is_enabled) + { + _pre_write_callback(data_size, _config); + } if(!has_stream(stream_idx)) { create_stream(stream_data, stream_idx); } @@ -202,8 +207,7 @@ bool MultiFileWriter::operator()(VectorType const& stream_data, _file_streams.at(stream_idx) ->write(reinterpret_cast( thrust::raw_pointer_cast(stream_data.data())), - stream_data.size() * - sizeof(typename VectorType::value_type)); + data_size); } return false; } From ddbec12d10a4e3a2dc0e4f67cba165813076e97c Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:12:58 +0000 Subject: [PATCH 58/69] method to read the wait arguments --- cpp/skyweaver/src/PipelineConfig.cpp | 58 ++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index 15605c2..1e33a0c 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -157,6 +157,64 @@ DedispersionPlan const& PipelineConfig::ddplan() const return _ddplan; } +std::size_t PipelineConfig::convertMemorySize(const std::string& str) const { + std::size_t lastCharPos = str.find_last_not_of("0123456789"); + std::string numberPart = str.substr(0, lastCharPos); + std::string unitPart = str.substr(lastCharPos); + + std::size_t number = std::stoull(numberPart); + + if (unitPart.empty()) + return number; + else if (unitPart == "K" || unitPart == "k") + return number * 1024; + else if (unitPart == "M" || unitPart == "m") + return number * 1024 * 1024; + else if (unitPart == "G" || unitPart == "g") + return number * 1024 * 1024 * 1024; + else + throw std::runtime_error("Invalid memory unit!"); +} + +void PipelineConfig::configure_wait(std::string argument) +{ + std::vector tokens; + std::string token; + std::istringstream tokenStream(argument); + int indx = 0; + _pre_write_config.wait.is_enabled = true; + while (std::getline(tokenStream, token, ':')) { + if(indx == 0) + { + errno = 0; + _pre_write_config.wait.iterations = std::stoi(token); + if (errno == ERANGE) { + throw std::runtime_error("Wait iteration number out of range!"); + } + if (_pre_write_config.wait.iterations < 0) _pre_write_config.wait.iterations = 0; + } else if(indx == 1) { + errno = 0; + _pre_write_config.wait.sleep_time = std::stoi(token); + if (errno == ERANGE) { + throw std::runtime_error("Sleep time out of range!"); + } + if (_pre_write_config.wait.sleep_time < 1) _pre_write_config.wait.sleep_time = 1; + } else if(indx == 2) { + if (!token.empty() && std::all_of(token.begin(), token.end(), ::isdigit)) + { + _pre_write_config.wait.min_free_space = std::stoull(token); + } else { + try { + _pre_write_config.wait.min_free_space = convertMemorySize(token); + } catch (std::runtime_error& e) { + std::cout << "Memory conversion error: " << e.what() << std::endl; + throw; + } + } + } + indx++; + } +} void PipelineConfig::enable_incoherent_dedispersion(bool enable) From 163b7ab5bc52ab3bde45401e5001a4c5531c9fef Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Fri, 20 Sep 2024 13:14:09 +0000 Subject: [PATCH 59/69] the pre write call back function --- cpp/skyweaver/src/skyweaver_cli.cu | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 7c656b9..1d7f1eb 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -239,6 +240,34 @@ void run_pipeline(Pipeline& pipeline, stopwatch.show_all_timings(); } +auto pre_write_callback = [] (std::size_t const size, skyweaver::MultiFileWriterConfig const& config) +{ + if (!config.pre_write.wait.is_enabled) return; + std::filesystem::space_info space = std::filesystem::space(config.output_dir); + size_t limit = std::min(config.pre_write.wait.min_free_space, size); + if(space.available >= limit) + return; + + BOOST_LOG_TRIVIAL(info) + << space.available + << " bytes available space is not enough. Need at least " + << limit + << " bytes in " << config.output_dir << "."; + BOOST_LOG_TRIVIAL(warning) << "Start pausing."; + int max_iterations = (config.pre_write.wait.iterations == 0) ? INT_MAX : config.pre_write.wait.iterations; + for (int i = 0; i < max_iterations; i++) + { + sleep(config.pre_write.wait.sleep_time); + space = std::filesystem::space(config.output_dir); + if (space.available >= limit) + { + BOOST_LOG_TRIVIAL(info) << "Space has been freed up. Will proceed."; + return; + } + } + throw std::runtime_error("Space for writing hasn't been freed up in time."); +}; + template void setup_pipeline(skyweaver::PipelineConfig& config) { From bfc3d39b20bc3ff2f1d9b8ad5f21bb26f4936ebb Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Thu, 8 May 2025 09:21:56 +0200 Subject: [PATCH 60/69] resolve merge conflict --- cpp/skyweaver/src/skyweaver_cli.cu | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 1d7f1eb..834b4e7 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -289,7 +289,8 @@ void setup_pipeline(skyweaver::PipelineConfig& config) typename IBWriterType::CreateStreamCallBackType create_stream_callback_ib = skyweaver::detail::create_dada_file_stream< skyweaver::BTFPowersH>; - IBWriterType ib_handler(config, "ib", create_stream_callback_ib); + + IBWriterType ib_handler(config, "ib", create_stream_callback_ib, pre_write_callback); using StatsWriterType = skyweaver::MultiFileWriter>; @@ -297,9 +298,10 @@ void setup_pipeline(skyweaver::PipelineConfig& config) create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; - StatsWriterType stats_handler(config, - "stats", - create_stream_callback_stats); + StatsWriterType stats_handler(config, + "stats", + create_stream_callback_stats, + pre_write_callback); if constexpr(enable_incoherent_dedispersion) { using CBWriterType = @@ -309,7 +311,7 @@ void setup_pipeline(skyweaver::PipelineConfig& config) skyweaver::detail::create_dada_file_stream< skyweaver::TDBPowersH>; skyweaver::MultiFileWriter> - cb_file_writer(config, "cb", create_stream_callback_cb); + cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); skyweaver::IncoherentDedispersionPipeline @@ -330,7 +332,7 @@ void setup_pipeline(skyweaver::PipelineConfig& config) create_stream_callback_cb = skyweaver::detail::create_dada_file_stream< skyweaver::TFBPowersD>; - CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb); + CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); skyweaver::BeamformerPipeline Date: Fri, 20 Sep 2024 13:15:02 +0000 Subject: [PATCH 61/69] read possible wait arguments --- cpp/skyweaver/src/skyweaver_cli.cu | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 834b4e7..ea301bc 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -503,7 +503,14 @@ int main(int argc, char** argv) [](std::size_t nthreads) { omp_set_num_threads(nthreads); }), "The number of threads to use for incoherent dedispersion") - // Logging options + ("wait-for-space", + po::value() + ->notifier( + [&config](std::string key) { config.configure_wait(key); }), + "Wait for enough disk space for the output. " + "::") + + // Logging options ("log-level", po::value()->default_value("info")->notifier( [](std::string level) { skyweaver::set_log_level(level); }), From 1e811426e789ce29c8fb8073f4580b238ae366fa Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Thu, 8 May 2025 09:27:00 +0200 Subject: [PATCH 62/69] resolve merge conflict --- cpp/skyweaver/MultiFileWriter.cuh | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index d889469..186558e 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -41,7 +41,6 @@ struct MultiFileWriterConfig { stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), suffix(suffix), output_basename("") {}; - std::string to_string() { return "header_size: " + std::to_string(header_size) + From 1f8513961550249ff111c9e40d1b8210a58ca9e1 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Thu, 8 May 2025 09:28:31 +0200 Subject: [PATCH 63/69] resolve merge conlict --- cpp/skyweaver/src/PipelineConfig.cpp | 61 +++++++++++++++------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index 1e33a0c..2d2c5a5 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -10,12 +10,12 @@ namespace skyweaver PipelineConfig::PipelineConfig() : _input_files({}), _check_input_contiguity(false), _dada_header_size(4096), _delay_file(""), _output_dir("./"), _max_output_filesize(10000000000000), - _output_file_prefix(""), - _enable_incoherent_dedispersion(true), _cfreq(1284000000.0), - _bw(13375000.0), _channel_frequencies_stale(true), + _output_file_prefix(""), _enable_incoherent_dedispersion(true), + _cfreq(1284000000.0), _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f), _output_statistics(true), _output_incoherent_beam(true) + _stokes_mode("I"), _output_level(24.0f), _output_statistics(true), + _output_incoherent_beam(true), _pre_write_config({{false, 0, 0, 0}}) { } @@ -157,20 +157,21 @@ DedispersionPlan const& PipelineConfig::ddplan() const return _ddplan; } -std::size_t PipelineConfig::convertMemorySize(const std::string& str) const { +std::size_t PipelineConfig::convertMemorySize(const std::string& str) const +{ std::size_t lastCharPos = str.find_last_not_of("0123456789"); - std::string numberPart = str.substr(0, lastCharPos); - std::string unitPart = str.substr(lastCharPos); + std::string numberPart = str.substr(0, lastCharPos); + std::string unitPart = str.substr(lastCharPos); std::size_t number = std::stoull(numberPart); - if (unitPart.empty()) + if(unitPart.empty()) return number; - else if (unitPart == "K" || unitPart == "k") + else if(unitPart == "K" || unitPart == "k") return number * 1024; - else if (unitPart == "M" || unitPart == "m") + else if(unitPart == "M" || unitPart == "m") return number * 1024 * 1024; - else if (unitPart == "G" || unitPart == "g") + else if(unitPart == "G" || unitPart == "g") return number * 1024 * 1024 * 1024; else throw std::runtime_error("Invalid memory unit!"); @@ -181,33 +182,36 @@ void PipelineConfig::configure_wait(std::string argument) std::vector tokens; std::string token; std::istringstream tokenStream(argument); - int indx = 0; + int indx = 0; _pre_write_config.wait.is_enabled = true; - while (std::getline(tokenStream, token, ':')) { - if(indx == 0) - { - errno = 0; + while(std::getline(tokenStream, token, ':')) { + if(indx == 0) { + errno = 0; _pre_write_config.wait.iterations = std::stoi(token); - if (errno == ERANGE) { - throw std::runtime_error("Wait iteration number out of range!"); + if(errno == ERANGE) { + throw std::runtime_error("Wait iteration number out of range!"); } - if (_pre_write_config.wait.iterations < 0) _pre_write_config.wait.iterations = 0; + if(_pre_write_config.wait.iterations < 0) + _pre_write_config.wait.iterations = 0; } else if(indx == 1) { - errno = 0; + errno = 0; _pre_write_config.wait.sleep_time = std::stoi(token); - if (errno == ERANGE) { - throw std::runtime_error("Sleep time out of range!"); + if(errno == ERANGE) { + throw std::runtime_error("Sleep time out of range!"); } - if (_pre_write_config.wait.sleep_time < 1) _pre_write_config.wait.sleep_time = 1; + if(_pre_write_config.wait.sleep_time < 1) + _pre_write_config.wait.sleep_time = 1; } else if(indx == 2) { - if (!token.empty() && std::all_of(token.begin(), token.end(), ::isdigit)) - { + if(!token.empty() && + std::all_of(token.begin(), token.end(), ::isdigit)) { _pre_write_config.wait.min_free_space = std::stoull(token); } else { try { - _pre_write_config.wait.min_free_space = convertMemorySize(token); - } catch (std::runtime_error& e) { - std::cout << "Memory conversion error: " << e.what() << std::endl; + _pre_write_config.wait.min_free_space = + convertMemorySize(token); + } catch(std::runtime_error& e) { + std::cout << "Memory conversion error: " << e.what() + << std::endl; throw; } } @@ -216,7 +220,6 @@ void PipelineConfig::configure_wait(std::string argument) } } - void PipelineConfig::enable_incoherent_dedispersion(bool enable) { _enable_incoherent_dedispersion = enable; From 79f0e19fce643d29250dcc54bde455dfe4de4e4f Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Mon, 23 Sep 2024 06:54:56 +0000 Subject: [PATCH 64/69] PreWriting could be used more broadly. So is_enabled shouldn't belong to the wait config --- cpp/skyweaver/PipelineConfig.hpp | 4 ++-- cpp/skyweaver/detail/MultiFileWriter.cu | 2 +- cpp/skyweaver/src/skyweaver_cli.cu | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index c323a0f..0974023 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -11,7 +11,6 @@ namespace skyweaver { struct WaitConfig { - bool is_enabled; int iterations; int sleep_time; std::size_t min_free_space; @@ -19,7 +18,8 @@ namespace skyweaver struct PreWriteConfig { - WaitConfig wait; + bool is_enabled; + WaitConfig wait; }; diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 630d1fd..bc05299 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -189,7 +189,7 @@ bool MultiFileWriter::operator()(VectorType const& stream_data, std::size_t stream_idx) { std::size_t const data_size = stream_data.size() * sizeof(typename VectorType::value_type); - if (_pre_write_callback != nullptr && _config.pre_write.wait.is_enabled) + if (_pre_write_callback != nullptr && _config.pre_write.is_enabled) { _pre_write_callback(data_size, _config); } diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index ea301bc..97f47cd 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -242,7 +242,7 @@ void run_pipeline(Pipeline& pipeline, auto pre_write_callback = [] (std::size_t const size, skyweaver::MultiFileWriterConfig const& config) { - if (!config.pre_write.wait.is_enabled) return; + if (!config.pre_write.is_enabled) return; std::filesystem::space_info space = std::filesystem::space(config.output_dir); size_t limit = std::min(config.pre_write.wait.min_free_space, size); if(space.available >= limit) From cea2de2025186feef834dd229a410b6ad9c691f5 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Wed, 25 Sep 2024 07:46:53 +0000 Subject: [PATCH 65/69] introduce unique_ptr --- cpp/skyweaver/src/skyweaver_cli.cu | 51 ++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 97f47cd..f36b107 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -294,38 +294,62 @@ void setup_pipeline(skyweaver::PipelineConfig& config) using StatsWriterType = skyweaver::MultiFileWriter>; +<<<<<<< HEAD +======= + std::unique_ptr stats_handler; + +>>>>>>> c89902c (introduce unique_ptr) typename StatsWriterType::CreateStreamCallBackType create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; +<<<<<<< HEAD StatsWriterType stats_handler(config, "stats", create_stream_callback_stats, pre_write_callback); +======= +>>>>>>> c89902c (introduce unique_ptr) if constexpr(enable_incoherent_dedispersion) { using CBWriterType = skyweaver::MultiFileWriter>; typename CBWriterType::CreateStreamCallBackType +<<<<<<< HEAD create_stream_callback_cb = skyweaver::detail::create_dada_file_stream< skyweaver::TDBPowersH>; skyweaver::MultiFileWriter> cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); +======= + create_stream_callback_cb = + skyweaver::detail::create_dada_file_stream>; + std::unique_ptr cb_file_writer; + if (config.pre_write_config().is_enabled) + { + stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats, pre_write_callback)); + cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb, pre_write_callback)); + }else{ + stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats)); + cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb)); + } + +>>>>>>> c89902c (introduce unique_ptr) skyweaver::IncoherentDedispersionPipeline - incoherent_dispersion_pipeline(config, cb_file_writer); + decltype(* cb_file_writer.get())> + incoherent_dispersion_pipeline(config, * cb_file_writer.get()); skyweaver::BeamformerPipeline pipeline(config, incoherent_dispersion_pipeline, ib_handler, - stats_handler); + * stats_handler.get()); run_pipeline(pipeline, config, file_reader, header); } else { +<<<<<<< HEAD using CBWriterType = skyweaver::MultiFileWriter>; typename CBWriterType::CreateStreamCallBackType @@ -334,10 +358,25 @@ void setup_pipeline(skyweaver::PipelineConfig& config) skyweaver::TFBPowersD>; CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); skyweaver::BeamformerPipeline>; + std::unique_ptr cb_file_writer; + typename CBWriterType::CreateStreamCallBackType + create_stream_callback_cb = + skyweaver::detail::create_dada_file_stream>; + if (config.pre_write_config().is_enabled) + { + cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb, pre_write_callback)); + }else{ + cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb)); + } + + skyweaver::BeamformerPipeline>>>>>> c89902c (introduce unique_ptr) decltype(ib_handler), - decltype(stats_handler), + decltype(* stats_handler.get()), BfTraits> - pipeline(config, cb_file_writer, ib_handler, stats_handler); + pipeline(config, * cb_file_writer.get(), ib_handler, * stats_handler.get()); run_pipeline(pipeline, config, file_reader, header); } } From 2ab6778c8f34135e77ab6683d3e003af1096c58e Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Wed, 25 Sep 2024 08:38:03 +0000 Subject: [PATCH 66/69] also IBWriter is a unique_ptr now --- cpp/skyweaver/src/skyweaver_cli.cu | 33 +++++++++--------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index f36b107..67a9378 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -290,27 +290,19 @@ void setup_pipeline(skyweaver::PipelineConfig& config) skyweaver::detail::create_dada_file_stream< skyweaver::BTFPowersH>; - IBWriterType ib_handler(config, "ib", create_stream_callback_ib, pre_write_callback); + std::unique_ptr ib_handler; using StatsWriterType = skyweaver::MultiFileWriter>; -<<<<<<< HEAD -======= - std::unique_ptr stats_handler; - ->>>>>>> c89902c (introduce unique_ptr) typename StatsWriterType::CreateStreamCallBackType create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; -<<<<<<< HEAD StatsWriterType stats_handler(config, "stats", create_stream_callback_stats, pre_write_callback); -======= ->>>>>>> c89902c (introduce unique_ptr) if constexpr(enable_incoherent_dedispersion) { using CBWriterType = skyweaver::MultiFileWriter>; @@ -321,35 +313,32 @@ void setup_pipeline(skyweaver::PipelineConfig& config) skyweaver::TDBPowersH>; skyweaver::MultiFileWriter> cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); -======= - create_stream_callback_cb = - skyweaver::detail::create_dada_file_stream>; std::unique_ptr cb_file_writer; if (config.pre_write_config().is_enabled) { + ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib, pre_write_callback)); stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats, pre_write_callback)); cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb, pre_write_callback)); }else{ + ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib)); stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats)); cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb)); } ->>>>>>> c89902c (introduce unique_ptr) skyweaver::IncoherentDedispersionPipeline incoherent_dispersion_pipeline(config, * cb_file_writer.get()); skyweaver::BeamformerPipeline pipeline(config, incoherent_dispersion_pipeline, - ib_handler, + * ib_handler.get(), * stats_handler.get()); run_pipeline(pipeline, config, file_reader, header); } else { -<<<<<<< HEAD using CBWriterType = skyweaver::MultiFileWriter>; typename CBWriterType::CreateStreamCallBackType @@ -358,25 +347,23 @@ void setup_pipeline(skyweaver::PipelineConfig& config) skyweaver::TFBPowersD>; CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); skyweaver::BeamformerPipeline>; std::unique_ptr cb_file_writer; - typename CBWriterType::CreateStreamCallBackType - create_stream_callback_cb = - skyweaver::detail::create_dada_file_stream>; if (config.pre_write_config().is_enabled) { + ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib, pre_write_callback)); cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb, pre_write_callback)); + stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats, pre_write_callback)); }else{ + ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib)); cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb)); + stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats)); } skyweaver::BeamformerPipeline>>>>>> c89902c (introduce unique_ptr) decltype(ib_handler), decltype(* stats_handler.get()), BfTraits> - pipeline(config, * cb_file_writer.get(), ib_handler, * stats_handler.get()); + pipeline(config, * cb_file_writer.get(), * ib_handler.get(), * stats_handler.get()); run_pipeline(pipeline, config, file_reader, header); } } From 8da53b05dbb4d8847c75ce37ab794263fed9d46d Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Mon, 12 May 2025 10:29:51 +0200 Subject: [PATCH 67/69] resolve merge conflict --- cpp/skyweaver/MultiFileWriter.cuh | 14 +-- cpp/skyweaver/PipelineConfig.hpp | 1 - cpp/skyweaver/detail/MultiFileWriter.cu | 28 +----- cpp/skyweaver/src/PipelineConfig.cpp | 118 ++++++------------------ cpp/skyweaver/src/skyweaver_cli.cu | 22 ++--- 5 files changed, 46 insertions(+), 137 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 350fdb0..94c65db 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -25,7 +25,7 @@ struct MultiFileWriterConfig { std::string output_basename; std::string suffix; PreWriteConfig pre_write; - + MultiFileWriterConfig() : header_size(4096), max_file_size(2147483647), stokes_mode("I"), output_dir("default/"), base_output_dir("default_base/"), inner_dir(""), prefix(""), extension(""), output_basename(""), suffix("") {}; @@ -41,6 +41,7 @@ struct MultiFileWriterConfig { stokes_mode(stokes_mode), output_dir(output_dir), prefix(prefix), extension(extension), suffix(suffix), output_basename("") {}; + std::string to_string() { return "header_size: " + std::to_string(header_size) + @@ -51,8 +52,6 @@ struct MultiFileWriterConfig { ", base_output_dir: " + base_output_dir + ", inner_dir: " + inner_dir + ", suffix: " + suffix; } - - } }; /** * @brief A class for handling writing of DescribedVectors @@ -62,14 +61,15 @@ template class MultiFileWriter { public: + using PreWriteCallback = std::function; using CreateStreamCallBackType = std::function( MultiFileWriterConfig const&, ObservationHeader const&, VectorType const&, std::size_t)>; + public: - using PreWriteCallback = std::function; /** * @brief Construct a new Multi File Writer object * @@ -81,13 +81,13 @@ class MultiFileWriter MultiFileWriter(PipelineConfig const& config, std::string tag, CreateStreamCallBackType create_stream_callback); - MultiFileWriter(MultiFileWriterConfig config, - std::string tag, - CreateStreamCallBackType create_stream_callback); MultiFileWriter(PipelineConfig const& config, std::string tag, CreateStreamCallBackType create_stream_callback, PreWriteCallback pre_write_callback); + MultiFileWriter(MultiFileWriterConfig config, + std::string tag, + CreateStreamCallBackType create_stream_callback); MultiFileWriter(MultiFileWriterConfig config, std::string tag, CreateStreamCallBackType create_stream_callback, diff --git a/cpp/skyweaver/PipelineConfig.hpp b/cpp/skyweaver/PipelineConfig.hpp index 0974023..79becc2 100644 --- a/cpp/skyweaver/PipelineConfig.hpp +++ b/cpp/skyweaver/PipelineConfig.hpp @@ -22,7 +22,6 @@ namespace skyweaver WaitConfig wait; }; - /** * @brief Class for wrapping the skyweaver pipeline configuration. */ diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 116b241..3c0a033 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -48,24 +48,7 @@ MultiFileWriter::MultiFileWriter( _config.stokes_mode = config.stokes_mode(); _config.base_output_dir = config.output_dir(); _config.inner_dir = ""; - _pre_write_callback = nullptr; -} - template -MultiFileWriter::MultiFileWriter(PipelineConfig const& config, - std::string tag, - CreateStreamCallBackType create_stream_callback, - PreWriteCallback pre_write_callback) - : _tag(tag), _create_stream_callback(create_stream_callback), _pre_write_callback(pre_write_callback) -{ - MultiFileWriterConfig writer_config; - writer_config.header_size = config.dada_header_size(); - writer_config.max_file_size = config.max_output_filesize(); - writer_config.stokes_mode = config.stokes_mode(); - writer_config.output_dir = config.output_dir(); - writer_config.pre_write = config.pre_write_config(); - _config = writer_config; - _config.pre_write = writer_config.pre_write; } template @@ -81,7 +64,7 @@ MultiFileWriter::MultiFileWriter( _config.stokes_mode = config.stokes_mode(); _config.base_output_dir = config.output_dir(); _config.inner_dir = ""; - _config.pre_write = writer_config.pre_write; + } template @@ -101,15 +84,6 @@ MultiFileWriter::MultiFileWriter( std::string tag, CreateStreamCallBackType create_stream_callback, PreWriteCallback pre_write_callback) - : _config(config), _tag(tag), _create_stream_callback(create_stream_callback), _pre_write_callback(pre_write_callback) -{ -} - -template MultiFileWriter::MultiFileWriter( - MultiFileWriterConfig config, - std::string tag, - CreateStreamCallBackType create_stream_callback, - PreWriteCallback pre_write_callback) : _config(config), _tag(tag), _create_stream_callback(create_stream_callback), _pre_write_callback(pre_write_callback) diff --git a/cpp/skyweaver/src/PipelineConfig.cpp b/cpp/skyweaver/src/PipelineConfig.cpp index f3debcf..79fafb1 100644 --- a/cpp/skyweaver/src/PipelineConfig.cpp +++ b/cpp/skyweaver/src/PipelineConfig.cpp @@ -10,12 +10,13 @@ namespace skyweaver PipelineConfig::PipelineConfig() : _input_files({}), _check_input_contiguity(false), _dada_header_size(4096), _delay_file(""), _output_dir("./"), _max_output_filesize(10000000000000), - _output_file_prefix(""), _enable_incoherent_dedispersion(true), - _cfreq(1284000000.0), _bw(13375000.0), _channel_frequencies_stale(true), + _output_file_prefix(""), + _enable_incoherent_dedispersion(true), _cfreq(1284000000.0), + _bw(13375000.0), _channel_frequencies_stale(true), _gulp_length_samps(4096), _start_time(0.0f), _duration(std::numeric_limits::infinity()), _total_nchans(4096), - _stokes_mode("I"), _output_level(24.0f), _output_statistics(true), - _output_incoherent_beam(true), _pre_write_config({{false, 0, 0, 0}}) + _stokes_mode("I"), _output_level(24.0f), _output_statistics(true), _output_incoherent_beam(true), + _pre_write_config({0, {false, 0, 0}}) { } @@ -157,21 +158,20 @@ DedispersionPlan const& PipelineConfig::ddplan() const return _ddplan; } -std::size_t PipelineConfig::convertMemorySize(const std::string& str) const -{ +std::size_t PipelineConfig::convertMemorySize(const std::string& str) const { std::size_t lastCharPos = str.find_last_not_of("0123456789"); - std::string numberPart = str.substr(0, lastCharPos); - std::string unitPart = str.substr(lastCharPos); + std::string numberPart = str.substr(0, lastCharPos); + std::string unitPart = str.substr(lastCharPos); std::size_t number = std::stoull(numberPart); - if(unitPart.empty()) + if (unitPart.empty()) return number; - else if(unitPart == "K" || unitPart == "k") + else if (unitPart == "K" || unitPart == "k") return number * 1024; - else if(unitPart == "M" || unitPart == "m") + else if (unitPart == "M" || unitPart == "m") return number * 1024 * 1024; - else if(unitPart == "G" || unitPart == "g") + else if (unitPart == "G" || unitPart == "g") return number * 1024 * 1024 * 1024; else throw std::runtime_error("Invalid memory unit!"); @@ -182,93 +182,33 @@ void PipelineConfig::configure_wait(std::string argument) std::vector tokens; std::string token; std::istringstream tokenStream(argument); - int indx = 0; + int indx = 0; _pre_write_config.is_enabled = true; - while(std::getline(tokenStream, token, ':')) { - if(indx == 0) { - errno = 0; - _pre_write_config.wait.iterations = std::stoi(token); - if(errno == ERANGE) { - throw std::runtime_error("Wait iteration number out of range!"); - } - if(_pre_write_config.wait.iterations < 0) - _pre_write_config.wait.iterations = 0; - } else if(indx == 1) { - errno = 0; - _pre_write_config.wait.sleep_time = std::stoi(token); - if(errno == ERANGE) { - throw std::runtime_error("Sleep time out of range!"); - } - if(_pre_write_config.wait.sleep_time < 1) - _pre_write_config.wait.sleep_time = 1; - } else if(indx == 2) { - if(!token.empty() && - std::all_of(token.begin(), token.end(), ::isdigit)) { - _pre_write_config.wait.min_free_space = std::stoull(token); - } else { - try { - _pre_write_config.wait.min_free_space = - convertMemorySize(token); - } catch(std::runtime_error& e) { - std::cout << "Memory conversion error: " << e.what() - << std::endl; - throw; - } - } - } - indx++; - } -} - -std::size_t number = std::stoull(numberPart); - -if(unitPart.empty()) - return number; -else if(unitPart == "K" || unitPart == "k") - return number * 1024; -else if(unitPart == "M" || unitPart == "m") - return number * 1024 * 1024; -else if(unitPart == "G" || unitPart == "g") - return number * 1024 * 1024 * 1024; -else - throw std::runtime_error("Invalid memory unit!"); -} - -void PipelineConfig::configure_wait(std::string argument) -{ - std::vector tokens; - std::string token; - std::istringstream tokenStream(argument); - int indx = 0; - _pre_write_config.wait.is_enabled = true; - while(std::getline(tokenStream, token, ':')) { - if(indx == 0) { - errno = 0; + while (std::getline(tokenStream, token, ':')) { + if(indx == 0) + { + errno = 0; _pre_write_config.wait.iterations = std::stoi(token); - if(errno == ERANGE) { - throw std::runtime_error("Wait iteration number out of range!"); + if (errno == ERANGE) { + throw std::runtime_error("Wait iteration number out of range!"); } - if(_pre_write_config.wait.iterations < 0) - _pre_write_config.wait.iterations = 0; + if (_pre_write_config.wait.iterations < 0) _pre_write_config.wait.iterations = 0; } else if(indx == 1) { - errno = 0; + errno = 0; _pre_write_config.wait.sleep_time = std::stoi(token); - if(errno == ERANGE) { - throw std::runtime_error("Sleep time out of range!"); + if (errno == ERANGE) { + throw std::runtime_error("Sleep time out of range!"); } - if(_pre_write_config.wait.sleep_time < 1) - _pre_write_config.wait.sleep_time = 1; + if (_pre_write_config.wait.sleep_time < 1) _pre_write_config.wait.sleep_time = 1; } else if(indx == 2) { - if(!token.empty() && - std::all_of(token.begin(), token.end(), ::isdigit)) { + if (!token.empty() && std::all_of(token.begin(), token.end(), ::isdigit)) + { _pre_write_config.wait.min_free_space = std::stoull(token); } else { try { - _pre_write_config.wait.min_free_space = - convertMemorySize(token); - } catch(std::runtime_error& e) { - std::cout << "Memory conversion error: " << e.what() - << std::endl; + _pre_write_config.wait.min_free_space = convertMemorySize(token); + } catch (std::runtime_error& e) { + std::cout << "Memory conversion error: " << e.what() << std::endl; throw; } } diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index aea8066..7987874 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -294,14 +294,13 @@ void setup_pipeline(skyweaver::PipelineConfig& config) using StatsWriterType = skyweaver::MultiFileWriter>; + + std::unique_ptr stats_handler; + typename StatsWriterType::CreateStreamCallBackType create_stream_callback_stats = skyweaver::detail::create_dada_file_stream< skyweaver::FPAStatsD>; - StatsWriterType stats_handler(config, - "stats", - create_stream_callback_stats, - pre_write_callback); if constexpr(enable_incoherent_dedispersion) { using CBWriterType = @@ -310,8 +309,7 @@ void setup_pipeline(skyweaver::PipelineConfig& config) create_stream_callback_cb = skyweaver::detail::create_dada_file_stream< skyweaver::TDBPowersH>; - skyweaver::MultiFileWriter> - cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); + std::unique_ptr cb_file_writer; if (config.pre_write_config().is_enabled) { @@ -340,13 +338,11 @@ void setup_pipeline(skyweaver::PipelineConfig& config) } else { using CBWriterType = skyweaver::MultiFileWriter>; + std::unique_ptr cb_file_writer; typename CBWriterType::CreateStreamCallBackType create_stream_callback_cb = skyweaver::detail::create_dada_file_stream< skyweaver::TFBPowersD>; - CBWriterType cb_file_writer(config, "cb", create_stream_callback_cb, pre_write_callback); - skyweaver::BeamformerPipeline cb_file_writer; if (config.pre_write_config().is_enabled) { ib_handler.reset(new IBWriterType(config, "ib", create_stream_callback_ib, pre_write_callback)); @@ -357,9 +353,8 @@ void setup_pipeline(skyweaver::PipelineConfig& config) cb_file_writer.reset(new CBWriterType(config, "cb", create_stream_callback_cb)); stats_handler.reset(new StatsWriterType(config, "stats", create_stream_callback_stats)); } - - skyweaver::BeamformerPipeline pipeline(config, * cb_file_writer.get(), * ib_handler.get(), * stats_handler.get()); @@ -528,6 +523,7 @@ int main(int argc, char** argv) [](std::size_t nthreads) { omp_set_num_threads(nthreads); }), "The number of threads to use for incoherent dedispersion") + // Waiting options ("wait-for-space", po::value() ->notifier( @@ -535,7 +531,7 @@ int main(int argc, char** argv) "Wait for enough disk space for the output. " "::") - // Logging options + // Logging options ("log-level", po::value()->default_value("info")->notifier( [](std::string level) { skyweaver::set_log_level(level); }), From 4325201df4de5fdd3ba55a301b3d93ae871613ba Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Wed, 14 May 2025 16:54:26 +0200 Subject: [PATCH 68/69] need to pass the prewrite config --- cpp/skyweaver/detail/MultiFileWriter.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 3c0a033..7c26aa1 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -63,6 +63,7 @@ MultiFileWriter::MultiFileWriter( _config.max_file_size = config.max_output_filesize(); _config.stokes_mode = config.stokes_mode(); _config.base_output_dir = config.output_dir(); + _config.pre_write = config.pre_write_config(); _config.inner_dir = ""; } From ae5d13ecad29860eb6483a9a3be5a5a13c44e454 Mon Sep 17 00:00:00 2001 From: Henning Fehrmann Date: Thu, 15 May 2025 12:56:29 +0200 Subject: [PATCH 69/69] need to work with base_output_dir now. Just compute the filesystem size --- cpp/skyweaver/MultiFileWriter.cuh | 2 +- cpp/skyweaver/detail/MultiFileWriter.cu | 7 +++---- cpp/skyweaver/src/skyweaver_cli.cu | 12 ++++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cpp/skyweaver/MultiFileWriter.cuh b/cpp/skyweaver/MultiFileWriter.cuh index 94c65db..be49c88 100644 --- a/cpp/skyweaver/MultiFileWriter.cuh +++ b/cpp/skyweaver/MultiFileWriter.cuh @@ -61,7 +61,7 @@ template class MultiFileWriter { public: - using PreWriteCallback = std::function; + using PreWriteCallback = std::function; using CreateStreamCallBackType = std::function( MultiFileWriterConfig const&, diff --git a/cpp/skyweaver/detail/MultiFileWriter.cu b/cpp/skyweaver/detail/MultiFileWriter.cu index 7c26aa1..d5743f0 100644 --- a/cpp/skyweaver/detail/MultiFileWriter.cu +++ b/cpp/skyweaver/detail/MultiFileWriter.cu @@ -63,7 +63,7 @@ MultiFileWriter::MultiFileWriter( _config.max_file_size = config.max_output_filesize(); _config.stokes_mode = config.stokes_mode(); _config.base_output_dir = config.output_dir(); - _config.pre_write = config.pre_write_config(); + _config.pre_write = config.pre_write_config(); _config.inner_dir = ""; } @@ -188,10 +188,9 @@ template bool MultiFileWriter::operator()(VectorType const& stream_data, std::size_t stream_idx) { - std::size_t const data_size = stream_data.size() * sizeof(typename VectorType::value_type); if (_pre_write_callback != nullptr && _config.pre_write.is_enabled) { - _pre_write_callback(data_size, _config); + _pre_write_callback(_config); } if(!has_stream(stream_idx)) { create_stream(stream_data, stream_idx); @@ -207,7 +206,7 @@ bool MultiFileWriter::operator()(VectorType const& stream_data, _file_streams.at(stream_idx) ->write(reinterpret_cast( thrust::raw_pointer_cast(stream_data.data())), - data_size); + stream_data.size() * sizeof(typename VectorType::value_type)); } return false; } diff --git a/cpp/skyweaver/src/skyweaver_cli.cu b/cpp/skyweaver/src/skyweaver_cli.cu index 7987874..39d44c5 100644 --- a/cpp/skyweaver/src/skyweaver_cli.cu +++ b/cpp/skyweaver/src/skyweaver_cli.cu @@ -240,25 +240,25 @@ void run_pipeline(Pipeline& pipeline, stopwatch.show_all_timings(); } -auto pre_write_callback = [] (std::size_t const size, skyweaver::MultiFileWriterConfig const& config) +auto pre_write_callback = [] (skyweaver::MultiFileWriterConfig const& config) { if (!config.pre_write.is_enabled) return; - std::filesystem::space_info space = std::filesystem::space(config.output_dir); - size_t limit = std::min(config.pre_write.wait.min_free_space, size); - if(space.available >= limit) + std::filesystem::space_info space = std::filesystem::space(config.base_output_dir); + size_t limit = config.pre_write.wait.min_free_space; + if(space.available >= config.pre_write.wait.min_free_space) return; BOOST_LOG_TRIVIAL(info) << space.available << " bytes available space is not enough. Need at least " << limit - << " bytes in " << config.output_dir << "."; + << " bytes in " << config.base_output_dir << "."; BOOST_LOG_TRIVIAL(warning) << "Start pausing."; int max_iterations = (config.pre_write.wait.iterations == 0) ? INT_MAX : config.pre_write.wait.iterations; for (int i = 0; i < max_iterations; i++) { sleep(config.pre_write.wait.sleep_time); - space = std::filesystem::space(config.output_dir); + space = std::filesystem::space(config.base_output_dir); if (space.available >= limit) { BOOST_LOG_TRIVIAL(info) << "Space has been freed up. Will proceed.";