diff --git a/Makefile b/Makefile index bfb3b3dac..536689c99 100644 --- a/Makefile +++ b/Makefile @@ -40,15 +40,13 @@ lint-py: python -m ruff format --check csp/ examples/ setup.py lint-cpp: - # clang-format --dry-run -Werror -i -style=file `find ./cpp/ -name "*.*pp"` - echo "C++ linting disabled for now" + clang-format --dry-run -Werror -i -style=file `find ./cpp/csp -type f -name "*.cpp" -o -name "*.h"` lint-docs: python -m mdformat --check docs/wiki/ README.md examples/ python -m codespell_lib docs/wiki/ README.md examples/ --skip "*.cpp,*.h" -# lint: lint-py lint-cpp ## run lints -lint: lint-py lint-docs ## run lints +lint: lint-py lint-cpp lint-docs ## run lints # Alias lints: lint @@ -58,8 +56,7 @@ fix-py: python -m ruff format csp/ examples/ setup.py fix-cpp: - # clang-format -i -style=file `find ./cpp/ -name "*.*pp"` - echo "C++ autoformatting disabled for now" + clang-format -i -style=file `find ./cpp/csp -type f -name "*.cpp" -o -name "*.h"` fix-docs: python -m mdformat docs/wiki/ README.md examples/ diff --git a/conda/dev-environment-unix.yml b/conda/dev-environment-unix.yml index 60b77eba4..15f5b107c 100644 --- a/conda/dev-environment-unix.yml +++ b/conda/dev-environment-unix.yml @@ -6,6 +6,7 @@ dependencies: - bison - brotli - bump-my-version + - clang-format>=19,<20 - cmake - codespell>=2.2.6,<2.3 - compilers diff --git a/conda/dev-environment-win.yml b/conda/dev-environment-win.yml index 8da992697..b47137a4c 100644 --- a/conda/dev-environment-win.yml +++ b/conda/dev-environment-win.yml @@ -5,6 +5,7 @@ channels: dependencies: - brotli - bump-my-version + - clang-format>=19,<20 - cmake - codespell>=2.2.6,<2.3 - compilers diff --git a/cpp/.clang-format b/cpp/.clang-format new file mode 100644 index 000000000..f89e2d6c8 --- /dev/null +++ b/cpp/.clang-format @@ -0,0 +1,69 @@ +--- +Language: Cpp +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignEscapedNewlinesLeft: false +AlignOperands: false +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: false +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + BeforeElse: true +BreakBeforeBinaryOperators: All +BreakBeforeBraces: Allman +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: true +ColumnLimit: 120 +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +IndentCaseLabels: true +IndentWidth: 4 +IndentWrappedFunctionNames: false +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: Inner +ObjCBlockIndentWidth: 4 +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: true +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Middle +SortIncludes: CaseInsensitive +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: true +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 4 +UseTab: Never \ No newline at end of file diff --git a/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp b/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp index f8e03586f..923233988 100644 --- a/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp +++ b/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp @@ -2,8 +2,8 @@ #include #include #include -#include #include +#include #include #include @@ -11,12 +11,8 @@ namespace csp { -INIT_CSP_ENUM( csp::adapters::kafka::KafkaStatusMessageType, - "OK", - "MSG_DELIVERY_FAILED", - "MSG_SEND_ERROR", - "MSG_RECV_ERROR" -); +INIT_CSP_ENUM( csp::adapters::kafka::KafkaStatusMessageType, "OK", "MSG_DELIVERY_FAILED", "MSG_SEND_ERROR", + "MSG_RECV_ERROR" ); } @@ -26,20 +22,23 @@ namespace csp::adapters::kafka class DeliveryReportCb : public RdKafka::DeliveryReportCb { public: - DeliveryReportCb( KafkaAdapterManager * mgr ) : m_adapterManager( mgr ) + DeliveryReportCb( KafkaAdapterManager * mgr ) + : m_adapterManager( mgr ) { } - void dr_cb( RdKafka::Message &message ) final + void dr_cb( RdKafka::Message & message ) final { /* If message.err() is non-zero the message delivery failed permanently * for the message. */ if( message.err() ) { - std::string msg = "KafkaPublisher: Message delivery failed for topic " + message.topic_name() + ". Failure: " + message.errstr(); - m_adapterManager -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_DELIVERY_FAILED, msg ); + std::string msg = "KafkaPublisher: Message delivery failed for topic " + message.topic_name() + + ". Failure: " + message.errstr(); + m_adapterManager->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_DELIVERY_FAILED, msg ); } } + private: KafkaAdapterManager * m_adapterManager; }; @@ -47,25 +46,28 @@ class DeliveryReportCb : public RdKafka::DeliveryReportCb class EventCb : public RdKafka::EventCb { public: - EventCb( KafkaAdapterManager * mgr ) : m_adapterManager( mgr ) {} + EventCb( KafkaAdapterManager * mgr ) + : m_adapterManager( mgr ) + { + } void event_cb( RdKafka::Event & event ) override { if( event.severity() < RdKafka::Event::EVENT_SEVERITY_NOTICE ) { - std::string errmsg = "KafkaConsumer: error ( " + std::to_string( event.err() ) + "): " + RdKafka::err2str( ( RdKafka::ErrorCode ) event.err() ) + ". Reason: " + event.str(); - m_adapterManager -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::GENERIC_ERROR, errmsg ); + std::string errmsg = "KafkaConsumer: error ( " + std::to_string( event.err() ) + + "): " + RdKafka::err2str( (RdKafka::ErrorCode)event.err() ) + ". Reason: " + event.str(); + m_adapterManager->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::GENERIC_ERROR, errmsg ); } if( event.type() == RdKafka::Event::EVENT_ERROR ) { - //We shutdown the app if its a fatal error OR if its an authentication issue which has plagued users multiple times - //Adding ERR__ALL_BROKERS_DOWN which happens when all brokers are down - if( event.fatal() || - event.err() == RdKafka::ErrorCode::ERR__AUTHENTICATION || - event.err() == RdKafka::ErrorCode::ERR__ALL_BROKERS_DOWN ) + // We shutdown the app if its a fatal error OR if its an authentication issue which has plagued users + // multiple times Adding ERR__ALL_BROKERS_DOWN which happens when all brokers are down + if( event.fatal() || event.err() == RdKafka::ErrorCode::ERR__AUTHENTICATION + || event.err() == RdKafka::ErrorCode::ERR__ALL_BROKERS_DOWN ) { - m_adapterManager -> forceShutdown( RdKafka::err2str( ( RdKafka::ErrorCode ) event.err() ) + event.str() ); + m_adapterManager->forceShutdown( RdKafka::err2str( (RdKafka::ErrorCode)event.err() ) + event.str() ); } } } @@ -74,15 +76,16 @@ class EventCb : public RdKafka::EventCb KafkaAdapterManager * m_adapterManager; }; -KafkaAdapterManager::KafkaAdapterManager( csp::Engine * engine, const Dictionary & properties ) : AdapterManager( engine ), - m_consumerIdx( 0 ), - m_producerPollThreadActive( false ), - m_unrecoverableError( false ) +KafkaAdapterManager::KafkaAdapterManager( csp::Engine * engine, const Dictionary & properties ) + : AdapterManager( engine ) + , m_consumerIdx( 0 ) + , m_producerPollThreadActive( false ) + , m_unrecoverableError( false ) { - m_maxThreads = properties.get( "max_threads" ); + m_maxThreads = properties.get( "max_threads" ); m_pollTimeoutMs = properties.get( "poll_timeout" ).asMilliseconds(); - m_eventCb = std::make_unique( this ); + m_eventCb = std::make_unique( this ); m_producerCb = std::make_unique( this ); std::string errstr; @@ -94,19 +97,19 @@ KafkaAdapterManager::KafkaAdapterManager( csp::Engine * engine, const Dictionary setConfProperties( m_consumerConf.get(), *properties.get( "rd_kafka_consumer_conf_properties" ) ); if( properties.exists( "start_offset" ) ) { - //used later in start since we need starttime + // used later in start since we need starttime m_startOffsetProperty = properties.getUntypedValue( "start_offset" ); } - if( m_consumerConf -> set( "event_cb", m_eventCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) + if( m_consumerConf->set( "event_cb", m_eventCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set consumer error cb: " << errstr ); m_producerConf.reset( RdKafka::Conf::create( RdKafka::Conf::CONF_GLOBAL ) ); setConfProperties( m_producerConf.get(), rdKafkaProperties ); setConfProperties( m_producerConf.get(), *properties.get( "rd_kafka_producer_conf_properties" ) ); - if( m_producerConf -> set( "dr_cb", m_producerCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) + if( m_producerConf->set( "dr_cb", m_producerCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set producer callback: " << errstr ); - if( m_producerConf -> set( "event_cb", m_eventCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) + if( m_producerConf->set( "event_cb", m_eventCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set producer error cb: " << errstr ); } @@ -116,7 +119,7 @@ KafkaAdapterManager::~KafkaAdapterManager() if( m_producerPollThreadActive ) { m_producerPollThreadActive = false; - m_producerPollThread -> join(); + m_producerPollThread->join(); } } @@ -126,31 +129,31 @@ void KafkaAdapterManager::setConfProperties( RdKafka::Conf * conf, const Diction for( auto it = properties.begin(); it != properties.end(); ++it ) { - std::string key = it.key(); + std::string key = it.key(); std::string value = properties.get( key ); - if( conf -> set( key, value, errstr ) != RdKafka::Conf::CONF_OK ) + if( conf->set( key, value, errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set property " << key << ": " << errstr ); } } void KafkaAdapterManager::forceShutdown( const std::string & err ) { - m_unrecoverableError = true; // So we can alert the producer to stop trying to flush + m_unrecoverableError = true; // So we can alert the producer to stop trying to flush forceConsumerReplayComplete(); try { - CSP_THROW( RuntimeException, "Kafka fatal error. " + err ); + CSP_THROW( RuntimeException, "Kafka fatal error. " + err ); } catch( const RuntimeException & ) { - rootEngine() -> shutdown( std::current_exception() ); + rootEngine()->shutdown( std::current_exception() ); } } void KafkaAdapterManager::forceConsumerReplayComplete() { for( auto & consumer : m_consumerVector ) - consumer -> forceReplayCompleted(); + consumer->forceReplayCompleted(); } void KafkaAdapterManager::start( DateTime starttime, DateTime endtime ) @@ -160,7 +163,7 @@ void KafkaAdapterManager::start( DateTime starttime, DateTime endtime ) if( !m_staticPublishers.empty() || !m_dynamicPublishers.empty() ) { m_producer.reset( RdKafka::Producer::create( m_producerConf.get(), errstr ) ); - if ( !m_producer ) + if( !m_producer ) { CSP_THROW( RuntimeException, "Failed to create producer: " << errstr ); } @@ -168,21 +171,21 @@ void KafkaAdapterManager::start( DateTime starttime, DateTime endtime ) // start all consumers for( auto & it : m_consumerVector ) - it -> start( starttime ); + it->start( starttime ); // start all publishers for( auto & it : m_staticPublishers ) - it.second -> start( m_producer ); + it.second->start( m_producer ); for( auto & it : m_dynamicPublishers ) - it -> start( m_producer ); + it->start( m_producer ); AdapterManager::start( starttime, endtime ); if( !m_staticPublishers.empty() || !m_dynamicPublishers.empty() ) { m_producerPollThreadActive = true; - m_producerPollThread = std::make_unique( [ this ](){ pollProducers(); } ); + m_producerPollThread = std::make_unique( [this]() { pollProducers(); } ); } } @@ -192,20 +195,20 @@ void KafkaAdapterManager::stop() // stop all consumers for( auto & it : m_consumerVector ) - it -> stop(); + it->stop(); if( m_producerPollThreadActive ) { m_producerPollThreadActive = false; - m_producerPollThread -> join(); + m_producerPollThread->join(); } // stop all publishers for( auto & it : m_staticPublishers ) - it.second -> stop(); + it.second->stop(); for( auto & it : m_dynamicPublishers ) - it -> stop(); + it->stop(); m_staticPublishers.clear(); m_dynamicPublishers.clear(); @@ -223,33 +226,35 @@ void KafkaAdapterManager::pollProducers() { while( m_producerPollThreadActive ) { - m_producer -> poll( m_pollTimeoutMs ); + m_producer->poll( m_pollTimeoutMs ); } try { while( true ) { - auto rc = m_producer -> flush( 5000 ); + auto rc = m_producer->flush( 5000 ); if( !rc || m_unrecoverableError ) break; if( rc != RdKafka::ERR__TIMED_OUT ) - CSP_THROW( RuntimeException, "KafkaProducer failed to flush pending msgs on shutdown: " << RdKafka::err2str( rc ) ); + CSP_THROW( RuntimeException, + "KafkaProducer failed to flush pending msgs on shutdown: " << RdKafka::err2str( rc ) ); } } catch( ... ) { - rootEngine() -> shutdown( std::current_exception() ); + rootEngine()->shutdown( std::current_exception() ); } } -PushInputAdapter * KafkaAdapterManager::getInputAdapter( CspTypePtr & type, PushMode pushMode, const Dictionary & properties ) +PushInputAdapter * KafkaAdapterManager::getInputAdapter( CspTypePtr & type, PushMode pushMode, + const Dictionary & properties ) { - std::string topic = properties.get( "topic" ); - std::string key = properties.get( "key" ); - KafkaSubscriber * subscriber = this -> getSubscriber( topic, key, properties ); - return subscriber -> getInputAdapter( type, pushMode, properties ); + std::string topic = properties.get( "topic" ); + std::string key = properties.get( "key" ); + KafkaSubscriber * subscriber = this->getSubscriber( topic, key, properties ); + return subscriber->getInputAdapter( type, pushMode, properties ); } OutputAdapter * KafkaAdapterManager::getOutputAdapter( CspTypePtr & type, const Dictionary & properties ) @@ -257,20 +262,20 @@ OutputAdapter * KafkaAdapterManager::getOutputAdapter( CspTypePtr & type, const std::string topic = properties.get( "topic" ); try { - auto key = properties.get( "key" ); - auto pair = TopicKeyPair( topic, key ); - KafkaPublisher * publisher = this -> getStaticPublisher( pair, properties ); - return publisher -> getOutputAdapter( type, properties, key ); + auto key = properties.get( "key" ); + auto pair = TopicKeyPair( topic, key ); + KafkaPublisher * publisher = this->getStaticPublisher( pair, properties ); + return publisher->getOutputAdapter( type, properties, key ); } catch( TypeError & e ) { - auto key = properties.get>( "key" ); + auto key = properties.get>( "key" ); std::vector keyFields; for( auto & it : key ) keyFields.emplace_back( std::get( it._data ) ); - KafkaPublisher * publisher = this -> getDynamicPublisher( topic, properties ); - return publisher -> getOutputAdapter( type, properties, keyFields ); + KafkaPublisher * publisher = this->getDynamicPublisher( topic, properties ); + return publisher->getOutputAdapter( type, properties, keyFields ); } } @@ -281,37 +286,38 @@ KafkaConsumer * KafkaAdapterManager::getConsumer( const std::string & topic, con // If we have reached m_maxThreads, then round-robin the topic onto a consumer (and insert it into the map) if( m_consumerMap.find( topic ) != m_consumerMap.end() ) { - return m_consumerMap[ topic ].get(); + return m_consumerMap[topic].get(); } if( m_consumerVector.size() < m_maxThreads ) { auto consumer = std::make_shared( this, properties ); m_consumerVector.emplace_back( consumer ); m_consumerMap.emplace( topic, consumer ); - return m_consumerMap[ topic ].get(); + return m_consumerMap[topic].get(); } - auto consumer = m_consumerVector[ m_consumerIdx++ ]; + auto consumer = m_consumerVector[m_consumerIdx++]; m_consumerMap.emplace( topic, consumer ); if( m_consumerIdx >= m_maxThreads ) m_consumerIdx = 0; return consumer.get(); } -KafkaSubscriber * KafkaAdapterManager::getSubscriber( const std::string & topic, const std::string & key, const Dictionary & properties ) +KafkaSubscriber * KafkaAdapterManager::getSubscriber( const std::string & topic, const std::string & key, + const Dictionary & properties ) { auto pair = TopicKeyPair( topic, key ); - auto rv = m_subscribers.emplace( pair, nullptr ); + auto rv = m_subscribers.emplace( pair, nullptr ); if( rv.second ) { std::unique_ptr subscriber( new KafkaSubscriber( this, properties ) ); - rv.first -> second = std::move( subscriber ); + rv.first->second = std::move( subscriber ); - this -> getConsumer( topic, properties ) -> addSubscriber( topic, key, rv.first -> second.get() ); + this->getConsumer( topic, properties )->addSubscriber( topic, key, rv.first->second.get() ); } - return rv.first -> second.get(); + return rv.first->second.get(); } // for static (string) keys, we create one publisher instance per pair @@ -322,10 +328,10 @@ KafkaPublisher * KafkaAdapterManager::getStaticPublisher( const TopicKeyPair & p if( rv.second ) { std::unique_ptr publisher( new KafkaPublisher( this, properties, pair.first ) ); - rv.first -> second = std::move( publisher ); + rv.first->second = std::move( publisher ); } - KafkaPublisher * p = rv.first -> second.get(); + KafkaPublisher * p = rv.first->second.get(); return p; } @@ -337,4 +343,4 @@ KafkaPublisher * KafkaAdapterManager::getDynamicPublisher( const std::string & t return p; } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaAdapterManager.h b/cpp/csp/adapters/kafka/KafkaAdapterManager.h index 7f6bf8fb8..1ce7cdc2e 100644 --- a/cpp/csp/adapters/kafka/KafkaAdapterManager.h +++ b/cpp/csp/adapters/kafka/KafkaAdapterManager.h @@ -1,12 +1,12 @@ #ifndef _IN_CSP_ADAPTERS_KAFKA_KAFKAADAPTERMANAGER_H #define _IN_CSP_ADAPTERS_KAFKA_KAFKAADAPTERMANAGER_H +#include #include #include #include #include #include -#include #include #include #include @@ -19,7 +19,7 @@ class Conf; class DeliveryReportCb; class EventCb; class Producer; -} +} // namespace RdKafka namespace csp::adapters::kafka { @@ -32,11 +32,11 @@ struct KafkaStatusMessageTypeTraits { enum _enum : unsigned char { - OK = 0, + OK = 0, MSG_DELIVERY_FAILED = 1, - MSG_SEND_ERROR = 2, - MSG_RECV_ERROR = 3, - GENERIC_ERROR = 4, + MSG_SEND_ERROR = 2, + MSG_RECV_ERROR = 3, + GENERIC_ERROR = 4, NUM_TYPES }; @@ -47,7 +47,7 @@ struct KafkaStatusMessageTypeTraits using KafkaStatusMessageType = csp::Enum; -//Top level AdapterManager object for all kafka adapters in the engine +// Top level AdapterManager object for all kafka adapters in the engine class KafkaAdapterManager final : public csp::AdapterManager { public: @@ -62,9 +62,9 @@ class KafkaAdapterManager final : public csp::AdapterManager DateTime processNextSimTimeSlice( DateTime time ) override; - //properties will have topic and fieldmap information, amongst other things + // properties will have topic and fieldmap information, amongst other things PushInputAdapter * getInputAdapter( CspTypePtr & type, PushMode pushMode, const Dictionary & properties ); - OutputAdapter * getOutputAdapter( CspTypePtr & type, const Dictionary & properties ); + OutputAdapter * getOutputAdapter( CspTypePtr & type, const Dictionary & properties ); KafkaConsumer * getConsumer( const std::string & topic, const Dictionary & properties ); @@ -77,34 +77,34 @@ class KafkaAdapterManager final : public csp::AdapterManager void forceShutdown( const std::string & err ); private: - using TopicKeyPair = std::pair; void setConfProperties( RdKafka::Conf * conf, const Dictionary & properties ); void pollProducers(); void forceConsumerReplayComplete(); - KafkaSubscriber * getSubscriber( const std::string & topic, const std::string & key, const Dictionary & properties ); - KafkaPublisher * getStaticPublisher( const TopicKeyPair & pair, const Dictionary & properties ); - KafkaPublisher * getDynamicPublisher( const std::string & topic, const Dictionary & properties ); + KafkaSubscriber * getSubscriber( const std::string & topic, const std::string & key, + const Dictionary & properties ); + KafkaPublisher * getStaticPublisher( const TopicKeyPair & pair, const Dictionary & properties ); + KafkaPublisher * getDynamicPublisher( const std::string & topic, const Dictionary & properties ); using ConsumerVector = std::vector>; - ConsumerVector m_consumerVector; + ConsumerVector m_consumerVector; using ConsumerMap = std::unordered_map>; - ConsumerMap m_consumerMap; + ConsumerMap m_consumerMap; using StaticPublishers = std::unordered_map, hash::hash_pair>; - StaticPublishers m_staticPublishers; + StaticPublishers m_staticPublishers; using DynamicPublishers = std::vector>; - DynamicPublishers m_dynamicPublishers; + DynamicPublishers m_dynamicPublishers; using Subscribers = std::unordered_map, hash::hash_pair>; - Subscribers m_subscribers; + Subscribers m_subscribers; - int m_pollTimeoutMs; - size_t m_maxThreads; - size_t m_consumerIdx; + int m_pollTimeoutMs; + size_t m_maxThreads; + size_t m_consumerIdx; std::unique_ptr m_eventCb; std::shared_ptr m_producer; @@ -113,11 +113,11 @@ class KafkaAdapterManager final : public csp::AdapterManager std::atomic m_producerPollThreadActive; std::atomic m_unrecoverableError; - std::unique_ptr m_consumerConf; - std::unique_ptr m_producerConf; - Dictionary::Value m_startOffsetProperty; + std::unique_ptr m_consumerConf; + std::unique_ptr m_producerConf; + Dictionary::Value m_startOffsetProperty; }; -} +} // namespace csp::adapters::kafka #endif diff --git a/cpp/csp/adapters/kafka/KafkaConsumer.cpp b/cpp/csp/adapters/kafka/KafkaConsumer.cpp index d4bf987ad..e21d3fa7b 100644 --- a/cpp/csp/adapters/kafka/KafkaConsumer.cpp +++ b/cpp/csp/adapters/kafka/KafkaConsumer.cpp @@ -8,26 +8,26 @@ namespace csp::adapters::kafka class RebalanceCb : public RdKafka::RebalanceCb { public: - RebalanceCb( KafkaConsumer & consumer ) : m_consumer( consumer ), - m_startOffset( RdKafka::Topic::OFFSET_INVALID ), - m_doneSeeking( false ) + RebalanceCb( KafkaConsumer & consumer ) + : m_consumer( consumer ) + , m_startOffset( RdKafka::Topic::OFFSET_INVALID ) + , m_doneSeeking( false ) { } void setStartOffset( int64_t offset ) { m_startOffset = offset; } - void setStartTime( DateTime time ) { m_startTime = time; } + void setStartTime( DateTime time ) { m_startTime = time; } - void rebalance_cb( RdKafka::KafkaConsumer *consumer, - RdKafka::ErrorCode err, - std::vector & partitions ) override + void rebalance_cb( RdKafka::KafkaConsumer * consumer, RdKafka::ErrorCode err, + std::vector & partitions ) override { if( err == RdKafka::ERR__ASSIGN_PARTITIONS ) { if( !m_doneSeeking ) { - std::unordered_map numPartitions; + std::unordered_map numPartitions; for( auto * partition : partitions ) - numPartitions[ partition -> topic() ] += 1; + numPartitions[partition->topic()] += 1; for( auto & entry : numPartitions ) m_consumer.setNumPartitions( entry.first, entry.second ); @@ -35,30 +35,34 @@ class RebalanceCb : public RdKafka::RebalanceCb if( !m_startTime.isNone() ) { for( auto * partition : partitions ) - partition -> set_offset( m_startTime.asMilliseconds() ); + partition->set_offset( m_startTime.asMilliseconds() ); - auto rc = consumer -> offsetsForTimes( partitions, 10000 ); + auto rc = consumer->offsetsForTimes( partitions, 10000 ); if( rc ) - CSP_THROW( RuntimeException, "Failed to get kafka offsets for starttime " << m_startTime << ": " << RdKafka::err2str( rc ) ); + CSP_THROW( RuntimeException, + "Failed to get kafka offsets for starttime " << m_startTime << ": " + << RdKafka::err2str( rc ) ); } else { for( auto * partition : partitions ) - partition -> set_offset( m_startOffset ); + partition->set_offset( m_startOffset ); } - auto rc = consumer -> assign( partitions ); + auto rc = consumer->assign( partitions ); if( rc ) - CSP_THROW( RuntimeException, "Failed to get kafka offsets for starttime " << m_startTime << ": " << RdKafka::err2str( rc ) ); + CSP_THROW( RuntimeException, + "Failed to get kafka offsets for starttime " << m_startTime << ": " + << RdKafka::err2str( rc ) ); m_doneSeeking = true; } else - consumer -> assign( partitions ); + consumer->assign( partitions ); } else { - consumer -> unassign(); + consumer->unassign(); } } @@ -69,15 +73,16 @@ class RebalanceCb : public RdKafka::RebalanceCb bool m_doneSeeking; }; -KafkaConsumer::KafkaConsumer( KafkaAdapterManager * mgr, const Dictionary & properties ) : m_mgr( mgr ), - m_running( false ) +KafkaConsumer::KafkaConsumer( KafkaAdapterManager * mgr, const Dictionary & properties ) + : m_mgr( mgr ) + , m_running( false ) { - if( mgr -> startOffsetProperty().index() > 0 ) + if( mgr->startOffsetProperty().index() > 0 ) m_rebalanceCb = std::make_unique( *this ); std::string errstr; - auto * conf = m_mgr -> getConsumerConf(); - if( conf -> set( "rebalance_cb", m_rebalanceCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) + auto * conf = m_mgr->getConsumerConf(); + if( conf->set( "rebalance_cb", m_rebalanceCb.get(), errstr ) != RdKafka::Conf::CONF_OK ) CSP_THROW( RuntimeException, "Failed to set rebalance callback: " << errstr ); m_consumer.reset( RdKafka::KafkaConsumer::create( conf, errstr ) ); @@ -105,20 +110,26 @@ void KafkaConsumer::addSubscriber( const std::string & topic, const std::string void KafkaConsumer::start( DateTime starttime ) { - //RebalanceCb is only used / available if we requested a start_offset + // RebalanceCb is only used / available if we requested a start_offset if( m_rebalanceCb ) { - auto & startOffsetProperty = m_mgr -> startOffsetProperty(); + auto & startOffsetProperty = m_mgr->startOffsetProperty(); if( std::holds_alternative( startOffsetProperty ) ) { - ReplayMode replayMode = ( ReplayMode ) std::get( startOffsetProperty ); + ReplayMode replayMode = (ReplayMode)std::get( startOffsetProperty ); switch( replayMode ) { - case ReplayMode::EARLIEST: m_rebalanceCb -> setStartOffset( RdKafka::Topic::OFFSET_BEGINNING ); break; - case ReplayMode::LATEST: m_rebalanceCb -> setStartOffset( RdKafka::Topic::OFFSET_END ); break; - case ReplayMode::START_TIME: m_rebalanceCb -> setStartTime( starttime ); break; - - case ReplayMode::NUM_TYPES: + case ReplayMode::EARLIEST: + m_rebalanceCb->setStartOffset( RdKafka::Topic::OFFSET_BEGINNING ); + break; + case ReplayMode::LATEST: + m_rebalanceCb->setStartOffset( RdKafka::Topic::OFFSET_END ); + break; + case ReplayMode::START_TIME: + m_rebalanceCb->setStartTime( starttime ); + break; + + case ReplayMode::NUM_TYPES: case ReplayMode::UNKNOWN: CSP_THROW( ValueError, "start_offset is unset" ); } @@ -126,38 +137,40 @@ void KafkaConsumer::start( DateTime starttime ) else if( std::holds_alternative( startOffsetProperty ) ) { auto dt = std::get( startOffsetProperty ); - m_rebalanceCb -> setStartTime( dt ); + m_rebalanceCb->setStartTime( dt ); } else if( std::holds_alternative( startOffsetProperty ) ) { auto delta = std::get( startOffsetProperty ); - m_rebalanceCb -> setStartTime( starttime - delta.abs() ); + m_rebalanceCb->setStartTime( starttime - delta.abs() ); } else CSP_THROW( TypeError, "Expected enum, datetime or timedelta for startOffset" ); } - //This is a bit convoluted, but basically if we dont have rebalanceCB set, that means we are in "groupid" mode - //which doesnt support seeking. We force the adapters into a live mode, because groupid mode leads to deadlocks - //on adapters that dont received any data since we dont have partition information available to declare them done ( we dont even connect to them all ) + // This is a bit convoluted, but basically if we dont have rebalanceCB set, that means we are in "groupid" mode + // which doesnt support seeking. We force the adapters into a live mode, because groupid mode leads to deadlocks + // on adapters that dont received any data since we dont have partition information available to declare them done ( + // we dont even connect to them all ) else forceReplayCompleted(); std::vector topics; - for (const auto& [topic, topic_data] : m_topics) + for( const auto & [topic, topic_data] : m_topics ) { topics.emplace_back( topic ); - // wildcard subscription has no guarantee of being in order + // wildcard subscription has no guarantee of being in order // we flag replay complete as soon as we identify it. if( topic_data.wildcardSubscriber ) - topic_data.wildcardSubscriber -> flagReplayComplete(); + topic_data.wildcardSubscriber->flagReplayComplete(); } - RdKafka::ErrorCode err = m_consumer -> subscribe( topics ); + RdKafka::ErrorCode err = m_consumer->subscribe( topics ); if( err ) - CSP_THROW( RuntimeException, "Failed to subscribe to " << m_topics.size() << " topics: " << RdKafka::err2str( err ) ); + CSP_THROW( RuntimeException, + "Failed to subscribe to " << m_topics.size() << " topics: " << RdKafka::err2str( err ) ); - m_running = true; - m_pollThread = std::make_unique( [ this ](){ poll(); } ); + m_running = true; + m_pollThread = std::make_unique( [this]() { poll(); } ); } void KafkaConsumer::stop() @@ -165,18 +178,18 @@ void KafkaConsumer::stop() if( m_running ) { m_running = false; - m_pollThread -> join(); + m_pollThread->join(); } if( m_consumer.get() ) { - m_consumer -> close(); + m_consumer->close(); m_consumer.reset(); } } void KafkaConsumer::setNumPartitions( const std::string & topic, size_t num ) { - auto & topicData = m_topics[ topic ]; + auto & topicData = m_topics[topic]; topicData.partitionLive.resize( num, false ); } @@ -192,71 +205,75 @@ void KafkaConsumer::poll() { while( m_running ) { - std::unique_ptr msg( m_consumer -> consume( m_mgr -> pollTimeoutMs() ) ); + std::unique_ptr msg( m_consumer->consume( m_mgr->pollTimeoutMs() ) ); - if( msg -> err() == RdKafka::ERR__TIMED_OUT ) + if( msg->err() == RdKafka::ERR__TIMED_OUT ) continue; - auto topicIt = m_topics.find( msg -> topic_name() ); + auto topicIt = m_topics.find( msg->topic_name() ); if( topicIt == m_topics.end() ) { - std::string errmsg = "KafkaConsumer: Message received on unknown topic: " + msg -> topic_name() + - " errcode: " + RdKafka::err2str( msg -> err() ) + " error: " + msg -> errstr(); - m_mgr -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); - - //We tend to accumulate more cases over time of error states that leave the engine deadlocked on PushPull adapters. - //This section is for cases where we get an error that is not topic specific, but is consumer specific, but we know its non-recoverable - //if it gets too long, or we realize that ANY error here should stop the engine, we can just always make it stop - if( msg -> err() == RdKafka::ERR_GROUP_AUTHORIZATION_FAILED ) - m_mgr -> forceShutdown( RdKafka::err2str( msg -> err() ) + " error: " + msg -> errstr() ); + std::string errmsg = "KafkaConsumer: Message received on unknown topic: " + msg->topic_name() + + " errcode: " + RdKafka::err2str( msg->err() ) + " error: " + msg->errstr(); + m_mgr->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); + + // We tend to accumulate more cases over time of error states that leave the engine deadlocked on + // PushPull adapters. This section is for cases where we get an error that is not topic specific, but is + // consumer specific, but we know its non-recoverable if it gets too long, or we realize that ANY error + // here should stop the engine, we can just always make it stop + if( msg->err() == RdKafka::ERR_GROUP_AUTHORIZATION_FAILED ) + m_mgr->forceShutdown( RdKafka::err2str( msg->err() ) + " error: " + msg->errstr() ); continue; } - auto & topicData = topicIt -> second; + auto & topicData = topicIt->second; - if( msg -> err() == RdKafka::ERR_NO_ERROR && msg -> len() ) + if( msg->err() == RdKafka::ERR_NO_ERROR && msg->len() ) { - if( !msg -> key() ) + if( !msg->key() ) { - std::string errmsg = "KafkaConsumer: Message received with null key on topic " + msg -> topic_name() + "."; - m_mgr -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); + std::string errmsg + = "KafkaConsumer: Message received with null key on topic " + msg->topic_name() + "."; + m_mgr->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); continue; } - //printf( "Msg %s:%s on %d\n", msg -> topic_name().c_str(), msg -> key() -> c_str(), msg -> partition() ); - auto subscribersIt = topicData.subscribers.find( *msg -> key() ); + // printf( "Msg %s:%s on %d\n", msg -> topic_name().c_str(), msg -> key() -> c_str(), msg -> partition() + // ); + auto subscribersIt = topicData.subscribers.find( *msg->key() ); if( subscribersIt == topicData.subscribers.end() && topicData.wildcardSubscriber == nullptr ) continue; auto & partitionLive = topicData.partitionLive; - if( ( uint32_t ) msg -> partition() >= partitionLive.size() ) - partitionLive.resize( msg -> partition() + 1, false ); + if( (uint32_t)msg->partition() >= partitionLive.size() ) + partitionLive.resize( msg->partition() + 1, false ); - bool live = partitionLive[ msg -> partition() ]; + bool live = partitionLive[msg->partition()]; if( subscribersIt != topicData.subscribers.end() ) { - for( auto it : subscribersIt -> second ) - it -> onMessage( msg.get(), live ); + for( auto it : subscribersIt->second ) + it->onMessage( msg.get(), live ); } - //Note we always have to tick wildcard as live because it can get messages from multiple - //partitions, some which may have done replaying and some not ( not to mention that data can be out of order ) + // Note we always have to tick wildcard as live because it can get messages from multiple + // partitions, some which may have done replaying and some not ( not to mention that data can be out of + // order ) if( topicData.wildcardSubscriber ) - topicData.wildcardSubscriber -> onMessage( msg.get(), true ); + topicData.wildcardSubscriber->onMessage( msg.get(), true ); } - //Not sure why, but it looks like we repeatedly get EOF callbacks even after the original one - //may want to look into this. Not an issue in practice, but seems like unnecessary overhead - else if( msg -> err() == RdKafka::ERR__PARTITION_EOF ) + // Not sure why, but it looks like we repeatedly get EOF callbacks even after the original one + // may want to look into this. Not an issue in practice, but seems like unnecessary overhead + else if( msg->err() == RdKafka::ERR__PARTITION_EOF ) { - auto const partition = msg -> partition(); - auto & partitionLive = topicData.partitionLive; - if( ( uint32_t ) partition >= partitionLive.size() ) + auto const partition = msg->partition(); + auto & partitionLive = topicData.partitionLive; + if( (uint32_t)partition >= partitionLive.size() ) partitionLive.resize( partition + 1, false ); - partitionLive[ partition ] = true; + partitionLive[partition] = true; - //need this gaurd since we get this repeatedly after initial EOF + // need this gaurd since we get this repeatedly after initial EOF if( !topicData.flaggedReplayComplete ) { bool allDone = true; @@ -269,27 +286,29 @@ void KafkaConsumer::poll() } } - //we need to flag end in case the topic doesnt have any incoming data, we cant stall the engine on the pull side of the adapter + // we need to flag end in case the topic doesnt have any incoming data, we cant stall the engine on + // the pull side of the adapter if( allDone ) topicData.markReplayComplete(); } } else { - //In most cases we should not get here, if we do then something is wrong - //safest bet is to release the pull adapter so it doesnt stall the engine and - //we can let the error msg through + // In most cases we should not get here, if we do then something is wrong + // safest bet is to release the pull adapter so it doesnt stall the engine and + // we can let the error msg through topicData.markReplayComplete(); - std::string errmsg = "KafkaConsumer: Message error on topic \"" + msg -> topic_name() + "\". errcode: " + RdKafka::err2str( msg -> err() ) + " error: " + msg -> errstr(); - m_mgr -> pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); + std::string errmsg = "KafkaConsumer: Message error on topic \"" + msg->topic_name() + + "\". errcode: " + RdKafka::err2str( msg->err() ) + " error: " + msg->errstr(); + m_mgr->pushStatus( StatusLevel::ERROR, KafkaStatusMessageType::MSG_RECV_ERROR, errmsg ); } } } catch( const Exception & err ) { - m_mgr -> rootEngine() -> shutdown( std::current_exception() ); + m_mgr->rootEngine()->shutdown( std::current_exception() ); } } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaConsumer.h b/cpp/csp/adapters/kafka/KafkaConsumer.h index 50cdf4e37..cb29b0b84 100644 --- a/cpp/csp/adapters/kafka/KafkaConsumer.h +++ b/cpp/csp/adapters/kafka/KafkaConsumer.h @@ -30,44 +30,43 @@ class KafkaConsumer void forceReplayCompleted(); private: - struct TopicData { - //Key -> Subscriber - using SubscriberMap = std::unordered_map>; - SubscriberMap subscribers; - KafkaSubscriber * wildcardSubscriber = nullptr; - std::vector partitionLive; - bool flaggedReplayComplete = false; + // Key -> Subscriber + using SubscriberMap = std::unordered_map>; + SubscriberMap subscribers; + KafkaSubscriber * wildcardSubscriber = nullptr; + std::vector partitionLive; + bool flaggedReplayComplete = false; void markReplayComplete() { if( !flaggedReplayComplete ) { // Flag all regular subscribers - for( auto& subscriberEntry : subscribers ) + for( auto & subscriberEntry : subscribers ) { - for( auto* subscriber : subscriberEntry.second ) - subscriber -> flagReplayComplete(); + for( auto * subscriber : subscriberEntry.second ) + subscriber->flagReplayComplete(); } - + // Handle wildcard subscriber if present if( wildcardSubscriber ) - wildcardSubscriber -> flagReplayComplete(); - + wildcardSubscriber->flagReplayComplete(); + flaggedReplayComplete = true; } } }; - std::unordered_map m_topics; - KafkaAdapterManager * m_mgr; - std::unique_ptr m_consumer; - std::unique_ptr m_rebalanceCb; - std::unique_ptr m_pollThread; - volatile bool m_running; + std::unordered_map m_topics; + KafkaAdapterManager * m_mgr; + std::unique_ptr m_consumer; + std::unique_ptr m_rebalanceCb; + std::unique_ptr m_pollThread; + volatile bool m_running; }; -} +} // namespace csp::adapters::kafka #endif diff --git a/cpp/csp/adapters/kafka/KafkaInputAdapter.cpp b/cpp/csp/adapters/kafka/KafkaInputAdapter.cpp index 296df8daa..16f0e628b 100644 --- a/cpp/csp/adapters/kafka/KafkaInputAdapter.cpp +++ b/cpp/csp/adapters/kafka/KafkaInputAdapter.cpp @@ -5,124 +5,144 @@ namespace csp::adapters::kafka { -KafkaInputAdapter::KafkaInputAdapter( Engine *engine, CspTypePtr &type, - PushMode pushMode, PushGroup *group, - const Dictionary &properties) - : PushPullInputAdapter( engine, type, pushMode, group, - properties.get( "adjust_out_of_order_time") ), - m_includeMsgBeforeStartTime( properties.get( "include_msg_before_start_time", false ) ) +KafkaInputAdapter::KafkaInputAdapter( Engine * engine, CspTypePtr & type, PushMode pushMode, PushGroup * group, + const Dictionary & properties ) + : PushPullInputAdapter( engine, type, pushMode, group, properties.get( "adjust_out_of_order_time" ) ) + , m_includeMsgBeforeStartTime( properties.get( "include_msg_before_start_time", false ) ) { - if( type -> type() != CspType::Type::STRUCT && - type -> type() != CspType::Type::STRING ) - CSP_THROW( RuntimeException, "Unsupported type: " << type -> type() ); - + if( type->type() != CspType::Type::STRUCT && type->type() != CspType::Type::STRING ) + CSP_THROW( RuntimeException, "Unsupported type: " << type->type() ); if( properties.exists( "meta_field_map" ) ) { - const CspStructType & structType = static_cast( *type ); - const Dictionary & metaFieldMap = *properties.get( "meta_field_map" ); + const CspStructType & structType = static_cast( *type ); + const Dictionary & metaFieldMap = *properties.get( "meta_field_map" ); - if( !metaFieldMap.empty() && type -> type() != CspType::Type::STRUCT ) + if( !metaFieldMap.empty() && type->type() != CspType::Type::STRUCT ) CSP_THROW( ValueError, "meta_field_map is not supported on non-struct types" ); if( metaFieldMap.exists( "partition" ) ) { std::string partitionFieldName = metaFieldMap.get( "partition" ); - m_partitionField = structType.meta() -> field( partitionFieldName ); + m_partitionField = structType.meta()->field( partitionFieldName ); if( !m_partitionField ) - CSP_THROW( ValueError, "field " << partitionFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_partitionField -> type() -> type() != CspType::Type::INT64 ) - CSP_THROW( ValueError, "field " << partitionFieldName << " must be of type int on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << partitionFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_partitionField->type()->type() != CspType::Type::INT64 ) + CSP_THROW( ValueError, + "field " << partitionFieldName << " must be of type int on struct type " + << structType.meta()->name() ); } if( metaFieldMap.exists( "offset" ) ) { std::string offsetFieldName = metaFieldMap.get( "offset" ); - m_offsetField = structType.meta() -> field( offsetFieldName ); + m_offsetField = structType.meta()->field( offsetFieldName ); if( !m_offsetField ) - CSP_THROW( ValueError, "field " << offsetFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_offsetField -> type() -> type() != CspType::Type::INT64 ) - CSP_THROW( ValueError, "field " << offsetFieldName << " must be of type int on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << offsetFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_offsetField->type()->type() != CspType::Type::INT64 ) + CSP_THROW( ValueError, + "field " << offsetFieldName << " must be of type int on struct type " + << structType.meta()->name() ); } if( metaFieldMap.exists( "live" ) ) { std::string liveFieldName = metaFieldMap.get( "live" ); - m_liveField = structType.meta() -> field( liveFieldName ); + m_liveField = structType.meta()->field( liveFieldName ); if( !m_liveField ) - CSP_THROW( ValueError, "field " << liveFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_liveField -> type() -> type() != CspType::Type::BOOL ) - CSP_THROW( ValueError, "field " << liveFieldName << " must be of type bool on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << liveFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_liveField->type()->type() != CspType::Type::BOOL ) + CSP_THROW( ValueError, + "field " << liveFieldName << " must be of type bool on struct type " + << structType.meta()->name() ); } if( metaFieldMap.exists( "timestamp" ) ) { std::string timestampFieldName = metaFieldMap.get( "timestamp" ); - m_timestampField = structType.meta() -> field( timestampFieldName ); + m_timestampField = structType.meta()->field( timestampFieldName ); if( !m_timestampField ) - CSP_THROW( ValueError, "field " << timestampFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_timestampField -> type() -> type() != CspType::Type::DATETIME ) - CSP_THROW( ValueError, "field " << timestampFieldName << " must be of type datetime on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << timestampFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_timestampField->type()->type() != CspType::Type::DATETIME ) + CSP_THROW( ValueError, + "field " << timestampFieldName << " must be of type datetime on struct type " + << structType.meta()->name() ); } if( metaFieldMap.exists( "key" ) ) { std::string keyFieldName = metaFieldMap.get( "key" ); - m_keyField = structType.meta() -> field( keyFieldName ); + m_keyField = structType.meta()->field( keyFieldName ); if( !m_keyField ) - CSP_THROW( ValueError, "field " << keyFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_keyField -> type() -> type() != CspType::Type::STRING ) - CSP_THROW( ValueError, "field " << keyFieldName << " must be of type string on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << keyFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_keyField->type()->type() != CspType::Type::STRING ) + CSP_THROW( ValueError, + "field " << keyFieldName << " must be of type string on struct type " + << structType.meta()->name() ); } if( properties.exists( "tick_timestamp_from_field" ) ) { - std::string timestampFieldName = properties.get("tick_timestamp_from_field"); - m_tickTimestampField = structType.meta() -> field( timestampFieldName ); + std::string timestampFieldName = properties.get( "tick_timestamp_from_field" ); + m_tickTimestampField = structType.meta()->field( timestampFieldName ); if( !m_tickTimestampField ) - CSP_THROW( ValueError, "field " << timestampFieldName << " is not a valid field on struct type " << structType.meta() -> name() ); - if( m_tickTimestampField -> type() -> type() != CspType::Type::DATETIME ) - CSP_THROW( ValueError, "field " << timestampFieldName << " must be of type datetime on struct type " << structType.meta() -> name() ); + CSP_THROW( ValueError, + "field " << timestampFieldName << " is not a valid field on struct type " + << structType.meta()->name() ); + if( m_tickTimestampField->type()->type() != CspType::Type::DATETIME ) + CSP_THROW( ValueError, + "field " << timestampFieldName << " must be of type datetime on struct type " + << structType.meta()->name() ); } } m_converter = utils::MessageStructConverterCache::instance().create( type, properties ); } -void KafkaInputAdapter::processMessage( RdKafka::Message* message, bool live, csp::PushBatch* batch ) +void KafkaInputAdapter::processMessage( RdKafka::Message * message, bool live, csp::PushBatch * batch ) { DateTime msgTime; - auto ts = message -> timestamp(); + auto ts = message->timestamp(); if( ts.type != RdKafka::MessageTimestamp::MSG_TIMESTAMP_NOT_AVAILABLE ) msgTime = DateTime::fromMilliseconds( ts.timestamp ); - if( type() -> type() == CspType::Type::STRUCT ) + if( type()->type() == CspType::Type::STRUCT ) { - auto tick = m_converter -> asStruct( message -> payload(), message -> len() ); + auto tick = m_converter->asStruct( message->payload(), message->len() ); if( m_partitionField ) - m_partitionField -> setValue( tick.get(), message -> partition() ); + m_partitionField->setValue( tick.get(), message->partition() ); if( m_offsetField ) - m_offsetField -> setValue( tick.get(), message -> offset() ); + m_offsetField->setValue( tick.get(), message->offset() ); if( m_liveField ) - m_liveField -> setValue( tick.get(), live ); + m_liveField->setValue( tick.get(), live ); if( m_timestampField && !msgTime.isNone() ) - m_timestampField -> setValue( tick.get(), msgTime ); + m_timestampField->setValue( tick.get(), msgTime ); if( m_keyField ) - m_keyField -> setValue( tick.get(), *message -> key() ); - + m_keyField->setValue( tick.get(), *message->key() ); + if( m_tickTimestampField ) - msgTime = m_tickTimestampField->value(tick.get()); + msgTime = m_tickTimestampField->value( tick.get() ); - bool pushLive = shouldPushLive(live, msgTime); + bool pushLive = shouldPushLive( live, msgTime ); if( shouldProcessMessage( pushLive, msgTime ) ) - pushTick(pushLive, msgTime, std::move(tick), batch); + pushTick( pushLive, msgTime, std::move( tick ), batch ); } - else if( type() -> type() == CspType::Type::STRING ) + else if( type()->type() == CspType::Type::STRING ) { - bool pushLive = shouldPushLive(live, msgTime); + bool pushLive = shouldPushLive( live, msgTime ); if( shouldProcessMessage( pushLive, msgTime ) ) - pushTick( pushLive, msgTime, std::string( ( const char * ) message -> payload(), message -> len() ) ); + pushTick( pushLive, msgTime, std::string( (const char *)message->payload(), message->len() ) ); } } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaInputAdapter.h b/cpp/csp/adapters/kafka/KafkaInputAdapter.h index 851498654..5359a0124 100644 --- a/cpp/csp/adapters/kafka/KafkaInputAdapter.h +++ b/cpp/csp/adapters/kafka/KafkaInputAdapter.h @@ -9,14 +9,13 @@ namespace csp::adapters::kafka { -class KafkaInputAdapter final: public PushPullInputAdapter +class KafkaInputAdapter final : public PushPullInputAdapter { public: - KafkaInputAdapter( Engine * engine, CspTypePtr & type, - PushMode pushMode, PushGroup * group, + KafkaInputAdapter( Engine * engine, CspTypePtr & type, PushMode pushMode, PushGroup * group, const Dictionary & properties ); - void processMessage( RdKafka::Message* message, bool live, csp::PushBatch* batch ); + void processMessage( RdKafka::Message * message, bool live, csp::PushBatch * batch ); private: inline bool shouldPushLive( bool pushLive, DateTime msgTime ) @@ -29,21 +28,20 @@ class KafkaInputAdapter final: public PushPullInputAdapter // This function encapsulates the logic for determining if a message should be processed // live always goes through, otherwise, we filter out // messages before engine starttime when not m_includeMsgBeforeStartTime - return pushLive || m_includeMsgBeforeStartTime || - msgTime >= rootEngine()->startTime(); + return pushLive || m_includeMsgBeforeStartTime || msgTime >= rootEngine()->startTime(); } utils::MessageStructConverterPtr m_converter; - StructFieldPtr m_partitionField; - StructFieldPtr m_offsetField; - StructFieldPtr m_liveField; - StructFieldPtr m_timestampField; - StructFieldPtr m_keyField; - StructFieldPtr m_tickTimestampField; + StructFieldPtr m_partitionField; + StructFieldPtr m_offsetField; + StructFieldPtr m_liveField; + StructFieldPtr m_timestampField; + StructFieldPtr m_keyField; + StructFieldPtr m_tickTimestampField; bool m_includeMsgBeforeStartTime; }; -} +} // namespace csp::adapters::kafka #endif diff --git a/cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp b/cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp index 4f453d468..ee1f4747a 100644 --- a/cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp +++ b/cpp/csp/adapters/kafka/KafkaOutputAdapter.cpp @@ -4,54 +4,55 @@ namespace csp::adapters::kafka { -KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, - CspTypePtr & type, const Dictionary & properties ) : OutputAdapter( engine ), - m_publisher( publisher ) +KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, + const Dictionary & properties ) + : OutputAdapter( engine ) + , m_publisher( publisher ) { if( !publisher.isRawBytes() ) - m_dataMapper = utils::OutputDataMapperCache::instance().create( type, *properties.get( "field_map" ) ); - else if( type -> type() != CspType::Type::STRING ) - CSP_THROW( TypeError, "RAW_BYTES output expected ts[str] got ts type " << type -> type() ); + m_dataMapper + = utils::OutputDataMapperCache::instance().create( type, *properties.get( "field_map" ) ); + else if( type->type() != CspType::Type::STRING ) + CSP_THROW( TypeError, "RAW_BYTES output expected ts[str] got ts type " << type->type() ); } -KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, - CspTypePtr & type, const Dictionary & properties, - const std::string & key ) : KafkaOutputAdapter( engine, publisher, type, properties ) +KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, + const Dictionary & properties, const std::string & key ) + : KafkaOutputAdapter( engine, publisher, type, properties ) { m_publisher.setKey( key ); } -KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, - CspTypePtr &type, const Dictionary & properties, - const std::vector & keyFields ) : KafkaOutputAdapter( engine, publisher, type, properties ) +KafkaOutputAdapter::KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, + const Dictionary & properties, const std::vector & keyFields ) + : KafkaOutputAdapter( engine, publisher, type, properties ) { addFields( keyFields, type ); } -KafkaOutputAdapter::~KafkaOutputAdapter() -{ -} +KafkaOutputAdapter::~KafkaOutputAdapter() {} void KafkaOutputAdapter::addFields( const std::vector & keyFields, CspTypePtr & type, size_t i ) { - std::string fieldName = keyFields[ i ]; - auto structMeta = static_cast( *type ).meta(); - auto structField = structMeta -> field( fieldName ); + std::string fieldName = keyFields[i]; + auto structMeta = static_cast( *type ).meta(); + auto structField = structMeta->field( fieldName ); if( !structField ) - CSP_THROW( InvalidArgument, "Struct type " << structMeta -> name() << " missing required field " << fieldName ); + CSP_THROW( InvalidArgument, "Struct type " << structMeta->name() << " missing required field " << fieldName ); if( i == keyFields.size() - 1 ) { - CSP_TRUE_OR_THROW_RUNTIME( structField -> type() -> type() == CspType::Type::STRING, - "Key field must be of type string, got " << structField -> type() -> type().asString() ); + CSP_TRUE_OR_THROW_RUNTIME( structField->type()->type() == CspType::Type::STRING, + "Key field must be of type string, got " << structField->type()->type().asString() ); m_structFields.emplace_back( structField ); } else { - CSP_TRUE_OR_THROW_RUNTIME( structField -> type() -> type() == CspType::Type::STRUCT, - "Non-key field must be of type struct, got " << structField -> type() -> type().asString() ); + CSP_TRUE_OR_THROW_RUNTIME( structField->type()->type() == CspType::Type::STRUCT, + "Non-key field must be of type struct, got " + << structField->type()->type().asString() ); m_structFields.emplace_back( structField ); - auto t = structField -> type(); + auto t = structField->type(); addFields( keyFields, t, i + 1 ); } } @@ -59,28 +60,28 @@ void KafkaOutputAdapter::addFields( const std::vector & keyFields, const std::string & KafkaOutputAdapter::getKey( const Struct * struct_ ) { for( size_t i = 0; i < m_structFields.size() - 1; ++i ) - struct_ = m_structFields[ i ] -> value( struct_ ).get(); + struct_ = m_structFields[i]->value( struct_ ).get(); - return m_structFields[ m_structFields.size() - 1 ] -> value( struct_ ); + return m_structFields[m_structFields.size() - 1]->value( struct_ ); } void KafkaOutputAdapter::executeImpl() { if( m_publisher.isRawBytes() ) { - const std::string & value = input() -> lastValueTyped(); + const std::string & value = input()->lastValueTyped(); m_publisher.send( value.c_str(), value.length() ); } else { auto & msgWriter = m_publisher.msgWriter(); msgWriter.processTick( *m_dataMapper, input() ); - + if( !m_structFields.empty() ) - m_publisher.setKey( getKey( input() -> lastValueTyped().get() ) ); - + m_publisher.setKey( getKey( input()->lastValueTyped().get() ) ); + m_publisher.scheduleEndCycleEvent(); } } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaOutputAdapter.h b/cpp/csp/adapters/kafka/KafkaOutputAdapter.h index 79bf66036..fd31827de 100644 --- a/cpp/csp/adapters/kafka/KafkaOutputAdapter.h +++ b/cpp/csp/adapters/kafka/KafkaOutputAdapter.h @@ -11,11 +11,13 @@ namespace csp::adapters::kafka class KafkaPublisher; -class KafkaOutputAdapter final: public OutputAdapter +class KafkaOutputAdapter final : public OutputAdapter { public: - KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, const Dictionary & properties, const std::string & key ); - KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, const Dictionary & properties, const std::vector & keyFields ); + KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, const Dictionary & properties, + const std::string & key ); + KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, const Dictionary & properties, + const std::vector & keyFields ); ~KafkaOutputAdapter(); void executeImpl() override; @@ -24,7 +26,7 @@ class KafkaOutputAdapter final: public OutputAdapter private: KafkaOutputAdapter( Engine * engine, KafkaPublisher & publisher, CspTypePtr & type, const Dictionary & properties ); - void addFields( const std::vector & keyFields, CspTypePtr & type, size_t i = 0 ); + void addFields( const std::vector & keyFields, CspTypePtr & type, size_t i = 0 ); const std::string & getKey( const Struct * struct_ ); KafkaPublisher & m_publisher; @@ -32,6 +34,6 @@ class KafkaOutputAdapter final: public OutputAdapter std::vector m_structFields; }; -} +} // namespace csp::adapters::kafka #endif diff --git a/cpp/csp/adapters/kafka/KafkaPublisher.cpp b/cpp/csp/adapters/kafka/KafkaPublisher.cpp index 7f1cdfa17..39084d4ae 100644 --- a/cpp/csp/adapters/kafka/KafkaPublisher.cpp +++ b/cpp/csp/adapters/kafka/KafkaPublisher.cpp @@ -1,49 +1,53 @@ -#include #include #include -#include #include +#include +#include #include namespace csp::adapters::kafka { -KafkaPublisher::KafkaPublisher( KafkaAdapterManager * mgr, const Dictionary & properties, std::string topic ) : m_adapterMgr( *mgr ), - m_engine( mgr -> engine() ), - m_topic( std::move( topic ) ) +KafkaPublisher::KafkaPublisher( KafkaAdapterManager * mgr, const Dictionary & properties, std::string topic ) + : m_adapterMgr( *mgr ) + , m_engine( mgr->engine() ) + , m_topic( std::move( topic ) ) { auto protocol = properties.get( "protocol" ); if( protocol == "JSON" ) m_msgWriter = std::make_shared( properties ); else if( protocol != "RAW_BYTES" ) - CSP_THROW( NotImplemented, "msg protocol " << protocol << " not currently supported for kafka output adapters" ); + CSP_THROW( NotImplemented, + "msg protocol " << protocol << " not currently supported for kafka output adapters" ); } -KafkaPublisher::~KafkaPublisher() -{ -} +KafkaPublisher::~KafkaPublisher() {} -OutputAdapter * KafkaPublisher::getOutputAdapter( CspTypePtr & type, const Dictionary & properties, const std::string & key ) +OutputAdapter * KafkaPublisher::getOutputAdapter( CspTypePtr & type, const Dictionary & properties, + const std::string & key ) { if( isRawBytes() && !m_adapters.empty() ) - CSP_THROW( RuntimeException, "Attempting to publish multiple timeseries to kafka key " << key << " with RAW_BYTES protocol. Only one output per key is allowed" ); + CSP_THROW( RuntimeException, + "Attempting to publish multiple timeseries to kafka key " + << key << " with RAW_BYTES protocol. Only one output per key is allowed" ); - //create and register adapter - auto adapter = m_engine -> createOwnedObject( *this, type, properties, key ); + // create and register adapter + auto adapter = m_engine->createOwnedObject( *this, type, properties, key ); m_adapters.emplace_back( adapter ); - return m_adapters[ m_adapters.size() - 1 ]; + return m_adapters[m_adapters.size() - 1]; } -OutputAdapter * KafkaPublisher::getOutputAdapter( CspTypePtr & type, const Dictionary & properties, const std::vector & keyFields ) +OutputAdapter * KafkaPublisher::getOutputAdapter( CspTypePtr & type, const Dictionary & properties, + const std::vector & keyFields ) { if( isRawBytes() ) CSP_THROW( RuntimeException, "vector of key fields is unsupported for RAW_BYTES protocol" ); - auto adapter = m_engine -> createOwnedObject( *this, type, properties, keyFields ); + auto adapter = m_engine->createOwnedObject( *this, type, properties, keyFields ); m_adapters.emplace_back( adapter ); - return m_adapters[ m_adapters.size() - 1 ]; + return m_adapters[m_adapters.size() - 1]; } PushInputAdapter * KafkaPublisher::getStatusAdapter() @@ -58,24 +62,26 @@ void KafkaPublisher::start( std::shared_ptr producer ) std::unique_ptr tconf( RdKafka::Conf::create( RdKafka::Conf::CONF_TOPIC ) ); std::string errstr; - m_kafkaTopic = std::shared_ptr( RdKafka::Topic::create( m_producer.get(), m_topic, tconf.get(), errstr ) ); + m_kafkaTopic + = std::shared_ptr( RdKafka::Topic::create( m_producer.get(), m_topic, tconf.get(), errstr ) ); if( !m_kafkaTopic ) - CSP_THROW( RuntimeException, "Failed to create RdKafka::Topic for producer on topic " << m_topic << ":" << errstr ); + CSP_THROW( RuntimeException, + "Failed to create RdKafka::Topic for producer on topic " << m_topic << ":" << errstr ); } void KafkaPublisher::stop() { - //Note its important to release the topic here because its underlying mem lives in the kafka context of the Producer - //if Producer is destroyed first, when we try to destroy the topic its a memory error + // Note its important to release the topic here because its underlying mem lives in the kafka context of the + // Producer if Producer is destroyed first, when we try to destroy the topic its a memory error m_kafkaTopic.reset(); } void KafkaPublisher::send( const void * data, size_t len ) { RdKafka::ErrorCode err; - err = m_producer -> produce( m_kafkaTopic.get(), RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY, - const_cast( data ), len, m_key.c_str(), m_key.length(), nullptr ); - + err = m_producer->produce( m_kafkaTopic.get(), RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY, + const_cast( data ), len, m_key.c_str(), m_key.length(), nullptr ); + if( err != RdKafka::ERR_NO_ERROR ) { std::string errMsg( "KafkaPublisher Error sending message " ); @@ -86,8 +92,8 @@ void KafkaPublisher::send( const void * data, size_t len ) void KafkaPublisher::onEndCycle() { - auto [data,len] = m_msgWriter -> finalize(); + auto [data, len] = m_msgWriter->finalize(); send( data, len ); } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaPublisher.h b/cpp/csp/adapters/kafka/KafkaPublisher.h index e48a1581d..89e0ae604 100644 --- a/cpp/csp/adapters/kafka/KafkaPublisher.h +++ b/cpp/csp/adapters/kafka/KafkaPublisher.h @@ -12,7 +12,7 @@ class DeliveryReportCb; class Producer; class Topic; -} +} // namespace RdKafka namespace csp::adapters::utils { @@ -31,7 +31,8 @@ class KafkaPublisher : public EndCycleListener ~KafkaPublisher(); OutputAdapter * getOutputAdapter( CspTypePtr & type, const Dictionary & properties, const std::string & key ); - OutputAdapter * getOutputAdapter( CspTypePtr & type, const Dictionary & properties, const std::vector & keyFields ); + OutputAdapter * getOutputAdapter( CspTypePtr & type, const Dictionary & properties, + const std::vector & keyFields ); PushInputAdapter * getStatusAdapter(); @@ -42,21 +43,17 @@ class KafkaPublisher : public EndCycleListener void onEndCycle() final; - bool isRawBytes() const { return ( bool ) !m_msgWriter; } + bool isRawBytes() const { return (bool)!m_msgWriter; } utils::MessageWriter & msgWriter() { return *m_msgWriter; } - void scheduleEndCycleEvent() - { - m_adapterMgr.rootEngine() -> scheduleEndCycleListener( this ); - } + void scheduleEndCycleEvent() { m_adapterMgr.rootEngine()->scheduleEndCycleListener( this ); } void send( const void * data, size_t len ); private: - - using Adapters = std::vector; - Adapters m_adapters; + using Adapters = std::vector; + Adapters m_adapters; KafkaAdapterManager & m_adapterMgr; Engine * m_engine; @@ -67,6 +64,6 @@ class KafkaPublisher : public EndCycleListener std::string m_key; }; -} +} // namespace csp::adapters::kafka #endif diff --git a/cpp/csp/adapters/kafka/KafkaSubscriber.cpp b/cpp/csp/adapters/kafka/KafkaSubscriber.cpp index 75734ea3c..092b84482 100644 --- a/cpp/csp/adapters/kafka/KafkaSubscriber.cpp +++ b/cpp/csp/adapters/kafka/KafkaSubscriber.cpp @@ -1,38 +1,38 @@ -#include #include +#include #include #include namespace csp::adapters::kafka { -KafkaSubscriber::KafkaSubscriber( KafkaAdapterManager * mgr, const Dictionary & properties ) : m_adapterMgr( *mgr ), - m_engine( mgr -> engine() ) +KafkaSubscriber::KafkaSubscriber( KafkaAdapterManager * mgr, const Dictionary & properties ) + : m_adapterMgr( *mgr ) + , m_engine( mgr->engine() ) { } -KafkaSubscriber::~KafkaSubscriber() -{ -} +KafkaSubscriber::~KafkaSubscriber() {} -PushInputAdapter * KafkaSubscriber::getInputAdapter( CspTypePtr & type, PushMode pushMode, const Dictionary & properties ) +PushInputAdapter * KafkaSubscriber::getInputAdapter( CspTypePtr & type, PushMode pushMode, + const Dictionary & properties ) { - //create and register adapter - auto adapter = m_engine -> createOwnedObject( type, pushMode, &m_pushGroup, properties ); + // create and register adapter + auto adapter = m_engine->createOwnedObject( type, pushMode, &m_pushGroup, properties ); m_adapters.emplace_back( adapter ); return adapter; } -void KafkaSubscriber::onMessage( RdKafka::Message* message, bool live ) +void KafkaSubscriber::onMessage( RdKafka::Message * message, bool live ) { - csp::PushBatch batch( m_engine -> rootEngine() ); + csp::PushBatch batch( m_engine->rootEngine() ); for( auto & adapter : m_adapters ) { try { - adapter -> processMessage( message, live, &batch ); + adapter->processMessage( message, live, &batch ); } catch( csp::Exception & err ) { @@ -44,7 +44,7 @@ void KafkaSubscriber::onMessage( RdKafka::Message* message, bool live ) void KafkaSubscriber::flagReplayComplete() { for( auto & adapter : m_adapters ) - adapter -> flagReplayComplete(); + adapter->flagReplayComplete(); } -} +} // namespace csp::adapters::kafka diff --git a/cpp/csp/adapters/kafka/KafkaSubscriber.h b/cpp/csp/adapters/kafka/KafkaSubscriber.h index 4f7d3c829..49a77b5d9 100644 --- a/cpp/csp/adapters/kafka/KafkaSubscriber.h +++ b/cpp/csp/adapters/kafka/KafkaSubscriber.h @@ -18,21 +18,18 @@ class KafkaSubscriber PushInputAdapter * getInputAdapter( CspTypePtr & type, PushMode pushMode, const Dictionary & properties ); - void onMessage( RdKafka::Message* message, bool live ); + void onMessage( RdKafka::Message * message, bool live ); void flagReplayComplete(); private: using Adapters = std::vector; - Adapters m_adapters; + Adapters m_adapters; KafkaAdapterManager & m_adapterMgr; Engine * m_engine; PushGroup m_pushGroup; }; - - - -} +} // namespace csp::adapters::kafka #endif diff --git a/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.cpp b/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.cpp index 382f13ac4..a35fea379 100644 --- a/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.cpp +++ b/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.cpp @@ -1,7 +1,7 @@ -#include -#include #include #include +#include +#include #include namespace csp::adapters::parquet @@ -11,16 +11,15 @@ ArrowIPCFileReaderWrapper::~ArrowIPCFileReaderWrapper() close(); } -void ArrowIPCFileReaderWrapper::open( const std::string &fileName ) +void ArrowIPCFileReaderWrapper::open( const std::string & fileName ) { FileReaderWrapper::open( fileName ); try { - PARQUET_ASSIGN_OR_THROW( m_fileReader, - ::arrow::ipc::RecordBatchStreamReader::Open( m_inputFile ) ); + PARQUET_ASSIGN_OR_THROW( m_fileReader, ::arrow::ipc::RecordBatchStreamReader::Open( m_inputFile ) ); } - catch( std::exception &e ) + catch( std::exception & e ) { FileReaderWrapper::close(); CSP_THROW( RuntimeException, "Failed to open " << fileName << ":" << e.what() ); @@ -33,10 +32,11 @@ void ArrowIPCFileReaderWrapper::close() FileReaderWrapper::close(); } -bool ArrowIPCFileReaderWrapper::readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> &dst ) +bool ArrowIPCFileReaderWrapper::readNextRowGroup( const std::vector neededColumns, + std::shared_ptr<::arrow::Table> & dst ) { std::shared_ptr recordBatch; - STATUS_OK_OR_THROW_RUNTIME( m_fileReader -> ReadNext( &recordBatch ), "Failed to read next record batch" ); + STATUS_OK_OR_THROW_RUNTIME( m_fileReader->ReadNext( &recordBatch ), "Failed to read next record batch" ); if( recordBatch == nullptr ) { dst = nullptr; @@ -44,22 +44,21 @@ bool ArrowIPCFileReaderWrapper::readNextRowGroup( const std::vector neededC } std::vector> tableColumns; std::vector> tableColumnSpecs; - tableColumns.reserve(neededColumns.size()); - tableColumnSpecs.reserve(neededColumns.size()); + tableColumns.reserve( neededColumns.size() ); + tableColumnSpecs.reserve( neededColumns.size() ); - for( auto &&columnIndex : neededColumns ) + for( auto && columnIndex : neededColumns ) { - tableColumns.push_back(recordBatch->column(columnIndex)); - tableColumnSpecs.push_back(recordBatch->schema()->field(columnIndex)); + tableColumns.push_back( recordBatch->column( columnIndex ) ); + tableColumnSpecs.push_back( recordBatch->schema()->field( columnIndex ) ); } dst = arrow::Table::Make( ::arrow::schema( tableColumnSpecs ), tableColumns ); return true; } - -void ArrowIPCFileReaderWrapper::getSchema( std::shared_ptr<::arrow::Schema> &dst ) +void ArrowIPCFileReaderWrapper::getSchema( std::shared_ptr<::arrow::Schema> & dst ) { - dst = m_fileReader -> schema(); + dst = m_fileReader->schema(); } -} \ No newline at end of file +} // namespace csp::adapters::parquet \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.h b/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.h index 7c95272e1..12fc8ab7a 100644 --- a/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.h +++ b/cpp/csp/adapters/parquet/ArrowIPCFileReaderWrapper.h @@ -1,8 +1,8 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_ArrowIPCFileReaderWrapper_H #define _IN_CSP_ADAPTERS_PARQUET_ArrowIPCFileReaderWrapper_H -#include #include +#include #include namespace csp::adapters::parquet @@ -11,16 +11,15 @@ class ArrowIPCFileReaderWrapper : public FileReaderWrapper { public: ~ArrowIPCFileReaderWrapper(); - virtual void open( const std::string &fileName ) override; + virtual void open( const std::string & fileName ) override; virtual void close() override; - bool readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> &dst ) override; - void getSchema( std::shared_ptr<::arrow::Schema> &dst ) override; + bool readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> & dst ) override; + void getSchema( std::shared_ptr<::arrow::Schema> & dst ) override; private: std::shared_ptr<::arrow::ipc::RecordBatchReader> m_fileReader; }; -} - +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp b/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp index daf23337e..d9a8d3aaf 100644 --- a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp +++ b/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp @@ -1,26 +1,24 @@ -#include -#include #include -#include #include +#include +#include +#include #include namespace csp::adapters::parquet { -void ArrowIPCFileWriterWrapper::openImpl( const std::string &fileName, const std::string &compression ) +void ArrowIPCFileWriterWrapper::openImpl( const std::string & fileName, const std::string & compression ) { CSP_TRUE_OR_THROW_RUNTIME( m_outputStream == nullptr, "Trying to open parquet file while previous was not closed" ); - PARQUET_ASSIGN_OR_THROW( - m_outputStream, - arrow::io::FileOutputStream::Open( fileName.c_str())); + PARQUET_ASSIGN_OR_THROW( m_outputStream, arrow::io::FileOutputStream::Open( fileName.c_str() ) ); arrow::ipc::IpcWriteOptions writeOptions; - writeOptions.codec = resolveCompressionCodec( compression ); + writeOptions.codec = resolveCompressionCodec( compression ); STATUS_OK_OR_THROW_RUNTIME( - ::arrow::ipc::MakeStreamWriter( m_outputStream.get(), getSchema(), writeOptions ).Value(&m_fileWriter), - "Failed to open arrow file writer" ); + ::arrow::ipc::MakeStreamWriter( m_outputStream.get(), getSchema(), writeOptions ).Value( &m_fileWriter ), + "Failed to open arrow file writer" ); } void ArrowIPCFileWriterWrapper::close() @@ -28,8 +26,8 @@ void ArrowIPCFileWriterWrapper::close() if( m_outputStream ) { // Let's move them first, if there are any exceptions, we still want the pointer to be null - std::shared_ptr<::arrow::io::FileOutputStream> outputStream{ std::move( m_outputStream ) }; - std::shared_ptr<::arrow::ipc::RecordBatchWriter> fileWriter{ std::move( m_fileWriter ) }; + std::shared_ptr<::arrow::io::FileOutputStream> outputStream{ std::move( m_outputStream ) }; + std::shared_ptr<::arrow::ipc::RecordBatchWriter> fileWriter{ std::move( m_fileWriter ) }; // Should be done by move constructor but let's be safe: m_outputStream = nullptr; m_fileWriter = nullptr; @@ -41,9 +39,10 @@ void ArrowIPCFileWriterWrapper::close() } } -void ArrowIPCFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> &table ) +void ArrowIPCFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> & table ) { - STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows()), "Failed to write to arrow file" ); + STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows() ), + "Failed to write to arrow file" ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.h b/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.h index 30d5e340d..1b359d288 100644 --- a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.h +++ b/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.h @@ -21,24 +21,23 @@ class ArrowIPCFileWriterWrapper : public FileWriterWrapper { public: ArrowIPCFileWriterWrapper( std::shared_ptr<::arrow::Schema> schema ) - : FileWriterWrapper( schema ) + : FileWriterWrapper( schema ) { } ~ArrowIPCFileWriterWrapper() { close(); } void close() override; - void writeTable( const std::shared_ptr<::arrow::Table> &table ) override; + void writeTable( const std::shared_ptr<::arrow::Table> & table ) override; protected: - void openImpl( const std::string &fileName, const std::string &compression ) override; + void openImpl( const std::string & fileName, const std::string & compression ) override; private: - std::shared_ptr<::arrow::io::FileOutputStream> m_outputStream; + std::shared_ptr<::arrow::io::FileOutputStream> m_outputStream; std::shared_ptr<::arrow::ipc::RecordBatchWriter> m_fileWriter; }; -} - +} // namespace csp::adapters::parquet #endif \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h index 136ef7027..8130348e5 100644 --- a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h +++ b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h @@ -1,13 +1,13 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_ArrowSingleColumnArrayBuilder_H #define _IN_CSP_ADAPTERS_PARQUET_ArrowSingleColumnArrayBuilder_H +#include +#include #include #include #include #include -#include #include -#include namespace csp::adapters::parquet { @@ -15,23 +15,24 @@ class ArrowSingleColumnArrayBuilder { public: ArrowSingleColumnArrayBuilder( std::string columnName, std::uint32_t chunkSize ) - : m_columnName( columnName ), m_chunkSize( chunkSize ) + : m_columnName( columnName ) + , m_chunkSize( chunkSize ) { } - ArrowSingleColumnArrayBuilder( const ArrowSingleColumnArrayBuilder &other ) = delete; - ArrowSingleColumnArrayBuilder &operator=( const ArrowSingleColumnArrayBuilder &other ) = delete; + ArrowSingleColumnArrayBuilder( const ArrowSingleColumnArrayBuilder & other ) = delete; + ArrowSingleColumnArrayBuilder & operator=( const ArrowSingleColumnArrayBuilder & other ) = delete; virtual ~ArrowSingleColumnArrayBuilder() {} - virtual std::shared_ptr getDataType() = 0; - virtual std::shared_ptr getBuilder() = 0; + virtual std::shared_ptr getDataType() = 0; + virtual std::shared_ptr getBuilder() = 0; virtual int64_t length() const = 0; - const std::string &getColumnName(){ return m_columnName; } + const std::string & getColumnName() { return m_columnName; } - std::uint32_t getChunkSize() const{ return m_chunkSize; } + std::uint32_t getChunkSize() const { return m_chunkSize; } // Called for each row of the output parquet row, if no value was provided a null should be appended virtual void handleRowFinished() = 0; @@ -50,32 +51,23 @@ class StructColumnArrayBuilder : public ArrowSingleColumnArrayBuilder using FieldValueSetter = std::function; StructColumnArrayBuilder( std::string columnName, std::uint32_t chunkSize, - const std::shared_ptr<::arrow::DataType> &type, - const std::vector &childArrayBuilders, - FieldValueSetter fieldValueSetter ) - : ArrowSingleColumnArrayBuilder( columnName, chunkSize ), - m_childArrayBuilders( childArrayBuilders ), - m_builderPtr( std::make_shared<::arrow::StructBuilder>( type, arrow::default_memory_pool(), - getArrowChildArrayBuilders( childArrayBuilders ) ) ), - m_fieldValueSetter( fieldValueSetter ), - m_hasValue( false ) + const std::shared_ptr<::arrow::DataType> & type, + const std::vector & childArrayBuilders, + FieldValueSetter fieldValueSetter ) + : ArrowSingleColumnArrayBuilder( columnName, chunkSize ) + , m_childArrayBuilders( childArrayBuilders ) + , m_builderPtr( std::make_shared<::arrow::StructBuilder>( type, arrow::default_memory_pool(), + getArrowChildArrayBuilders( childArrayBuilders ) ) ) + , m_fieldValueSetter( fieldValueSetter ) + , m_hasValue( false ) { } - virtual std::shared_ptr getDataType() override - { - return m_builderPtr -> type(); - } + virtual std::shared_ptr getDataType() override { return m_builderPtr->type(); } - virtual std::shared_ptr getBuilder() override - { - return m_builderPtr; - } + virtual std::shared_ptr getBuilder() override { return m_builderPtr; } - virtual int64_t length() const override - { - return m_builderPtr -> length(); - } + virtual int64_t length() const override { return m_builderPtr->length(); } // Called for each row of the output parquet row, if no value was provided a null should be appended virtual void handleRowFinished() override @@ -83,15 +75,15 @@ class StructColumnArrayBuilder : public ArrowSingleColumnArrayBuilder if( m_hasValue ) { m_hasValue = false; - for( auto &childBuilder : m_childArrayBuilders ) + for( auto & childBuilder : m_childArrayBuilders ) { - childBuilder -> handleRowFinished(); + childBuilder->handleRowFinished(); } - CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr -> Append().ok(), "Failed to append struct" ); + CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr->Append().ok(), "Failed to append struct" ); } else { - STATUS_OK_OR_THROW_RUNTIME( m_builderPtr -> AppendNull(), "Failed to create arrow array" ); + STATUS_OK_OR_THROW_RUNTIME( m_builderPtr->AppendNull(), "Failed to create arrow array" ); } } @@ -99,25 +91,25 @@ class StructColumnArrayBuilder : public ArrowSingleColumnArrayBuilder virtual std::shared_ptr buildArray() override { std::shared_ptr array; - STATUS_OK_OR_THROW_RUNTIME( m_builderPtr -> Finish( &array ), "Failed to create arrow array" ); + STATUS_OK_OR_THROW_RUNTIME( m_builderPtr->Finish( &array ), "Failed to create arrow array" ); return array; } - void setValue( const Struct *value ) + void setValue( const Struct * value ) { m_hasValue = true; m_fieldValueSetter( value ); } private: - static std::vector> getArrowChildArrayBuilders( - const std::vector &childArrayBuilders ) + static std::vector> + getArrowChildArrayBuilders( const std::vector & childArrayBuilders ) { std::vector> res; - for( auto &columnBuilder : childArrayBuilders ) + for( auto & columnBuilder : childArrayBuilders ) { - res.push_back( columnBuilder -> getBuilder() ); + res.push_back( columnBuilder->getBuilder() ); } return res; } @@ -129,50 +121,39 @@ class StructColumnArrayBuilder : public ArrowSingleColumnArrayBuilder bool m_hasValue; }; - class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder { public: using ColumnBuilderPtr = std::shared_ptr; ListColumnArrayBuilder( std::string columnName, std::uint32_t chunkSize, - const std::shared_ptr& valueBuilder, - const DialectGenericListWriterInterface::Ptr& listWriterInterface) - : ArrowSingleColumnArrayBuilder( columnName, chunkSize ), - m_valueBuilder( valueBuilder ), - m_builderPtr( std::make_shared<::arrow::ListBuilder>( arrow::default_memory_pool(), m_valueBuilder ) ), - m_listWriterInterface(listWriterInterface) + const std::shared_ptr & valueBuilder, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ) + : ArrowSingleColumnArrayBuilder( columnName, chunkSize ) + , m_valueBuilder( valueBuilder ) + , m_builderPtr( std::make_shared<::arrow::ListBuilder>( arrow::default_memory_pool(), m_valueBuilder ) ) + , m_listWriterInterface( listWriterInterface ) { - } - virtual std::shared_ptr getDataType() override - { - return m_builderPtr -> type(); - } + virtual std::shared_ptr getDataType() override { return m_builderPtr->type(); } - virtual std::shared_ptr getBuilder() override - { - return m_builderPtr; - } + virtual std::shared_ptr getBuilder() override { return m_builderPtr; } - virtual int64_t length() const override - { - return m_builderPtr -> length(); - } + virtual int64_t length() const override { return m_builderPtr->length(); } // Called for each row of the output parquet row, if no value was provided a null should be appended virtual void handleRowFinished() override { if( m_value.has_value() ) { - CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr -> Append().ok(), "Failed to append list" ); - m_listWriterInterface->writeItems(m_value.value()); + CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr->Append().ok(), "Failed to append list" ); + m_listWriterInterface->writeItems( m_value.value() ); m_value.reset(); } else { - STATUS_OK_OR_THROW_RUNTIME( m_builderPtr -> AppendNull(), "Failed write null arrow list" ); + STATUS_OK_OR_THROW_RUNTIME( m_builderPtr->AppendNull(), "Failed write null arrow list" ); } } @@ -180,19 +161,17 @@ class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder virtual std::shared_ptr buildArray() override { std::shared_ptr array; - STATUS_OK_OR_THROW_RUNTIME( m_builderPtr -> Finish( &array ), "Failed to create arrow list array" ); + STATUS_OK_OR_THROW_RUNTIME( m_builderPtr->Finish( &array ), "Failed to create arrow list array" ); return array; } - void setValue( const DialectGenericType& value) - { - m_value = value; - } + void setValue( const DialectGenericType & value ) { m_value = value; } private: - static std::shared_ptr createValueArrayBuilderForType( const std::shared_ptr<::arrow::DataType> &childType) + static std::shared_ptr + createValueArrayBuilderForType( const std::shared_ptr<::arrow::DataType> & childType ) { - switch(childType->id()) + switch( childType->id() ) { case arrow::Type::INT64: return std::make_shared(); @@ -201,18 +180,20 @@ class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder case arrow::Type::BOOL: return std::make_shared(); default: - CSP_THROW(TypeError, "Trying to create arrow list array builder for unsupported element type " << childType->name()); + CSP_THROW( TypeError, + "Trying to create arrow list array builder for unsupported element type " + << childType->name() ); } } - static std::vector> getArrowChildArrayBuilders( - const std::vector &childArrayBuilders ) + static std::vector> + getArrowChildArrayBuilders( const std::vector & childArrayBuilders ) { std::vector> res; - for( auto &columnBuilder : childArrayBuilders ) + for( auto & columnBuilder : childArrayBuilders ) { - res.push_back( columnBuilder -> getBuilder() ); + res.push_back( columnBuilder->getBuilder() ); } return res; } @@ -224,35 +205,25 @@ class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder std::optional m_value; }; - -template< typename ValueType, typename ArrowBuilderType > +template class BaseTypedArrayBuilder : public ArrowSingleColumnArrayBuilder { public: - using ValueTypeT = ValueType; + using ValueTypeT = ValueType; using ArrowBuilderTypeT = ArrowBuilderType; BaseTypedArrayBuilder( std::string columnName, std::uint32_t chunkSize ) - : ArrowSingleColumnArrayBuilder( columnName, chunkSize ), - m_builderPtr( std::make_shared() ) + : ArrowSingleColumnArrayBuilder( columnName, chunkSize ) + , m_builderPtr( std::make_shared() ) { - CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr -> Reserve( getChunkSize() ).ok(), "Failed to reserve arrow array size" ); + CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr->Reserve( getChunkSize() ).ok(), "Failed to reserve arrow array size" ); } - std::shared_ptr getBuilder() override - { - return m_builderPtr; - } + std::shared_ptr getBuilder() override { return m_builderPtr; } - std::shared_ptr getDataType() override - { - return m_builderPtr -> type(); - } + std::shared_ptr getDataType() override { return m_builderPtr->type(); } - virtual int64_t length() const override - { - return m_builderPtr -> length(); - } + virtual int64_t length() const override { return m_builderPtr->length(); } void handleRowFinished() override { @@ -262,7 +233,7 @@ class BaseTypedArrayBuilder : public ArrowSingleColumnArrayBuilder } else { - STATUS_OK_OR_THROW_RUNTIME( m_builderPtr -> AppendNull(), "Failed to append null to arrow array" ); + STATUS_OK_OR_THROW_RUNTIME( m_builderPtr->AppendNull(), "Failed to append null to arrow array" ); } m_value = nullptr; } @@ -270,42 +241,35 @@ class BaseTypedArrayBuilder : public ArrowSingleColumnArrayBuilder std::shared_ptr buildArray() override { std::shared_ptr array; - CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr -> Finish( &array ).ok(), "Failed to create arrow array" ); + CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr->Finish( &array ).ok(), "Failed to create arrow array" ); return array; } - void setValue( const ValueType &value ) - { - m_value = &value; - } + void setValue( const ValueType & value ) { m_value = &value; } protected: - template< typename ...BuilderArgs > - BaseTypedArrayBuilder( std::string columnName, std::uint32_t chunkSize, BuilderArgs ...args ) - : ArrowSingleColumnArrayBuilder( columnName, chunkSize ), - m_builderPtr( std::make_shared( args... ) ) + template + BaseTypedArrayBuilder( std::string columnName, std::uint32_t chunkSize, BuilderArgs... args ) + : ArrowSingleColumnArrayBuilder( columnName, chunkSize ) + , m_builderPtr( std::make_shared( args... ) ) { - } virtual void pushValueToArray() = 0; protected: std::shared_ptr m_builderPtr; - const ValueType *m_value = nullptr; + const ValueType * m_value = nullptr; }; -template< typename ValueType, typename ArrowBuilderType > +template class PrimitiveTypedArrayBuilder : public BaseTypedArrayBuilder { public: using BaseTypedArrayBuilder::BaseTypedArrayBuilder; protected: - void pushValueToArray() - { - [[maybe_unused]] auto status = this -> m_builderPtr -> Append( *this -> m_value ); - } + void pushValueToArray() { [[maybe_unused]] auto status = this->m_builderPtr->Append( *this->m_value ); } }; class StringArrayBuilder : public BaseTypedArrayBuilder @@ -316,8 +280,8 @@ class StringArrayBuilder : public BaseTypedArrayBuilder m_builderPtr -> Append( value.c_str(), value.length() ), + const std::string & value = *m_value; + STATUS_OK_OR_THROW_RUNTIME( this->m_builderPtr->Append( value.c_str(), value.length() ), "Failed to append value to arrow array" ); } }; @@ -330,8 +294,8 @@ class BytesArrayBuilder : public BaseTypedArrayBuilder m_builderPtr -> Append( value.c_str(), value.length() ), + const std::string & value = *m_value; + STATUS_OK_OR_THROW_RUNTIME( this->m_builderPtr->Append( value.c_str(), value.length() ), "Failed to append value to arrow array" ); } }; @@ -341,7 +305,7 @@ class [[maybe_unused]] CStringArrayBuilder : public BaseTypedArrayBuilder::BaseTypedArrayBuilder; - [[maybe_unused]] void setValueCopyPtr( const char *value ) + [[maybe_unused]] void setValueCopyPtr( const char * value ) { m_ptrCopy = value; setValue( m_ptrCopy ); @@ -350,31 +314,28 @@ class [[maybe_unused]] CStringArrayBuilder : public BaseTypedArrayBuilder m_builderPtr -> Append( value ), "Failed to append value to arrow array" ); + const char * value = *m_value; + STATUS_OK_OR_THROW_RUNTIME( this->m_builderPtr->Append( value ), "Failed to append value to arrow array" ); } private: - const char *m_ptrCopy; + const char * m_ptrCopy; }; - class DatetimeArrayBuilder : public BaseTypedArrayBuilder { public: DatetimeArrayBuilder( std::string columnName, std::uint32_t chunkSize ) - : BaseTypedArrayBuilder( columnName, - chunkSize, - std::make_shared( arrow::TimeUnit::NANO, - "UTC" ), - arrow::default_memory_pool() ) + : BaseTypedArrayBuilder( + columnName, chunkSize, std::make_shared( arrow::TimeUnit::NANO, "UTC" ), + arrow::default_memory_pool() ) { } protected: void pushValueToArray() { - STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( m_value -> asNanoseconds() ), + STATUS_OK_OR_THROW_RUNTIME( this->m_builderPtr->Append( m_value->asNanoseconds() ), "Failed to append timestamp value to arrow array" ); } }; @@ -383,8 +344,7 @@ class DateArrayBuilder : public BaseTypedArrayBuilder( columnName, - chunkSize ) + : BaseTypedArrayBuilder( columnName, chunkSize ) { } @@ -392,11 +352,12 @@ class DateArrayBuilder : public BaseTypedArrayBuilder m_builderPtr -> Append( timedelta.days() ), "Failed to append date value to arrow array" ); + STATUS_OK_OR_THROW_RUNTIME( this->m_builderPtr->Append( timedelta.days() ), + "Failed to append date value to arrow array" ); } private: - static csp::Date &getOrigin() + static csp::Date & getOrigin() { static csp::Date origin{ 1970, 1, 1 }; return origin; @@ -407,15 +368,17 @@ class TimeArrayBuilder : public BaseTypedArrayBuilder( columnName, chunkSize, - std::make_shared( arrow::TimeUnit::NANO ), arrow::default_memory_pool() ) + : BaseTypedArrayBuilder( + columnName, chunkSize, std::make_shared( arrow::TimeUnit::NANO ), + arrow::default_memory_pool() ) { } protected: void pushValueToArray() { - STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( m_value -> asNanoseconds() ), "Failed to append time value to arrow array" ); + STATUS_OK_OR_THROW_RUNTIME( this->m_builderPtr->Append( m_value->asNanoseconds() ), + "Failed to append time value to arrow array" ); } }; @@ -423,30 +386,30 @@ class TimedeltaArrayBuilder : public BaseTypedArrayBuilder( - columnName, chunkSize, std::make_shared( arrow::TimeUnit::NANO ), - arrow::default_memory_pool() ) + : BaseTypedArrayBuilder( + columnName, chunkSize, std::make_shared( arrow::TimeUnit::NANO ), + arrow::default_memory_pool() ) { } protected: void pushValueToArray() { - STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( m_value -> asNanoseconds() ), + STATUS_OK_OR_THROW_RUNTIME( this->m_builderPtr->Append( m_value->asNanoseconds() ), "Failed to append date value to arrow array" ); } }; -using BoolArrayBuilder = PrimitiveTypedArrayBuilder; -using Int8ArrayBuilder = PrimitiveTypedArrayBuilder; -using Int16ArrayBuilder = PrimitiveTypedArrayBuilder; -using Int32ArrayBuilder = PrimitiveTypedArrayBuilder; -using Int64ArrayBuilder = PrimitiveTypedArrayBuilder; -using UInt8ArrayBuilder = PrimitiveTypedArrayBuilder; +using BoolArrayBuilder = PrimitiveTypedArrayBuilder; +using Int8ArrayBuilder = PrimitiveTypedArrayBuilder; +using Int16ArrayBuilder = PrimitiveTypedArrayBuilder; +using Int32ArrayBuilder = PrimitiveTypedArrayBuilder; +using Int64ArrayBuilder = PrimitiveTypedArrayBuilder; +using UInt8ArrayBuilder = PrimitiveTypedArrayBuilder; using UInt16ArrayBuilder = PrimitiveTypedArrayBuilder; using UInt32ArrayBuilder = PrimitiveTypedArrayBuilder; using UInt64ArrayBuilder = PrimitiveTypedArrayBuilder; using DoubleArrayBuilder = PrimitiveTypedArrayBuilder; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h b/cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h index c88097f6f..0aa7b74fd 100644 --- a/cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h +++ b/cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h @@ -1,10 +1,10 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_DialectGenericListReaderInterface_H #define _IN_CSP_ADAPTERS_PARQUET_DialectGenericListReaderInterface_H -#include -#include -#include #include +#include +#include +#include namespace csp::adapters::parquet { @@ -12,37 +12,37 @@ namespace csp::adapters::parquet class DialectGenericListReaderInterface { public: - using Ptr = std::shared_ptr; - virtual ~DialectGenericListReaderInterface() = default; + using Ptr = std::shared_ptr; + virtual ~DialectGenericListReaderInterface() = default; virtual csp::DialectGenericType create( uint32_t size ) = 0; /** - * This form of creation is needed for elements that need to know the element size in advance, such as numpy arrays of strings + * This form of creation is needed for elements that need to know the element size in advance, such as numpy arrays + * of strings * @param size * @return */ virtual csp::DialectGenericType create( uint32_t size, uint32_t maxElementSize ) = 0; - virtual CspTypePtr getValueType() = 0; + virtual CspTypePtr getValueType() = 0; }; -template< typename T > +template class TypedDialectGenericListReaderInterface : public DialectGenericListReaderInterface { public: using Ptr = std::shared_ptr>; - CspTypePtr getValueType() override{ return CspType::fromCType::type(); } + CspTypePtr getValueType() override { return CspType::fromCType::type(); } /** - * Return a raw data buffer for the object that was generated using the call to create. Should return the pointer to internal buffer - * that can be written directly into if that's supported, otherwise, should return nullptr in which case setValue will be used for - * inserting the elements + * Return a raw data buffer for the object that was generated using the call to create. Should return the pointer to + * internal buffer that can be written directly into if that's supported, otherwise, should return nullptr in which + * case setValue will be used for inserting the elements * @param list: The list object that was created using the call to create function * @return Internal write buffer of the list of nullptr if the list doesn't support writing to internal buffer */ - virtual T *getRawDataBuffer( const csp::DialectGenericType &list ) const{ return nullptr; } - virtual void setValue( const csp::DialectGenericType &list, int index, const T &value ) = 0; + virtual T * getRawDataBuffer( const csp::DialectGenericType & list ) const { return nullptr; } + virtual void setValue( const csp::DialectGenericType & list, int index, const T & value ) = 0; }; - -} +} // namespace csp::adapters::parquet #endif \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/DialectGenericListWriterInterface.h b/cpp/csp/adapters/parquet/DialectGenericListWriterInterface.h index ab01d82ea..ec4b9ff6a 100644 --- a/cpp/csp/adapters/parquet/DialectGenericListWriterInterface.h +++ b/cpp/csp/adapters/parquet/DialectGenericListWriterInterface.h @@ -1,10 +1,10 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_DialectGenericListWriterInterface_H #define _IN_CSP_ADAPTERS_PARQUET_DialectGenericListWriterInterface_H -#include -#include -#include #include +#include +#include +#include namespace csp::adapters::parquet { @@ -13,13 +13,13 @@ class DialectGenericListWriterInterface { public: using ArrayBuilderPtr = std::shared_ptr; - using Ptr = std::shared_ptr; + using Ptr = std::shared_ptr; - virtual void writeItems( const csp::DialectGenericType &listObject ) = 0; - virtual ~DialectGenericListWriterInterface() = default; + virtual void writeItems( const csp::DialectGenericType & listObject ) = 0; + virtual ~DialectGenericListWriterInterface() = default; }; -template< typename T > +template class TypedDialectGenericListWriterInterface : public DialectGenericListWriterInterface { public: @@ -28,27 +28,17 @@ class TypedDialectGenericListWriterInterface : public DialectGenericListWriterIn TypedDialectGenericListWriterInterface() { m_writeFunc = []( const T & ) - { - CSP_THROW( RuntimeException, "Write function for TypedDialectGenericListWriterInterface is not set" ); - }; - } - - void setWriteFunction( const WriteFuncT &func ) - { - m_writeFunc = func; + { CSP_THROW( RuntimeException, "Write function for TypedDialectGenericListWriterInterface is not set" ); }; } + void setWriteFunction( const WriteFuncT & func ) { m_writeFunc = func; } - void writeValue( const T &value ) - { - m_writeFunc( value ); - } + void writeValue( const T & value ) { m_writeFunc( value ); } private: std::function m_writeFunc; }; - -} +} // namespace csp::adapters::parquet #endif \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/FileReaderWrapper.cpp b/cpp/csp/adapters/parquet/FileReaderWrapper.cpp index 682883b99..780a3598e 100644 --- a/cpp/csp/adapters/parquet/FileReaderWrapper.cpp +++ b/cpp/csp/adapters/parquet/FileReaderWrapper.cpp @@ -1,8 +1,7 @@ -#include #include +#include #include - namespace csp::adapters::parquet { @@ -11,15 +10,13 @@ FileReaderWrapper::~FileReaderWrapper() close(); } -void FileReaderWrapper::open( const std::string &fileName ) +void FileReaderWrapper::open( const std::string & fileName ) { if( m_inputFile ) { close(); } - PARQUET_ASSIGN_OR_THROW( - m_inputFile, - arrow::io::ReadableFile::Open( fileName ) ); + PARQUET_ASSIGN_OR_THROW( m_inputFile, arrow::io::ReadableFile::Open( fileName ) ); m_fileName = fileName; } @@ -32,4 +29,4 @@ void FileReaderWrapper::close() } } -} \ No newline at end of file +} // namespace csp::adapters::parquet \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/FileReaderWrapper.h b/cpp/csp/adapters/parquet/FileReaderWrapper.h index b489b9f5a..e712c337d 100644 --- a/cpp/csp/adapters/parquet/FileReaderWrapper.h +++ b/cpp/csp/adapters/parquet/FileReaderWrapper.h @@ -14,7 +14,7 @@ namespace arrow { class Schema; class Table; -} +} // namespace arrow namespace csp::adapters::parquet { @@ -24,18 +24,18 @@ class FileReaderWrapper public: using Ptr = std::unique_ptr; - FileReaderWrapper() = default; - FileReaderWrapper( const FileReaderWrapper & ) = delete; - FileReaderWrapper &operator=( const FileReaderWrapper & ) = delete; + FileReaderWrapper() = default; + FileReaderWrapper( const FileReaderWrapper & ) = delete; + FileReaderWrapper & operator=( const FileReaderWrapper & ) = delete; virtual ~FileReaderWrapper(); - virtual void open( const std::string &fileName ); + virtual void open( const std::string & fileName ); virtual void close(); - virtual bool readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> &dst ) = 0; - virtual void getSchema(std::shared_ptr<::arrow::Schema>& dst) = 0; + virtual bool readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> & dst ) = 0; + virtual void getSchema( std::shared_ptr<::arrow::Schema> & dst ) = 0; - const std::string &getCurrentFileName() const{ return m_fileName; } + const std::string & getCurrentFileName() const { return m_fileName; } protected: std::shared_ptr<::arrow::io::ReadableFile> m_inputFile; @@ -46,5 +46,5 @@ class FileReaderWrapper using FileReaderWrapperPtr = FileReaderWrapper::Ptr; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/FileWriterWrapper.cpp b/cpp/csp/adapters/parquet/FileWriterWrapper.cpp index 02e846311..71e22d28c 100644 --- a/cpp/csp/adapters/parquet/FileWriterWrapper.cpp +++ b/cpp/csp/adapters/parquet/FileWriterWrapper.cpp @@ -2,48 +2,48 @@ #include #include #include -#include #include +#include namespace csp::adapters::parquet { std::unordered_map FileWriterWrapper::m_compressionNameMapping{ - { "", ::arrow::Compression::UNCOMPRESSED }, - { "snappy", ::arrow::Compression::SNAPPY }, - { "gzip", ::arrow::Compression::GZIP }, - { "brotli", ::arrow::Compression::BROTLI }, - { "zstd", ::arrow::Compression::ZSTD }, - { "lz4", ::arrow::Compression::LZ4 }, - { "lz4_frame", ::arrow::Compression::LZ4_FRAME }, - { "lzo", ::arrow::Compression::LZO }, - { "bz2", ::arrow::Compression::BZ2 } -}; + { "", ::arrow::Compression::UNCOMPRESSED }, + { "snappy", ::arrow::Compression::SNAPPY }, + { "gzip", ::arrow::Compression::GZIP }, + { "brotli", ::arrow::Compression::BROTLI }, + { "zstd", ::arrow::Compression::ZSTD }, + { "lz4", ::arrow::Compression::LZ4 }, + { "lz4_frame", ::arrow::Compression::LZ4_FRAME }, + { "lzo", ::arrow::Compression::LZO }, + { "bz2", ::arrow::Compression::BZ2 } }; -void FileWriterWrapper::open( const std::string &fileName, const std::string &compression, bool allowOverwrite ) +void FileWriterWrapper::open( const std::string & fileName, const std::string & compression, bool allowOverwrite ) { if( !allowOverwrite ) { CSP_TRUE_OR_THROW_RUNTIME( !utils::fileExists( fileName ), - "Trying to overwrite existing file " << fileName << " while allow_overwrite is false" ); + "Trying to overwrite existing file " << fileName + << " while allow_overwrite is false" ); } utils::mkdir( utils::dirname( fileName ) ); openImpl( fileName, compression ); } -::arrow::Compression::type FileWriterWrapper::resolveCompression( const std::string &compression ) +::arrow::Compression::type FileWriterWrapper::resolveCompression( const std::string & compression ) { auto it = m_compressionNameMapping.find( compression ); CSP_TRUE_OR_THROW_RUNTIME( it != m_compressionNameMapping.end(), "Unable to resolve compression: " << compression ); return it->second; } -std::unique_ptr FileWriterWrapper::resolveCompressionCodec( const std::string &compression ) +std::unique_ptr FileWriterWrapper::resolveCompressionCodec( const std::string & compression ) { auto compressionType = resolveCompression( compression ); - auto res = arrow::util::Codec::Create( compressionType ); - STATUS_OK_OR_THROW_RUNTIME(res.status(), "Failed to create arrow codec for " << compressionType ); - return std::move(res.ValueUnsafe()); + auto res = arrow::util::Codec::Create( compressionType ); + STATUS_OK_OR_THROW_RUNTIME( res.status(), "Failed to create arrow codec for " << compressionType ); + return std::move( res.ValueUnsafe() ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/FileWriterWrapper.h b/cpp/csp/adapters/parquet/FileWriterWrapper.h index fbe82f387..af0629c8b 100644 --- a/cpp/csp/adapters/parquet/FileWriterWrapper.h +++ b/cpp/csp/adapters/parquet/FileWriterWrapper.h @@ -1,17 +1,17 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_FileWriterWrapper_H #define _IN_CSP_ADAPTERS_PARQUET_FileWriterWrapper_H -#include -#include -#include #include +#include +#include +#include namespace arrow { class Schema; class Table; -} +} // namespace arrow namespace csp::adapters::parquet { @@ -20,35 +20,32 @@ class FileWriterWrapper { public: FileWriterWrapper( std::shared_ptr<::arrow::Schema> schema ) - : m_schema( schema ) + : m_schema( schema ) { } - FileWriterWrapper( const FileWriterWrapper &other ) = delete; - FileWriterWrapper &operator=( const FileWriterWrapper &other ) = delete; + FileWriterWrapper( const FileWriterWrapper & other ) = delete; + FileWriterWrapper & operator=( const FileWriterWrapper & other ) = delete; virtual ~FileWriterWrapper() {} - void open( const std::string &fileName, const std::string &compression, bool allowOverwrite = false ); - virtual void close() = 0; - virtual void writeTable( const std::shared_ptr<::arrow::Table> &table ) = 0; + void open( const std::string & fileName, const std::string & compression, bool allowOverwrite = false ); + virtual void close() = 0; + virtual void writeTable( const std::shared_ptr<::arrow::Table> & table ) = 0; - const std::shared_ptr<::arrow::Schema> &getSchema() const { return m_schema; } + const std::shared_ptr<::arrow::Schema> & getSchema() const { return m_schema; } protected: - virtual void openImpl( const std::string &fileName, const std::string &compression ) = 0; - + virtual void openImpl( const std::string & fileName, const std::string & compression ) = 0; - static ::arrow::Compression::type resolveCompression( const std::string &compression ); - static std::unique_ptr resolveCompressionCodec( const std::string &compression ); + static ::arrow::Compression::type resolveCompression( const std::string & compression ); + static std::unique_ptr resolveCompressionCodec( const std::string & compression ); private: std::shared_ptr<::arrow::Schema> m_schema; static std::unordered_map m_compressionNameMapping; - }; -} - +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp b/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp index 3b103283f..176ed0a17 100644 --- a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp +++ b/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp @@ -1,20 +1,21 @@ -#include -#include +#include +#include +#include #include +#include #include #include #include -#include -#include -#include -#include +#include #include +#include namespace csp::adapters::parquet { -FileWriterWrapperContainer::WriterPtr FileWriterWrapperContainer::createSingleFileWrapper( const std::shared_ptr &schema, - bool isWriteArrowBinary ) +FileWriterWrapperContainer::WriterPtr +FileWriterWrapperContainer::createSingleFileWrapper( const std::shared_ptr & schema, + bool isWriteArrowBinary ) { if( isWriteArrowBinary ) { @@ -26,76 +27,78 @@ FileWriterWrapperContainer::WriterPtr FileWriterWrapperContainer::createSingleFi } } -SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer( std::shared_ptr schema, bool isWriteArrowBinary ) - : m_fileWriterWrapper( createSingleFileWrapper( schema, isWriteArrowBinary ) ) +SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer( std::shared_ptr schema, + bool isWriteArrowBinary ) + : m_fileWriterWrapper( createSingleFileWrapper( schema, isWriteArrowBinary ) ) { } -void SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer::open( const std::string &fileName, const std::string &compression, - bool allowOverwrite ) +void SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer::open( const std::string & fileName, + const std::string & compression, + bool allowOverwrite ) { - m_fileWriterWrapper -> open( fileName, compression, allowOverwrite ); - setOpen(true); + m_fileWriterWrapper->open( fileName, compression, allowOverwrite ); + setOpen( true ); } void SingleFileWriterWrapperContainer::close() { - m_fileWriterWrapper -> close(); - setOpen(false); + m_fileWriterWrapper->close(); + setOpen( false ); } -void SingleFileWriterWrapperContainer::writeData( const std::vector> &columnBuilders ) +void SingleFileWriterWrapperContainer::writeData( + const std::vector> & columnBuilders ) { std::vector> columns; columns.reserve( columnBuilders.size() ); - for( auto &&columnBuilder:columnBuilders ) + for( auto && columnBuilder : columnBuilders ) { - columns.push_back( columnBuilder -> buildArray() ); + columns.push_back( columnBuilder->buildArray() ); } - auto table = arrow::Table::Make( m_fileWriterWrapper -> getSchema(), columns ); + auto table = arrow::Table::Make( m_fileWriterWrapper->getSchema(), columns ); - m_fileWriterWrapper -> writeTable( table ); + m_fileWriterWrapper->writeTable( table ); } - -MultipleFileWriterWrapperContainer::MultipleFileWriterWrapperContainer( std::shared_ptr schema, bool isWriteArrowBinary ) +MultipleFileWriterWrapperContainer::MultipleFileWriterWrapperContainer( std::shared_ptr schema, + bool isWriteArrowBinary ) { - auto &fields = schema -> fields(); + auto & fields = schema->fields(); m_fileWriterWrappers.reserve( fields.size() ); - for( auto &&field:fields ) + for( auto && field : fields ) { std::vector> curFields; std::string fileExtension = isWriteArrowBinary ? ".arrow" : ".parquet"; - std::string fileName = field -> name() + fileExtension; + std::string fileName = field->name() + fileExtension; - m_fileWriterWrappers.push_back( { fileName, - createSingleFileWrapper( arrow::schema( { field } ), isWriteArrowBinary ) } ); + m_fileWriterWrappers.push_back( + { fileName, createSingleFileWrapper( arrow::schema( { field } ), isWriteArrowBinary ) } ); } } -void MultipleFileWriterWrapperContainer::open( const std::string &fileName, const std::string &compression, bool allowOverwrite ) +void MultipleFileWriterWrapperContainer::open( const std::string & fileName, const std::string & compression, + bool allowOverwrite ) { - for( auto &&record : m_fileWriterWrappers ) + for( auto && record : m_fileWriterWrappers ) { - record.m_fileWriterWrapper -> open( fileName + '/' + record.m_fileName, - compression, - allowOverwrite ); - + record.m_fileWriterWrapper->open( fileName + '/' + record.m_fileName, compression, allowOverwrite ); } - setOpen(true); + setOpen( true ); } void MultipleFileWriterWrapperContainer::close() { - for( auto &&record : m_fileWriterWrappers ) + for( auto && record : m_fileWriterWrappers ) { - record.m_fileWriterWrapper -> close(); + record.m_fileWriterWrapper->close(); } - setOpen(false); + setOpen( false ); } -void MultipleFileWriterWrapperContainer::writeData( const std::vector> &columnBuilders ) +void MultipleFileWriterWrapperContainer::writeData( + const std::vector> & columnBuilders ) { std::vector> columns; columns.reserve( 1 ); @@ -105,11 +108,11 @@ void MultipleFileWriterWrapperContainer::writeData( const std::vector buildArray() ); - auto &fileWriterWrapper = m_fileWriterWrappers[ i ].m_fileWriterWrapper; - auto table = arrow::Table::Make( fileWriterWrapper -> getSchema(), columns ); - fileWriterWrapper -> writeTable( table ); + columns.push_back( columnBuilders[i]->buildArray() ); + auto & fileWriterWrapper = m_fileWriterWrappers[i].m_fileWriterWrapper; + auto table = arrow::Table::Make( fileWriterWrapper->getSchema(), columns ); + fileWriterWrapper->writeTable( table ); } } -} \ No newline at end of file +} // namespace csp::adapters::parquet \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.h b/cpp/csp/adapters/parquet/FileWriterWrapperContainer.h index debb1f733..ec474ac42 100644 --- a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.h +++ b/cpp/csp/adapters/parquet/FileWriterWrapperContainer.h @@ -1,10 +1,10 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_FileWriterWrapperContainer_H #define _IN_CSP_ADAPTERS_PARQUET_FileWriterWrapperContainer_H +#include +#include #include #include -#include -#include namespace arrow { @@ -18,24 +18,26 @@ class ArrowSingleColumnArrayBuilder; class FileWriterWrapperContainer { public: - FileWriterWrapperContainer() = default; - FileWriterWrapperContainer( FileWriterWrapperContainer & ) = delete; - FileWriterWrapperContainer &operator=( const FileWriterWrapperContainer & ) = delete; - virtual ~FileWriterWrapperContainer() = default; + FileWriterWrapperContainer() = default; + FileWriterWrapperContainer( FileWriterWrapperContainer & ) = delete; + FileWriterWrapperContainer & operator=( const FileWriterWrapperContainer & ) = delete; + virtual ~FileWriterWrapperContainer() = default; + + virtual void open( const std::string & fileName, const std::string & compression, bool allowOverwrite = false ) = 0; - virtual void open( const std::string &fileName, const std::string &compression, bool allowOverwrite = false ) = 0; + bool isOpen() const { return m_isOpen; } - bool isOpen() const{ return m_isOpen; } + virtual void close() = 0; + virtual void writeData( const std::vector> & columnBuilders ) = 0; - virtual void close() = 0; - virtual void writeData( const std::vector> &columnBuilders ) = 0; protected: - void setOpen( bool open ){ m_isOpen = open; } + void setOpen( bool open ) { m_isOpen = open; } protected: using WriterPtr = std::unique_ptr; - static WriterPtr createSingleFileWrapper( const std::shared_ptr &schema, bool isWriteArrowBinary ); + static WriterPtr createSingleFileWrapper( const std::shared_ptr & schema, bool isWriteArrowBinary ); + private: bool m_isOpen = false; }; @@ -44,9 +46,12 @@ class SingleFileWriterWrapperContainer final : public FileWriterWrapperContainer { public: SingleFileWriterWrapperContainer( std::shared_ptr schema, bool isWriteArrowBinary ); - virtual void open( const std::string &fileName, const std::string &compression, bool allowOverwrite = false ) override; + virtual void open( const std::string & fileName, const std::string & compression, + bool allowOverwrite = false ) override; virtual void close() override; - virtual void writeData( const std::vector> &columnBuilders ) override; + virtual void + writeData( const std::vector> & columnBuilders ) override; + private: std::unique_ptr m_fileWriterWrapper; }; @@ -55,9 +60,12 @@ class MultipleFileWriterWrapperContainer : public FileWriterWrapperContainer { public: MultipleFileWriterWrapperContainer( std::shared_ptr schema, bool isWriteArrowBinary ); - virtual void open( const std::string &fileName, const std::string &compression, bool allowOverwrite = false ) override; + virtual void open( const std::string & fileName, const std::string & compression, + bool allowOverwrite = false ) override; virtual void close() override; - virtual void writeData( const std::vector> &columnBuilders ) override; + virtual void + writeData( const std::vector> & columnBuilders ) override; + private: struct SingleFileWrapperInfo { @@ -69,6 +77,6 @@ class MultipleFileWriterWrapperContainer : public FileWriterWrapperContainer std::vector m_fileWriterWrappers; }; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp index 325dd2497..193595d88 100644 --- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp +++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp @@ -4,11 +4,10 @@ namespace csp::adapters::parquet { -ParquetDictBasketOutputWriter::ParquetDictBasketOutputWriter( - ParquetOutputAdapterManager *outputAdapterManager, - const std::string &columnName ) - : - ParquetWriter( outputAdapterManager, false ), m_nextCycleIndex( 0 ) +ParquetDictBasketOutputWriter::ParquetDictBasketOutputWriter( ParquetOutputAdapterManager * outputAdapterManager, + const std::string & columnName ) + : ParquetWriter( outputAdapterManager, false ) + , m_nextCycleIndex( 0 ) { m_symbolOutputAdapter = getScalarOutputHandler( CspType::STRING(), columnName + "__csp_symbol" ); m_cycleIndexOutputAdapter = createScalarOutputHandler( CspType::UINT16(), columnName + "__csp_value_count" ); @@ -18,127 +17,124 @@ void ParquetDictBasketOutputWriter::start() { ParquetWriter::start(); m_indexFileWriterContainer = std::make_unique( - arrow::schema( { arrow::field( m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getColumnName(), - m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getDataType() ) } ), - m_adapterMgr.isWriteArrowBinary() ); + arrow::schema( { arrow::field( m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 )->getColumnName(), + m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 )->getDataType() ) } ), + m_adapterMgr.isWriteArrowBinary() ); if( !m_adapterMgr.getFileName().empty() ) { - m_indexFileWriterContainer -> open( m_adapterMgr.getFileName(), - m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); - + m_indexFileWriterContainer->open( m_adapterMgr.getFileName(), m_adapterMgr.getCompression(), + m_adapterMgr.isAllowOverwrite() ); } } void ParquetDictBasketOutputWriter::stop() { - auto &&indexColumnArrayBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ); - if( indexColumnArrayBuilder -> length() > 0 ) + auto && indexColumnArrayBuilder = m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 ); + if( indexColumnArrayBuilder->length() > 0 ) { - CSP_TRUE_OR_THROW_RUNTIME(isFileOpen(), "On stop ParquetDictBasketOutputWriter has data to write but no open file"); - m_indexFileWriterContainer -> writeData( { indexColumnArrayBuilder } ); + CSP_TRUE_OR_THROW_RUNTIME( isFileOpen(), + "On stop ParquetDictBasketOutputWriter has data to write but no open file" ); + m_indexFileWriterContainer->writeData( { indexColumnArrayBuilder } ); } - m_indexFileWriterContainer -> close(); + m_indexFileWriterContainer->close(); m_indexFileWriterContainer = nullptr; ParquetWriter::stop(); } -void ParquetDictBasketOutputWriter::writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ) +void ParquetDictBasketOutputWriter::writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ) { m_adapterMgr.scheduleEndCycle(); - m_symbolOutputAdapter -> writeValue( valueKey ); + m_symbolOutputAdapter->writeValue( valueKey ); ParquetWriter::onEndCycle(); ++m_nextCycleIndex; } - void ParquetDictBasketOutputWriter::onEndCycle() { - if(isFileOpen()) + if( isFileOpen() ) { - m_cycleIndexOutputAdapter -> writeValue( m_nextCycleIndex ); - auto &&indexColumnArrayBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ); - indexColumnArrayBuilder -> handleRowFinished(); - if( indexColumnArrayBuilder -> length() >= getChunkSize() ) + m_cycleIndexOutputAdapter->writeValue( m_nextCycleIndex ); + auto && indexColumnArrayBuilder = m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 ); + indexColumnArrayBuilder->handleRowFinished(); + if( indexColumnArrayBuilder->length() >= getChunkSize() ) { - m_indexFileWriterContainer -> writeData( { indexColumnArrayBuilder } ); + m_indexFileWriterContainer->writeData( { indexColumnArrayBuilder } ); } m_nextCycleIndex = 0; } else { - CSP_TRUE_OR_THROW_RUNTIME(m_nextCycleIndex==0, "ParquetDictBasketOutputWriter has non 0 index with no open file"); + CSP_TRUE_OR_THROW_RUNTIME( m_nextCycleIndex == 0, + "ParquetDictBasketOutputWriter has non 0 index with no open file" ); } } -void ParquetDictBasketOutputWriter::onFileNameChange( const std::string &fileName ) +void ParquetDictBasketOutputWriter::onFileNameChange( const std::string & fileName ) { ParquetWriter::onFileNameChange( fileName ); - if( m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> length() > 0 ) + if( m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 )->length() > 0 ) { - CSP_TRUE_OR_THROW_RUNTIME( m_indexFileWriterContainer -> isOpen(), "Trying to write basket index data to closed file" ); - m_indexFileWriterContainer -> writeData( { m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) } ); + CSP_TRUE_OR_THROW_RUNTIME( m_indexFileWriterContainer->isOpen(), + "Trying to write basket index data to closed file" ); + m_indexFileWriterContainer->writeData( { m_cycleIndexOutputAdapter->getColumnArrayBuilder( 0 ) } ); } - if (m_indexFileWriterContainer->isOpen()) + if( m_indexFileWriterContainer->isOpen() ) { m_indexFileWriterContainer->close(); } - if(!fileName.empty()) + if( !fileName.empty() ) { - m_indexFileWriterContainer - -> open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); + m_indexFileWriterContainer->open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); } - } -SingleColumnParquetOutputHandler *ParquetDictBasketOutputWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name ) +SingleColumnParquetOutputHandler * ParquetDictBasketOutputWriter::createScalarOutputHandler( CspTypePtr type, + const std::string & name ) { m_allHandlers.push_back( std::make_unique( m_engine, *this, type, name ) ); - return static_cast(m_allHandlers.back().get()); + return static_cast( m_allHandlers.back().get() ); } -StructParquetOutputHandler *ParquetDictBasketOutputWriter::createStructOutputHandler( CspTypePtr type, - const DictionaryPtr &fieldMap ) +StructParquetOutputHandler * ParquetDictBasketOutputWriter::createStructOutputHandler( CspTypePtr type, + const DictionaryPtr & fieldMap ) { m_allHandlers.push_back( std::make_unique( m_engine, *this, type, fieldMap ) ); - return static_cast(m_allHandlers.back().get()); + return static_cast( m_allHandlers.back().get() ); } - -ParquetScalarDictBasketOutputWriter::ParquetScalarDictBasketOutputWriter( ParquetOutputAdapterManager *outputAdapterManager, - const std::string &columnName, - CspTypePtr cspTypePtr ) - : ParquetDictBasketOutputWriter( outputAdapterManager, columnName ) +ParquetScalarDictBasketOutputWriter::ParquetScalarDictBasketOutputWriter( + ParquetOutputAdapterManager * outputAdapterManager, const std::string & columnName, CspTypePtr cspTypePtr ) + : ParquetDictBasketOutputWriter( outputAdapterManager, columnName ) { m_valueOutputAdapter = getScalarOutputHandler( cspTypePtr, columnName ); } -void ParquetScalarDictBasketOutputWriter::writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ) +void ParquetScalarDictBasketOutputWriter::writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ) { - m_valueOutputAdapter -> writeValueFromTs( ts ); - ParquetDictBasketOutputWriter::writeValue(valueKey, ts); + m_valueOutputAdapter->writeValueFromTs( ts ); + ParquetDictBasketOutputWriter::writeValue( valueKey, ts ); } -ParquetStructDictBasketOutputWriter::ParquetStructDictBasketOutputWriter( ParquetOutputAdapterManager *outputAdapterManager, - const std::string &columnName, - CspTypePtr cspTypePtr ) - : ParquetDictBasketOutputWriter( outputAdapterManager, columnName ) +ParquetStructDictBasketOutputWriter::ParquetStructDictBasketOutputWriter( + ParquetOutputAdapterManager * outputAdapterManager, const std::string & columnName, CspTypePtr cspTypePtr ) + : ParquetDictBasketOutputWriter( outputAdapterManager, columnName ) { // We don't support fieldMap for now, only default field map - auto structMetaPtr = std::static_pointer_cast( cspTypePtr ) -> meta().get(); - DictionaryPtr dict = std::make_shared(); - for(auto&& field:structMetaPtr->fields()) + auto structMetaPtr = std::static_pointer_cast( cspTypePtr )->meta().get(); + DictionaryPtr dict = std::make_shared(); + for( auto && field : structMetaPtr->fields() ) { - dict->insert(field->fieldname(), columnName + "." + field->fieldname()); + dict->insert( field->fieldname(), columnName + "." + field->fieldname() ); } m_valueOutputAdapter = getStructOutputHandler( cspTypePtr, dict ); } -void ParquetStructDictBasketOutputWriter::writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ) +void ParquetStructDictBasketOutputWriter::writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ) { - m_valueOutputAdapter -> writeValueFromTs( ts ); - ParquetDictBasketOutputWriter::writeValue(valueKey, ts); + m_valueOutputAdapter->writeValueFromTs( ts ); + ParquetDictBasketOutputWriter::writeValue( valueKey, ts ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h index 5b23dbee2..3168cc171 100644 --- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h +++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h @@ -14,21 +14,22 @@ class ParquetOutputAdapterManager; class ParquetDictBasketOutputWriter : public ParquetWriter { public: - ParquetDictBasketOutputWriter( ParquetOutputAdapterManager *outputAdapterManager, const std::string &columnName ); + ParquetDictBasketOutputWriter( ParquetOutputAdapterManager * outputAdapterManager, const std::string & columnName ); void start() override; void stop() override; - virtual void writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ); + virtual void writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ); void onEndCycle() override final; - void onFileNameChange( const std::string &fileName ) override; + void onFileNameChange( const std::string & fileName ) override; + protected: - SingleColumnParquetOutputHandler *createScalarOutputHandler( CspTypePtr type, const std::string &name ) override; - StructParquetOutputHandler *createStructOutputHandler( CspTypePtr type, const DictionaryPtr &fieldMap ) override; + SingleColumnParquetOutputHandler * createScalarOutputHandler( CspTypePtr type, const std::string & name ) override; + StructParquetOutputHandler * createStructOutputHandler( CspTypePtr type, const DictionaryPtr & fieldMap ) override; private: - SingleColumnParquetOutputHandler *m_symbolOutputAdapter; - SingleColumnParquetOutputHandler *m_cycleIndexOutputAdapter; + SingleColumnParquetOutputHandler * m_symbolOutputAdapter; + SingleColumnParquetOutputHandler * m_cycleIndexOutputAdapter; std::uint16_t m_nextCycleIndex; std::vector> m_allHandlers; std::unique_ptr m_indexFileWriterContainer; @@ -37,26 +38,27 @@ class ParquetDictBasketOutputWriter : public ParquetWriter class ParquetScalarDictBasketOutputWriter final : public ParquetDictBasketOutputWriter { public: - ParquetScalarDictBasketOutputWriter( ParquetOutputAdapterManager *outputAdapterManager, const std::string &columnName, - CspTypePtr cspTypePtr ); + ParquetScalarDictBasketOutputWriter( ParquetOutputAdapterManager * outputAdapterManager, + const std::string & columnName, CspTypePtr cspTypePtr ); + + void writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ) override; - void writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ) override; private: - SingleColumnParquetOutputHandler *m_valueOutputAdapter; + SingleColumnParquetOutputHandler * m_valueOutputAdapter; }; class ParquetStructDictBasketOutputWriter final : public ParquetDictBasketOutputWriter { public: - ParquetStructDictBasketOutputWriter( ParquetOutputAdapterManager *outputAdapterManager, const std::string &columnName, - CspTypePtr cspTypePtr ); + ParquetStructDictBasketOutputWriter( ParquetOutputAdapterManager * outputAdapterManager, + const std::string & columnName, CspTypePtr cspTypePtr ); + + void writeValue( const std::string & valueKey, const TimeSeriesProvider * ts ) override; - void writeValue( const std::string &valueKey, const TimeSeriesProvider *ts ) override; private: - StructParquetOutputHandler *m_valueOutputAdapter; + StructParquetOutputHandler * m_valueOutputAdapter; }; -} - +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.cpp b/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.cpp index 3e2992606..8ef0a62e1 100644 --- a/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.cpp +++ b/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.cpp @@ -1,7 +1,7 @@ +#include #include #include #include -#include namespace csp::adapters::parquet { @@ -10,15 +10,15 @@ ParquetFileReaderWrapper::~ParquetFileReaderWrapper() close(); } -void ParquetFileReaderWrapper::open( const std::string &fileName ) +void ParquetFileReaderWrapper::open( const std::string & fileName ) { FileReaderWrapper::open( fileName ); try { STATUS_OK_OR_THROW_RUNTIME( - ::parquet::arrow::OpenFile( m_inputFile, arrow::default_memory_pool(), &m_fileReader ), - "Failed to open parquet file " << fileName ); + ::parquet::arrow::OpenFile( m_inputFile, arrow::default_memory_pool(), &m_fileReader ), + "Failed to open parquet file " << fileName ); } catch( ... ) { @@ -34,11 +34,12 @@ void ParquetFileReaderWrapper::close() FileReaderWrapper::close(); } -bool ParquetFileReaderWrapper::readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> &dst ) +bool ParquetFileReaderWrapper::readNextRowGroup( const std::vector neededColumns, + std::shared_ptr<::arrow::Table> & dst ) { - if( m_fileReader -> num_row_groups() > m_nextRowGroup ) + if( m_fileReader->num_row_groups() > m_nextRowGroup ) { - STATUS_OK_OR_THROW_RUNTIME( m_fileReader -> ReadRowGroup( m_nextRowGroup, neededColumns, &dst ), + STATUS_OK_OR_THROW_RUNTIME( m_fileReader->ReadRowGroup( m_nextRowGroup, neededColumns, &dst ), "Failed to read row group " << m_nextRowGroup << " from file " << m_fileName ); ++m_nextRowGroup; return true; @@ -47,14 +48,12 @@ bool ParquetFileReaderWrapper::readNextRowGroup( const std::vector neededCo { dst = nullptr; return false; - } } -void ParquetFileReaderWrapper::getSchema( std::shared_ptr<::arrow::Schema> &dst ) +void ParquetFileReaderWrapper::getSchema( std::shared_ptr<::arrow::Schema> & dst ) { - STATUS_OK_OR_THROW_RUNTIME( m_fileReader -> GetSchema( &dst ), - "Failed to get schema from file " << m_fileName ); + STATUS_OK_OR_THROW_RUNTIME( m_fileReader->GetSchema( &dst ), "Failed to get schema from file " << m_fileName ); } -} \ No newline at end of file +} // namespace csp::adapters::parquet \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.h b/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.h index b3a5c902d..5f09a0190 100644 --- a/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.h +++ b/cpp/csp/adapters/parquet/ParquetFileReaderWrapper.h @@ -1,7 +1,7 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetFileReaderWrapper_H #define _IN_CSP_ADAPTERS_PARQUET_ParquetFileReaderWrapper_H -#include +#include #include namespace parquet::arrow @@ -16,16 +16,17 @@ class ParquetFileReaderWrapper : public FileReaderWrapper { public: ~ParquetFileReaderWrapper(); - virtual void open( const std::string &fileName ) override; + virtual void open( const std::string & fileName ) override; virtual void close() override; - virtual bool readNextRowGroup( const std::vector neededColumns, std::shared_ptr<::arrow::Table> &dst ) override; - void getSchema( std::shared_ptr<::arrow::Schema> &dst ) override; + virtual bool readNextRowGroup( const std::vector neededColumns, + std::shared_ptr<::arrow::Table> & dst ) override; + void getSchema( std::shared_ptr<::arrow::Schema> & dst ) override; private: std::unique_ptr<::parquet::arrow::FileReader> m_fileReader; int m_nextRowGroup; }; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp b/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp index badd963a3..661293d0e 100644 --- a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp +++ b/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp @@ -1,30 +1,28 @@ -#include -#include #include #include +#include +#include #include namespace csp::adapters::parquet { -void ParquetFileWriterWrapper::openImpl( const std::string &fileName, const std::string &compression ) +void ParquetFileWriterWrapper::openImpl( const std::string & fileName, const std::string & compression ) { CSP_TRUE_OR_THROW_RUNTIME( m_outputStream == nullptr, "Trying to open parquet file while previous was not closed" ); - PARQUET_ASSIGN_OR_THROW( - m_outputStream, - arrow::io::FileOutputStream::Open( fileName.c_str())); + PARQUET_ASSIGN_OR_THROW( m_outputStream, arrow::io::FileOutputStream::Open( fileName.c_str() ) ); ::parquet::WriterProperties::Builder builder; - builder.compression( resolveCompression( compression )); - builder.version(::parquet::ParquetVersion::PARQUET_2_0 ); + builder.compression( resolveCompression( compression ) ); + builder.version( ::parquet::ParquetVersion::PARQUET_2_0 ); ::parquet::ArrowWriterProperties::Builder arrowBuilder; arrowBuilder.store_schema(); - STATUS_OK_OR_THROW_RUNTIME( - ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build(), - &m_fileWriter ), - "Failed to open parquet file writer" ); + STATUS_OK_OR_THROW_RUNTIME( ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), + m_outputStream, builder.build(), + arrowBuilder.build(), &m_fileWriter ), + "Failed to open parquet file writer" ); } void ParquetFileWriterWrapper::close() @@ -38,17 +36,18 @@ void ParquetFileWriterWrapper::close() m_outputStream = nullptr; m_fileWriter = nullptr; - if(fileWriter) + if( fileWriter ) STATUS_OK_OR_THROW_RUNTIME( fileWriter->Close(), "Failed to close parquet file writer" ); - if(outputStream) + if( outputStream ) STATUS_OK_OR_THROW_RUNTIME( outputStream->Close(), "Failed to close parquet output stream" ); } } -void ParquetFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> &table ) +void ParquetFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> & table ) { - CSP_TRUE_OR_THROW_RUNTIME(m_fileWriter, "File writer is null!!!"); - STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows()), "Failed to write to parquet file" ); + CSP_TRUE_OR_THROW_RUNTIME( m_fileWriter, "File writer is null!!!" ); + STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows() ), + "Failed to write to parquet file" ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.h b/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.h index b73f53456..51c07338f 100644 --- a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.h +++ b/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.h @@ -20,24 +20,23 @@ class ParquetFileWriterWrapper : public FileWriterWrapper { public: ParquetFileWriterWrapper( std::shared_ptr<::arrow::Schema> schema ) - : FileWriterWrapper( schema ) + : FileWriterWrapper( schema ) { } ~ParquetFileWriterWrapper() { close(); } void close() override; - void writeTable( const std::shared_ptr<::arrow::Table> &table ) override; + void writeTable( const std::shared_ptr<::arrow::Table> & table ) override; protected: - void openImpl( const std::string &fileName, const std::string &compression ) override; + void openImpl( const std::string & fileName, const std::string & compression ) override; private: std::shared_ptr<::arrow::io::FileOutputStream> m_outputStream; std::unique_ptr<::parquet::arrow::FileWriter> m_fileWriter; }; -} - +} // namespace csp::adapters::parquet #endif \ No newline at end of file diff --git a/cpp/csp/adapters/parquet/ParquetInputAdapterManager.cpp b/cpp/csp/adapters/parquet/ParquetInputAdapterManager.cpp index 4109830a8..7dc427bbb 100644 --- a/cpp/csp/adapters/parquet/ParquetInputAdapterManager.cpp +++ b/cpp/csp/adapters/parquet/ParquetInputAdapterManager.cpp @@ -1,32 +1,34 @@ +#include #include -#include #include #include -#include +#include namespace csp::adapters::parquet { -ParquetInputAdapterManager::ParquetInputAdapterManager( csp::Engine *engine, const Dictionary &properties, - GeneratorPtr generatorPtr, - TableGeneratorPtr tableGeneratorPtr) : - AdapterManager( engine ), - m_fileNameGeneratorReplicator( generatorPtr ? std::make_shared( generatorPtr ) : nullptr ), - m_time_shift(0, 0), - m_tableGenerator(tableGeneratorPtr), - m_reader() +ParquetInputAdapterManager::ParquetInputAdapterManager( csp::Engine * engine, const Dictionary & properties, + GeneratorPtr generatorPtr, TableGeneratorPtr tableGeneratorPtr ) + : AdapterManager( engine ) + , m_fileNameGeneratorReplicator( generatorPtr ? std::make_shared( generatorPtr ) + : nullptr ) + , m_time_shift( 0, 0 ) + , m_tableGenerator( tableGeneratorPtr ) + , m_reader() { - CSP_TRUE_OR_THROW_RUNTIME(!generatorPtr || !m_tableGenerator, "Trying to set both generatorPtr and tableGeneratorPtr"); - CSP_TRUE_OR_THROW_RUNTIME(generatorPtr || m_tableGenerator, "Either generatorPtr or tableGeneratorPtr must be set"); - - m_symbolColumn = properties.get( "symbol_column", "" ); - m_timeColumn = properties.get( "time_column", "" ); - m_defaultTimezone = properties.get( "tz", "UTC" ); - m_splitColumnsToFiles = properties.get( "split_columns_to_files" ); - m_isArrowIPC = properties.get( "is_arrow_ipc", false ); + CSP_TRUE_OR_THROW_RUNTIME( !generatorPtr || !m_tableGenerator, + "Trying to set both generatorPtr and tableGeneratorPtr" ); + CSP_TRUE_OR_THROW_RUNTIME( generatorPtr || m_tableGenerator, + "Either generatorPtr or tableGeneratorPtr must be set" ); + + m_symbolColumn = properties.get( "symbol_column", "" ); + m_timeColumn = properties.get( "time_column", "" ); + m_defaultTimezone = properties.get( "tz", "UTC" ); + m_splitColumnsToFiles = properties.get( "split_columns_to_files" ); + m_isArrowIPC = properties.get( "is_arrow_ipc", false ); m_allowOverlappingPeriods = properties.get( "allow_overlapping_periods", false ); - m_allowMissingColumns = properties.get( "allow_missing_columns", false ); - m_allowMissingFiles = properties.get( "allow_missing_files", false ); + m_allowMissingColumns = properties.get( "allow_missing_columns", false ); + m_allowMissingFiles = properties.get( "allow_missing_files", false ); properties.tryGet( "start_time", m_startTime ); properties.tryGet( "end_time", m_endTime ); properties.tryGet( "time_shift", m_time_shift ); @@ -36,25 +38,23 @@ ParquetInputAdapterManager::ParquetInputAdapterManager( csp::Engine *engine, con "Only UTC default timezone is supported, got:" << m_defaultTimezone ); } -ParquetInputAdapterManager::~ParquetInputAdapterManager() -{ -} +ParquetInputAdapterManager::~ParquetInputAdapterManager() {} void ParquetInputAdapterManager::start( DateTime starttime, DateTime endtime ) { if( !m_startTime.isNone() ) { - starttime = std::max(starttime, m_startTime); + starttime = std::max( starttime, m_startTime ); } AdapterManager::start( starttime, endtime ); CSP_TRUE_OR_THROW_RUNTIME( m_reader == nullptr, "Starting parquet adapter manager more than once" ); - if(m_fileNameGeneratorReplicator) + if( m_fileNameGeneratorReplicator ) { - m_fileNameGeneratorReplicator -> init( starttime, endtime ); + m_fileNameGeneratorReplicator->init( starttime, endtime ); } else { - m_tableGenerator->init(starttime, endtime); + m_tableGenerator->init( starttime, endtime ); } std::optional symbolColumn; @@ -67,52 +67,49 @@ void ParquetInputAdapterManager::start( DateTime starttime, DateTime endtime ) } neededColumns.insert( m_timeColumn ); - for( auto &&it : m_dictBasketInputAdapters ) + for( auto && it : m_dictBasketInputAdapters ) { neededColumns.insert( it.first + "__csp_value_count" ); } m_reader = initializeParquetReader( symbolColumn, neededColumns, m_simInputAdapters, true, true ); - if(m_reader == nullptr) + if( m_reader == nullptr ) { return; } - for( auto &&it : m_dictBasketInputAdapters ) + for( auto && it : m_dictBasketInputAdapters ) { - auto valueCountColumnName = it.first + "__csp_value_count"; - std::string basketSymbolColumn = it.first + +"__csp_symbol"; - DictBasketReaderRecord record{ ( *m_reader )[ valueCountColumnName ], nullptr }; - record.m_valueCountColumn -> ensureType( CspType::UINT16() ); + auto valueCountColumnName = it.first + "__csp_value_count"; + std::string basketSymbolColumn = it.first + +"__csp_symbol"; + DictBasketReaderRecord record{ ( *m_reader )[valueCountColumnName], nullptr }; + record.m_valueCountColumn->ensureType( CspType::UINT16() ); neededColumns.insert( valueCountColumnName ); - record.m_reader = initializeParquetReader( basketSymbolColumn, { basketSymbolColumn }, it.second, - false ); + record.m_reader = initializeParquetReader( basketSymbolColumn, { basketSymbolColumn }, it.second, false ); m_dictBasketReaders.push_back( std::move( record ) ); } - m_timestampColumnAdapter = ( *m_reader )[ m_timeColumn ]; + m_timestampColumnAdapter = ( *m_reader )[m_timeColumn]; CSP_TRUE_OR_THROW_RUNTIME( m_timestampColumnAdapter.valid(), "m_timestampColumnAdapter is NULL" ); - m_timestampColumnAdapter -> ensureType( CspType::DATETIME() ); - + m_timestampColumnAdapter->ensureType( CspType::DATETIME() ); } -std::unique_ptr ParquetInputAdapterManager::initializeParquetReader( const std::optional &symbolColumn, - const std::set &neededColumns, - const ParquetInputAdapterManager::AdaptersBySymbol &adaptersBySymbol, - bool subscribeAllOnEmptySymbol, - bool nullOnEmpty) const +std::unique_ptr ParquetInputAdapterManager::initializeParquetReader( + const std::optional & symbolColumn, const std::set & neededColumns, + const ParquetInputAdapterManager::AdaptersBySymbol & adaptersBySymbol, bool subscribeAllOnEmptySymbol, + bool nullOnEmpty ) const { std::set neededColumnsCopy{ neededColumns }; - for( auto &&adaptersForSymbol : adaptersBySymbol ) + for( auto && adaptersForSymbol : adaptersBySymbol ) { - for( auto &&columnAdapterEntryIt : adaptersForSymbol.second.m_adaptersByColumnName ) + for( auto && columnAdapterEntryIt : adaptersForSymbol.second.m_adaptersByColumnName ) { - neededColumnsCopy.insert(columnAdapterEntryIt.first); + neededColumnsCopy.insert( columnAdapterEntryIt.first ); } - for( auto &&structAdapterEntryIt : adaptersForSymbol.second.m_structAdapters ) + for( auto && structAdapterEntryIt : adaptersForSymbol.second.m_structAdapters ) { - auto &&fieldMap = structAdapterEntryIt.first.fieldMap(); - for( auto it = fieldMap -> begin(); it != fieldMap -> end(); ++it ) + auto && fieldMap = structAdapterEntryIt.first.fieldMap(); + for( auto it = fieldMap->begin(); it != fieldMap->end(); ++it ) { neededColumnsCopy.insert( it.key() ); } @@ -124,69 +121,72 @@ std::unique_ptr ParquetInputAdapterManager::initializeParquetRead if( m_splitColumnsToFiles ) { - CSP_TRUE_OR_THROW_RUNTIME(m_fileNameGeneratorReplicator, "Trying to read split columns from file while reading in memory tables"); - reader.reset( new MultipleFileParquetReader( m_fileNameGeneratorReplicator, columns, m_isArrowIPC, m_allowMissingColumns, symbolColumn ) ); + CSP_TRUE_OR_THROW_RUNTIME( m_fileNameGeneratorReplicator, + "Trying to read split columns from file while reading in memory tables" ); + reader.reset( new MultipleFileParquetReader( m_fileNameGeneratorReplicator, columns, m_isArrowIPC, + m_allowMissingColumns, symbolColumn ) ); } else { - if(m_fileNameGeneratorReplicator) + if( m_fileNameGeneratorReplicator ) { - reader.reset( new SingleFileParquetReader( m_fileNameGeneratorReplicator -> getGeneratorReplica(), columns, m_isArrowIPC, - m_allowMissingColumns, m_allowMissingFiles, symbolColumn ) ); + reader.reset( new SingleFileParquetReader( m_fileNameGeneratorReplicator->getGeneratorReplica(), columns, + m_isArrowIPC, m_allowMissingColumns, m_allowMissingFiles, + symbolColumn ) ); } else { - reader.reset( new InMemoryTableParquetReader( m_tableGenerator, columns, m_allowMissingColumns, symbolColumn ) ); + reader.reset( + new InMemoryTableParquetReader( m_tableGenerator, columns, m_allowMissingColumns, symbolColumn ) ); } } - if(!reader->isEmpty()) + if( !reader->isEmpty() ) { - for( auto &symMapPair : adaptersBySymbol ) + for( auto & symMapPair : adaptersBySymbol ) { std::optional symbol; - if( !subscribeAllOnEmptySymbol || - !std::holds_alternative( symMapPair.first ) || - ( std::holds_alternative( symMapPair.first ) && - std::get( symMapPair.first ) != "" ) ) + if( !subscribeAllOnEmptySymbol || !std::holds_alternative( symMapPair.first ) + || ( std::holds_alternative( symMapPair.first ) + && std::get( symMapPair.first ) != "" ) ) { symbol = symMapPair.first; } - for( auto &columnAdapterPair:symMapPair.second.m_adaptersByColumnName ) + for( auto & columnAdapterPair : symMapPair.second.m_adaptersByColumnName ) { - auto &&columnAdapter = ( *reader )[ columnAdapterPair.first ]; - auto &&listReaderInterface = columnAdapterPair.second.m_listReaderInterface; + auto && columnAdapter = ( *reader )[columnAdapterPair.first]; + auto && listReaderInterface = columnAdapterPair.second.m_listReaderInterface; CSP_TRUE_OR_THROW_RUNTIME( !columnAdapter->isListType() || listReaderInterface != nullptr, "Column " << columnAdapterPair.first << " is a list column in parquet file " - << reader -> getCurFileOrTableName() << " while subscribing as non list" ); - CSP_TRUE_OR_THROW_RUNTIME( columnAdapter->isListType() || listReaderInterface == nullptr, - "Column " << columnAdapterPair.first << " is a list is non list in parquet file " - << reader -> getCurFileOrTableName() << " while subscribing to it as list" ); - - if(columnAdapter->isListType()) + << reader->getCurFileOrTableName() + << " while subscribing as non list" ); + CSP_TRUE_OR_THROW_RUNTIME( + columnAdapter->isListType() || listReaderInterface == nullptr, + "Column " << columnAdapterPair.first << " is a list is non list in parquet file " + << reader->getCurFileOrTableName() << " while subscribing to it as list" ); + + if( columnAdapter->isListType() ) { - reader->addListSubscriber(columnAdapterPair.first, columnAdapterPair.second.m_adapter, - symbol, listReaderInterface); - + reader->addListSubscriber( columnAdapterPair.first, columnAdapterPair.second.m_adapter, symbol, + listReaderInterface ); } else { - reader -> addSubscriber( columnAdapterPair.first, columnAdapterPair.second.m_adapter, - symbol ); + reader->addSubscriber( columnAdapterPair.first, columnAdapterPair.second.m_adapter, symbol ); } } - for( auto &structAdapterPair:symMapPair.second.m_structAdapters ) + for( auto & structAdapterPair : symMapPair.second.m_structAdapters ) { CSP_TRUE_OR_THROW_RUNTIME( structAdapterPair.second.m_listReaderInterface == nullptr, "Struct adapter is not expected to have list reader interface set" ); - reader -> getStructAdapter( structAdapterPair.first ).addSubscriber( - structAdapterPair.second.m_adapter, symbol ); + reader->getStructAdapter( structAdapterPair.first ) + .addSubscriber( structAdapterPair.second.m_adapter, symbol ); } } } - if(!reader->start() && nullOnEmpty) + if( !reader->start() && nullOnEmpty ) { - return std::unique_ptr(nullptr); + return std::unique_ptr( nullptr ); } return reader; } @@ -197,33 +197,34 @@ void ParquetInputAdapterManager::stop() m_dictBasketReaders.clear(); m_timestampColumnAdapter = nullptr; m_fileNameGeneratorReplicator = nullptr; - m_tableGenerator = nullptr; + m_tableGenerator = nullptr; AdapterManager::stop(); } DateTime ParquetInputAdapterManager::processNextSimTimeSlice( DateTime time ) { - if( unlikely( !m_reader || !m_reader -> hasData() ) ) + if( unlikely( !m_reader || !m_reader->hasData() ) ) { return DateTime::NONE(); } auto data_reference_time = time - m_time_shift; - auto nextDataTime = m_timestampColumnAdapter -> getCurValue(); + auto nextDataTime = m_timestampColumnAdapter->getCurValue(); while( !nextDataTime.value().isNone() && nextDataTime.value() < data_reference_time ) { - for( auto &&dictBasketRecord:m_dictBasketReaders ) + for( auto && dictBasketRecord : m_dictBasketReaders ) { - auto numValuesToSkip = dictBasketRecord.m_valueCountColumn -> getCurValue().value(); - dictBasketRecord.m_reader -> skipRows( numValuesToSkip ); + auto numValuesToSkip = dictBasketRecord.m_valueCountColumn->getCurValue().value(); + dictBasketRecord.m_reader->skipRows( numValuesToSkip ); } - if(!m_reader -> skipRow()) + if( !m_reader->skipRow() ) { nextDataTime = DateTime::NONE(); break; } - nextDataTime = m_timestampColumnAdapter -> getCurValue(); + nextDataTime = m_timestampColumnAdapter->getCurValue(); } - if( unlikely( nextDataTime.value().isNone() || ( !m_endTime.isNone() && ( m_endTime - m_time_shift ) < nextDataTime ) ) ) + if( unlikely( nextDataTime.value().isNone() + || ( !m_endTime.isNone() && ( m_endTime - m_time_shift ) < nextDataTime ) ) ) { return DateTime::NONE(); } @@ -233,43 +234,44 @@ DateTime ParquetInputAdapterManager::processNextSimTimeSlice( DateTime time ) return nextDataTime.value() + m_time_shift; } - CSP_TRUE_OR_THROW_RUNTIME( data_reference_time == nextDataTime, "Expected time " << nextDataTime.value() << " got " << data_reference_time ); + CSP_TRUE_OR_THROW_RUNTIME( data_reference_time == nextDataTime, + "Expected time " << nextDataTime.value() << " got " << data_reference_time ); do { - for( auto &&dictBasketRecord:m_dictBasketReaders ) + for( auto && dictBasketRecord : m_dictBasketReaders ) { - auto numValuesToDispatch = dictBasketRecord.m_valueCountColumn -> getCurValue().value(); + auto numValuesToDispatch = dictBasketRecord.m_valueCountColumn->getCurValue().value(); for( uint16_t i = 0; i < numValuesToDispatch; ++i ) { - dictBasketRecord.m_reader -> dispatchRow(); + dictBasketRecord.m_reader->dispatchRow(); } } - m_reader -> dispatchRow(); + m_reader->dispatchRow(); - nextDataTime = m_reader -> hasData() ? m_timestampColumnAdapter -> getCurValue() : DateTime::NONE(); + nextDataTime = m_reader->hasData() ? m_timestampColumnAdapter->getCurValue() : DateTime::NONE(); } while( !nextDataTime.value().isNone() && nextDataTime == data_reference_time ); - if( unlikely( nextDataTime -> isNone() ) ) + if( unlikely( nextDataTime->isNone() ) ) { return DateTime::NONE(); } - if(m_allowOverlappingPeriods && nextDataTime.value() < data_reference_time) + if( m_allowOverlappingPeriods && nextDataTime.value() < data_reference_time ) { return time + TimeDelta( 0, 1 ); } return nextDataTime.value() + m_time_shift; } - -ManagedSimInputAdapter *ParquetInputAdapterManager::getInputAdapter( CspTypePtr &type, const Dictionary &properties, PushMode pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface ) +ManagedSimInputAdapter * +ParquetInputAdapterManager::getInputAdapter( CspTypePtr & type, const Dictionary & properties, PushMode pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { CSP_TRUE_OR_THROW( !m_pushMode.has_value() || m_pushMode.value() == pushMode, NotImplemented, - "Subscribing with varying push modes is not currently supported. previous=" << m_pushMode.value() - << " current=" << pushMode ); - m_pushMode = pushMode; + "Subscribing with varying push modes is not currently supported. previous=" + << m_pushMode.value() << " current=" << pushMode ); + m_pushMode = pushMode; std::string basketName = properties.get( "basket_name", "" ); utils::Symbol symbol = ""; @@ -290,19 +292,20 @@ ManagedSimInputAdapter *ParquetInputAdapterManager::getInputAdapter( CspTypePtr } else { - CSP_TRUE_OR_THROW(listReaderInterface == nullptr, NotImplemented, "Reading of baskets of arrays is unsupported"); + CSP_TRUE_OR_THROW( listReaderInterface == nullptr, NotImplemented, + "Reading of baskets of arrays is unsupported" ); return getDictBasketAdapter( type, properties, pushMode, symbol, basketName ); } } ManagedSimInputAdapter * -ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr &type, const Dictionary &properties, const PushMode &pushMode, - const utils::Symbol &symbol, - const DialectGenericListReaderInterface::Ptr &listReaderInterface ) +ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr & type, const Dictionary & properties, + const PushMode & pushMode, const utils::Symbol & symbol, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { if( pushMode == PushMode::NON_COLLAPSING ) { - if( std::holds_alternative( symbol) && std::get( symbol ).empty() ) + if( std::holds_alternative( symbol ) && std::get( symbol ).empty() ) { m_subscribedForAll = true; } @@ -310,16 +313,15 @@ ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr &type, const Dic { m_subscribedBySymbol = true; } - // If we subscribe both by symbol and subscribe all, we might have weird issues. Consider the following scenario: - // ts1: subscribe AAPL - // ts2: subscribe IBM - // ts3: subscribe_all - // Assume that the file has 2 entries, both on the same timestamp, one for AAPL, one for IBM. - // Then ts1 and ts2 will output on the first cycle the AAPL and IBM data, ts3 will output data for AAPL only, and on next - // cycle will output the data for IBM. This behavior of same data ticking on different cycles could create a lot of issues - // and generally buggy so we want to avoid it and forbid subscribing both ways. - CSP_TRUE_OR_THROW( !m_subscribedBySymbol || !m_subscribedForAll, NotImplemented, - "Subscribing both by symbol and without symbol for same parquet reader is not currently supported" ); + // If we subscribe both by symbol and subscribe all, we might have weird issues. Consider the following + // scenario: ts1: subscribe AAPL ts2: subscribe IBM ts3: subscribe_all Assume that the file has 2 entries, both + // on the same timestamp, one for AAPL, one for IBM. Then ts1 and ts2 will output on the first cycle the AAPL + // and IBM data, ts3 will output data for AAPL only, and on next cycle will output the data for IBM. This + // behavior of same data ticking on different cycles could create a lot of issues and generally buggy so we want + // to avoid it and forbid subscribing both ways. + CSP_TRUE_OR_THROW( + !m_subscribedBySymbol || !m_subscribedForAll, NotImplemented, + "Subscribing both by symbol and without symbol for same parquet reader is not currently supported" ); } auto fieldMap = properties.getUntypedValue( "field_map" ); @@ -330,7 +332,8 @@ ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr &type, const Dic } else if( std::holds_alternative( fieldMap ) ) { - CSP_TRUE_OR_THROW(listReaderInterface == nullptr, NotImplemented, "Reading of arrays of structs is unsupported"); + CSP_TRUE_OR_THROW( listReaderInterface == nullptr, NotImplemented, + "Reading of arrays of structs is unsupported" ); auto dictFieldMap = properties.get( "field_map" ); return getStructAdapter( type, symbol, dictFieldMap, pushMode ); } @@ -342,14 +345,15 @@ ParquetInputAdapterManager::getRegularAdapter( const CspTypePtr &type, const Dic CSP_THROW( RuntimeException, "Reached unreachable code" ); } -ManagedSimInputAdapter *ParquetInputAdapterManager::getDictBasketAdapter( const CspTypePtr &type, - const Dictionary &properties, const PushMode &pushMode, - const utils::Symbol &symbol, - const std::string &basketName ) +ManagedSimInputAdapter * ParquetInputAdapterManager::getDictBasketAdapter( const CspTypePtr & type, + const Dictionary & properties, + const PushMode & pushMode, + const utils::Symbol & symbol, + const std::string & basketName ) { auto fieldMap = properties.getUntypedValue( "field_map" ); - auto &&basketRecord = m_dictBasketInputAdapters[ basketName ]; + auto && basketRecord = m_dictBasketInputAdapters[basketName]; if( std::holds_alternative( fieldMap ) ) { @@ -370,11 +374,10 @@ ManagedSimInputAdapter *ParquetInputAdapterManager::getDictBasketAdapter( const CSP_THROW( RuntimeException, "Reached unreachable code" ); } -ManagedSimInputAdapter * -ParquetInputAdapterManager::getOrCreateSingleColumnAdapter( ParquetInputAdapterManager::AdaptersBySymbol &inputAdaptersContainer, - const CspTypePtr &type, const utils::Symbol &symbol, const std::string &field, - const PushMode &pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface ) +ManagedSimInputAdapter * ParquetInputAdapterManager::getOrCreateSingleColumnAdapter( + ParquetInputAdapterManager::AdaptersBySymbol & inputAdaptersContainer, const CspTypePtr & type, + const utils::Symbol & symbol, const std::string & field, const PushMode & pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { auto itBySymbol = inputAdaptersContainer.find( symbol ); if( itBySymbol == inputAdaptersContainer.end() ) @@ -382,33 +385,32 @@ ParquetInputAdapterManager::getOrCreateSingleColumnAdapter( ParquetInputAdapterM itBySymbol = inputAdaptersContainer.emplace( symbol, AdaptersSingleSymbol() ).first; } - auto itByColumn = itBySymbol -> second.m_adaptersByColumnName.find( field ); - const CspTypePtr& adapterType = (listReaderInterface==nullptr) ? type : CspType::DIALECT_GENERIC(); + auto itByColumn = itBySymbol->second.m_adaptersByColumnName.find( field ); + const CspTypePtr & adapterType = ( listReaderInterface == nullptr ) ? type : CspType::DIALECT_GENERIC(); - if( itByColumn == itBySymbol -> second.m_adaptersByColumnName.end() ) + if( itByColumn == itBySymbol->second.m_adaptersByColumnName.end() ) { - itByColumn = itBySymbol -> second.m_adaptersByColumnName.emplace( - field, AdapterInfo{ engine() -> createOwnedObject( - adapterType, this, - pushMode ), listReaderInterface } ).first; + itByColumn = itBySymbol->second.m_adaptersByColumnName + .emplace( field, + AdapterInfo{ engine()->createOwnedObject( adapterType, this, + pushMode ), + listReaderInterface } ) + .first; } - return itByColumn -> second.m_adapter; + return itByColumn->second.m_adapter; } ManagedSimInputAdapter * -ParquetInputAdapterManager::getSingleColumnAdapter( const CspTypePtr &type, const utils::Symbol &symbol, - const std::string &field, PushMode pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface) +ParquetInputAdapterManager::getSingleColumnAdapter( const CspTypePtr & type, const utils::Symbol & symbol, + const std::string & field, PushMode pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { - return getOrCreateSingleColumnAdapter( m_simInputAdapters, type, symbol, field, - pushMode, listReaderInterface ); + return getOrCreateSingleColumnAdapter( m_simInputAdapters, type, symbol, field, pushMode, listReaderInterface ); } - -ManagedSimInputAdapter *ParquetInputAdapterManager::getOrCreateStructColumnAdapter( AdaptersBySymbol &inputAdaptersContainer, - const CspTypePtr &type, const utils::Symbol &symbol, - const csp::DictionaryPtr &fieldMap, - const PushMode &pushMode ) +ManagedSimInputAdapter * ParquetInputAdapterManager::getOrCreateStructColumnAdapter( + AdaptersBySymbol & inputAdaptersContainer, const CspTypePtr & type, const utils::Symbol & symbol, + const csp::DictionaryPtr & fieldMap, const PushMode & pushMode ) { auto itBySymbol = inputAdaptersContainer.find( symbol ); if( itBySymbol == inputAdaptersContainer.end() ) @@ -418,25 +420,23 @@ ManagedSimInputAdapter *ParquetInputAdapterManager::getOrCreateStructColumnAdapt StructAdapterInfo key{ type, fieldMap }; - auto itByColumn = itBySymbol -> second.m_structAdapters.find( key ); + auto itByColumn = itBySymbol->second.m_structAdapters.find( key ); - if( itByColumn == itBySymbol -> second.m_structAdapters.end() ) + if( itByColumn == itBySymbol->second.m_structAdapters.end() ) { - itByColumn = itBySymbol -> second.m_structAdapters.emplace( - key, engine() -> createOwnedObject( - type, this, - pushMode ) ).first; + itByColumn = itBySymbol->second.m_structAdapters + .emplace( key, engine()->createOwnedObject( type, this, pushMode ) ) + .first; } - return itByColumn -> second.m_adapter; - + return itByColumn->second.m_adapter; } -ManagedSimInputAdapter * -ParquetInputAdapterManager::getStructAdapter( const CspTypePtr &type, const utils::Symbol &symbol, - const csp::DictionaryPtr &fieldMap, PushMode pushMode ) +ManagedSimInputAdapter * ParquetInputAdapterManager::getStructAdapter( const CspTypePtr & type, + const utils::Symbol & symbol, + const csp::DictionaryPtr & fieldMap, + PushMode pushMode ) { return getOrCreateStructColumnAdapter( m_simInputAdapters, type, symbol, fieldMap, pushMode ); } - -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetInputAdapterManager.h b/cpp/csp/adapters/parquet/ParquetInputAdapterManager.h index baa67e19c..444963888 100644 --- a/cpp/csp/adapters/parquet/ParquetInputAdapterManager.h +++ b/cpp/csp/adapters/parquet/ParquetInputAdapterManager.h @@ -1,63 +1,66 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetInputAdapterManager_H #define _IN_CSP_ADAPTERS_PARQUET_ParquetInputAdapterManager_H +#include #include #include #include #include #include -#include +#include #include #include -#include -#include - +#include namespace csp::adapters::parquet { - -//Top level AdapterManager object for all parquet adapters in the engine +// Top level AdapterManager object for all parquet adapters in the engine class ParquetInputAdapterManager final : public csp::AdapterManager { public: - using GeneratorPtr = csp::Generator::Ptr; + using GeneratorPtr = csp::Generator::Ptr; using TableGeneratorPtr = csp::Generator, csp::DateTime, csp::DateTime>::Ptr; - ParquetInputAdapterManager( csp::Engine *engine, const Dictionary &properties, GeneratorPtr generatorPtr, TableGeneratorPtr tableGeneratorPtr ); + ParquetInputAdapterManager( csp::Engine * engine, const Dictionary & properties, GeneratorPtr generatorPtr, + TableGeneratorPtr tableGeneratorPtr ); ~ParquetInputAdapterManager(); - const char *name() const override{ return "ParquetInputAdapterManager"; } + const char * name() const override { return "ParquetInputAdapterManager"; } - //start subscriptions and processing thread + // start subscriptions and processing thread void start( DateTime starttime, DateTime endtime ) override; - //stop subscriptions and processing thread + // stop subscriptions and processing thread void stop() override; DateTime processNextSimTimeSlice( DateTime time ) override; - ManagedSimInputAdapter *getInputAdapter( CspTypePtr &type, const Dictionary &properties, PushMode pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface = nullptr ); + ManagedSimInputAdapter * getInputAdapter( CspTypePtr & type, const Dictionary & properties, PushMode pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface + = nullptr ); + private: using StructAdapterInfo = csp::adapters::utils::StructAdapterInfo; struct AdapterInfo { - AdapterInfo(ManagedSimInputAdapter* adapter, const DialectGenericListReaderInterface::Ptr listReaderInterface = nullptr) - : m_adapter(adapter), m_listReaderInterface(listReaderInterface) + AdapterInfo( ManagedSimInputAdapter * adapter, + const DialectGenericListReaderInterface::Ptr listReaderInterface = nullptr ) + : m_adapter( adapter ) + , m_listReaderInterface( listReaderInterface ) { } - ManagedSimInputAdapter * m_adapter; + ManagedSimInputAdapter * m_adapter; DialectGenericListReaderInterface::Ptr m_listReaderInterface = nullptr; }; struct AdaptersSingleSymbol { - using AdaptersByColumnName = std::unordered_map; + using AdaptersByColumnName = std::unordered_map; using StructAdaptersByStructKey = std::unordered_map; - AdaptersByColumnName m_adaptersByColumnName; + AdaptersByColumnName m_adaptersByColumnName; StructAdaptersByStructKey m_structAdapters; }; @@ -67,40 +70,41 @@ class ParquetInputAdapterManager final : public csp::AdapterManager std::unique_ptr m_reader; }; - using AdaptersBySymbol = std::unordered_map; + using AdaptersBySymbol = std::unordered_map; using DictBasketSymbolAdapters = std::unordered_map; - std::unique_ptr initializeParquetReader( const std::optional &symbolColumn, - const std::set &neededColumns, - const AdaptersBySymbol &adaptersBySymbol, + std::unique_ptr initializeParquetReader( const std::optional & symbolColumn, + const std::set & neededColumns, + const AdaptersBySymbol & adaptersBySymbol, bool subscribeAllOnEmptySymbol = true, - bool nullOnEmpty = false ) const; - - ManagedSimInputAdapter *getRegularAdapter( const CspTypePtr &type, - const Dictionary &properties, const PushMode &pushMode, const utils::Symbol &symbol, - const DialectGenericListReaderInterface::Ptr &listReaderInterface = nullptr); - ManagedSimInputAdapter *getDictBasketAdapter( const CspTypePtr &type, - const Dictionary &properties, const PushMode &pushMode, const utils::Symbol &symbol, - const std::string &basketName ); - - ManagedSimInputAdapter *getOrCreateSingleColumnAdapter( AdaptersBySymbol &inputAdaptersContainer, - const CspTypePtr &type, const utils::Symbol &symbol, - const std::string &field, const PushMode &pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface = nullptr ); - ManagedSimInputAdapter *getSingleColumnAdapter( const CspTypePtr &type, - const utils::Symbol &symbol, const std::string &field, PushMode pushMode, - const DialectGenericListReaderInterface::Ptr &listReaderInterface = nullptr); - ManagedSimInputAdapter *getOrCreateStructColumnAdapter( AdaptersBySymbol &inputAdaptersContainer, - const CspTypePtr &type, const utils::Symbol &symbol, - const csp::DictionaryPtr &fieldMap, const PushMode &pushMode ); - ManagedSimInputAdapter *getStructAdapter( const CspTypePtr &type, const utils::Symbol &symbol, - const csp::DictionaryPtr &fieldMap, PushMode pushMode ); - + bool nullOnEmpty = false ) const; + + ManagedSimInputAdapter * getRegularAdapter( const CspTypePtr & type, const Dictionary & properties, + const PushMode & pushMode, const utils::Symbol & symbol, + const DialectGenericListReaderInterface::Ptr & listReaderInterface + = nullptr ); + ManagedSimInputAdapter * getDictBasketAdapter( const CspTypePtr & type, const Dictionary & properties, + const PushMode & pushMode, const utils::Symbol & symbol, + const std::string & basketName ); + + ManagedSimInputAdapter * + getOrCreateSingleColumnAdapter( AdaptersBySymbol & inputAdaptersContainer, const CspTypePtr & type, + const utils::Symbol & symbol, const std::string & field, const PushMode & pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface = nullptr ); + ManagedSimInputAdapter * getSingleColumnAdapter( const CspTypePtr & type, const utils::Symbol & symbol, + const std::string & field, PushMode pushMode, + const DialectGenericListReaderInterface::Ptr & listReaderInterface + = nullptr ); + ManagedSimInputAdapter * getOrCreateStructColumnAdapter( AdaptersBySymbol & inputAdaptersContainer, + const CspTypePtr & type, const utils::Symbol & symbol, + const csp::DictionaryPtr & fieldMap, + const PushMode & pushMode ); + ManagedSimInputAdapter * getStructAdapter( const CspTypePtr & type, const utils::Symbol & symbol, + const csp::DictionaryPtr & fieldMap, PushMode pushMode ); DictBasketSymbolAdapters m_dictBasketInputAdapters; AdaptersBySymbol m_simInputAdapters; - FileNameGeneratorReplicator::Ptr m_fileNameGeneratorReplicator; csp::DateTime m_startTime; csp::DateTime m_endTime; @@ -118,12 +122,10 @@ class ParquetInputAdapterManager final : public csp::AdapterManager ColumnAdapterReference m_timestampColumnAdapter; std::vector m_dictBasketReaders; std::optional m_pushMode; - bool m_subscribedBySymbol = false; - bool m_subscribedForAll = false; - - + bool m_subscribedBySymbol = false; + bool m_subscribedForAll = false; }; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp index f73092254..ba6d3ae93 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp +++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp @@ -12,12 +12,11 @@ uint32_t ParquetOutputHandler::getChunkSize() const return m_parquetWriter.getChunkSize(); } - -SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, - std::string columnName ) - : ParquetOutputHandler( parquetWriter, type ) +SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine * engine, ParquetWriter & parquetWriter, + CspTypePtr & type, std::string columnName ) + : ParquetOutputHandler( parquetWriter, type ) { - switch( m_type -> type() ) + switch( m_type->type() ) { case CspType::TypeTraits::BOOL: createColumnBuilder( columnName ); @@ -58,13 +57,13 @@ SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engi case CspType::TypeTraits::DATE: createColumnBuilder( columnName ); break; - case CspType::TypeTraits::TIME: - createColumnBuilder( columnName ); - break; + case CspType::TypeTraits::TIME: + createColumnBuilder( columnName ); + break; case CspType::TypeTraits::STRING: { - const CspStringType &strType = static_cast(*type); + const CspStringType & strType = static_cast( *type ); if( strType.isBytes() ) { createColumnBuilder( columnName ); @@ -77,39 +76,39 @@ SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engi } case CspType::TypeTraits::ENUM: { - auto enumMetaPtr = std::static_pointer_cast( type ) -> meta(); + auto enumMetaPtr = std::static_pointer_cast( type )->meta(); createEnumColumnBuilder( columnName, enumMetaPtr ); break; } default: { - CSP_THROW( TypeError, "Writing of " << m_type -> type().asString() << " to parquet is not supported" ); + CSP_THROW( TypeError, "Writing of " << m_type->type().asString() << " to parquet is not supported" ); } } } - -template< typename ColumnBuilder > -inline void SingleColumnParquetOutputHandler::createColumnBuilder( const std::string &columnName ) +template +inline void SingleColumnParquetOutputHandler::createColumnBuilder( const std::string & columnName ) { m_columnArrayBuilder = std::make_unique( columnName, getChunkSize() ); m_valueHandler = std::make_unique( - [ this ]( const TimeSeriesProvider *input ) - { - static_cast(this -> m_columnArrayBuilder.get()) - -> setValue( input -> lastValueTyped() ); - } ); + [this]( const TimeSeriesProvider * input ) + { + static_cast( this->m_columnArrayBuilder.get() ) + ->setValue( input->lastValueTyped() ); + } ); } -void SingleColumnParquetOutputHandler::createEnumColumnBuilder( const std::string &columnName, CspEnumMeta::Ptr enumMetaPtr ) +void SingleColumnParquetOutputHandler::createEnumColumnBuilder( const std::string & columnName, + CspEnumMeta::Ptr enumMetaPtr ) { m_columnArrayBuilder = std::make_unique( columnName, getChunkSize() ); m_valueHandler = std::make_unique( - [ this ]( const TimeSeriesProvider *input ) - { - static_cast(this -> m_columnArrayBuilder.get()) - -> setValue( input -> lastValueTyped().name() ); - } ); + [this]( const TimeSeriesProvider * input ) + { + static_cast( this->m_columnArrayBuilder.get() ) + ->setValue( input->lastValueTyped().name() ); + } ); } void SingleColumnParquetOutputAdapter::executeImpl() @@ -118,58 +117,62 @@ void SingleColumnParquetOutputAdapter::executeImpl() m_parquetWriter.scheduleEndCycleEvent(); } -ListColumnParquetOutputHandler::ListColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &elemType, - const std::string &columnName, - DialectGenericListWriterInterface::Ptr &listWriterInterface ) - : ParquetOutputHandler( parquetWriter, CspType::DIALECT_GENERIC() ), - m_columnArrayBuilder( - std::make_shared( columnName, getChunkSize(), createValueBuilder( elemType, listWriterInterface ), - listWriterInterface ) ) +ListColumnParquetOutputHandler::ListColumnParquetOutputHandler( + Engine * engine, ParquetWriter & parquetWriter, CspTypePtr & elemType, const std::string & columnName, + DialectGenericListWriterInterface::Ptr & listWriterInterface ) + : ParquetOutputHandler( parquetWriter, CspType::DIALECT_GENERIC() ) + , m_columnArrayBuilder( std::make_shared( + columnName, getChunkSize(), createValueBuilder( elemType, listWriterInterface ), listWriterInterface ) ) { m_valueHandler = std::make_unique( - [ this ]( const TimeSeriesProvider *input ) - { - static_cast(this -> m_columnArrayBuilder.get()) - -> setValue( input -> lastValueTyped() ); - } ); + [this]( const TimeSeriesProvider * input ) + { + static_cast( this->m_columnArrayBuilder.get() ) + ->setValue( input->lastValueTyped() ); + } ); } namespace { -template< typename A, typename V = typename A::value_type > -inline std::shared_ptr<::arrow::ArrayBuilder> makeArrayAndAttachToWriter( DialectGenericListWriterInterface::Ptr &listWriterInterface ) -{ - auto&& typedWriter = std::dynamic_pointer_cast>( listWriterInterface ); - auto& listWriterInterfaceRef = *listWriterInterface; - CSP_TRUE_OR_THROW( typedWriter != nullptr, TypeError, - "Expected " << typeid( TypedDialectGenericListWriterInterface ).name() << " " << " got " << - typeid( listWriterInterfaceRef ).name() ); - - auto res = std::make_shared(); - typedWriter -> setWriteFunction( - [ res ]( const V &value ){ STATUS_OK_OR_THROW_RUNTIME( res -> Append( value ), "Failed to append value to list array" ); } ); - return res; -} + template + inline std::shared_ptr<::arrow::ArrayBuilder> + makeArrayAndAttachToWriter( DialectGenericListWriterInterface::Ptr & listWriterInterface ) + { + auto && typedWriter + = std::dynamic_pointer_cast>( listWriterInterface ); + auto & listWriterInterfaceRef = *listWriterInterface; + CSP_TRUE_OR_THROW( typedWriter != nullptr, TypeError, + "Expected " << typeid( TypedDialectGenericListWriterInterface ).name() << " " << " got " + << typeid( listWriterInterfaceRef ).name() ); + + auto res = std::make_shared(); + typedWriter->setWriteFunction( + [res]( const V & value ) + { STATUS_OK_OR_THROW_RUNTIME( res->Append( value ), "Failed to append value to list array" ); } ); + return res; + } -} +} // namespace -std::shared_ptr<::arrow::ArrayBuilder> ListColumnParquetOutputHandler::createValueBuilder( const CspTypePtr &elemType, - DialectGenericListWriterInterface::Ptr &listWriterInterface ) +std::shared_ptr<::arrow::ArrayBuilder> +ListColumnParquetOutputHandler::createValueBuilder( const CspTypePtr & elemType, + DialectGenericListWriterInterface::Ptr & listWriterInterface ) { - switch( elemType -> type() ) + switch( elemType->type() ) { case CspType::TypeTraits::BOOL: - return makeArrayAndAttachToWriter(listWriterInterface); + return makeArrayAndAttachToWriter( listWriterInterface ); case CspType::TypeTraits::INT64: - return makeArrayAndAttachToWriter(listWriterInterface); + return makeArrayAndAttachToWriter( listWriterInterface ); case CspType::TypeTraits::DOUBLE: - return makeArrayAndAttachToWriter(listWriterInterface); + return makeArrayAndAttachToWriter( listWriterInterface ); case CspType::TypeTraits::STRING: - return makeArrayAndAttachToWriter(listWriterInterface); + return makeArrayAndAttachToWriter( listWriterInterface ); default: { CSP_THROW( TypeError, - "Writing of list with elements of type " << elemType -> type().asString() << " to parquet is not supported" ); + "Writing of list with elements of type " << elemType->type().asString() + << " to parquet is not supported" ); } } } @@ -180,36 +183,35 @@ void ListColumnParquetOutputAdapter::executeImpl() m_parquetWriter.scheduleEndCycleEvent(); } -StructParquetOutputHandler::StructParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, - DictionaryPtr fieldMap ) - : ParquetOutputHandler( parquetWriter, type ) +StructParquetOutputHandler::StructParquetOutputHandler( Engine * engine, ParquetWriter & parquetWriter, + CspTypePtr & type, DictionaryPtr fieldMap ) + : ParquetOutputHandler( parquetWriter, type ) { - auto structMetaPtr = std::static_pointer_cast( type ) -> meta().get(); + auto structMetaPtr = std::static_pointer_cast( type )->meta().get(); - for( auto it = fieldMap -> begin(); it != fieldMap -> end(); ++it ) + for( auto it = fieldMap->begin(); it != fieldMap->end(); ++it ) { createColumnBuilder( structMetaPtr, it.value(), it.key(), nullptr ); } } -void StructParquetOutputHandler::writeValueFromTs( const TimeSeriesProvider *input ) +void StructParquetOutputHandler::writeValueFromTs( const TimeSeriesProvider * input ) { - const Struct *structData = input -> lastValueTyped().get(); + const Struct * structData = input->lastValueTyped().get(); - for( auto &&valueHandler: m_valueHandlers ) + for( auto && valueHandler : m_valueHandlers ) { valueHandler( structData ); } m_parquetWriter.scheduleEndCycleEvent(); } -inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createColumnBuilder( - const StructMeta *structMeta, - const std::string &columnName, const std::string &structFieldName, - const std::string *path ) +inline StructParquetOutputHandler::ColumnBuilderResultType +StructParquetOutputHandler::createColumnBuilder( const StructMeta * structMeta, const std::string & columnName, + const std::string & structFieldName, const std::string * path ) { - auto fieldPtr = structMeta -> field( structFieldName ).get(); - switch( fieldPtr -> type() -> type() ) + auto fieldPtr = structMeta->field( structFieldName ).get(); + switch( fieldPtr->type()->type() ) { case CspType::TypeTraits::BOOL: return createColumnBuilder( fieldPtr, columnName, path ); @@ -246,32 +248,32 @@ inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHa case CspType::TypeTraits::STRUCT: return createStructColumnBuilder( fieldPtr, columnName, path ); default: - CSP_THROW( TypeError, "Writing of column " << columnName << " of type " << fieldPtr -> type() -> type().asString() - << " to parquet is not supported" ); + CSP_THROW( TypeError, + "Writing of column " << columnName << " of type " << fieldPtr->type()->type().asString() + << " to parquet is not supported" ); } } -template< typename ColumnBuilder > -inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createColumnBuilder( - const StructField *field, - const std::string &columnName, - const std::string *path ) +template +inline StructParquetOutputHandler::ColumnBuilderResultType +StructParquetOutputHandler::createColumnBuilder( const StructField * field, const std::string & columnName, + const std::string * path ) { - std::shared_ptr columnBuilderPtr = std::make_shared( resolveFullColumnName( path, columnName ), - getChunkSize() ); + std::shared_ptr columnBuilderPtr + = std::make_shared( resolveFullColumnName( path, columnName ), getChunkSize() ); auto columnBuilderRawPtr = columnBuilderPtr.get(); - using T = typename ColumnBuilder::ValueTypeT; + using T = typename ColumnBuilder::ValueTypeT; - ValueHandler res = [ field, columnBuilderRawPtr ]( const Struct *s ) + ValueHandler res = [field, columnBuilderRawPtr]( const Struct * s ) { - if( field -> isSet( s ) ) + if( field->isSet( s ) ) { - columnBuilderRawPtr -> setValue( field -> value( s ) ); + columnBuilderRawPtr->setValue( field->value( s ) ); } }; - // We need to collect on the top level value handlers and array builders. Value handlers and builders of nested structs are stored - // in the struct field handlers. Path is non null in this case. + // We need to collect on the top level value handlers and array builders. Value handlers and builders of nested + // structs are stored in the struct field handlers. Path is non null in this case. if( !path ) { m_valueHandlers.push_back( res ); @@ -280,20 +282,20 @@ inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHa return { columnBuilderPtr, res }; } -inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createEnumColumnBuilder( - const StructField *field, - const std::string &columnName, - const std::string *path ) +inline StructParquetOutputHandler::ColumnBuilderResultType +StructParquetOutputHandler::createEnumColumnBuilder( const StructField * field, const std::string & columnName, + const std::string * path ) { - auto columnBuilderPtr{ std::make_shared( resolveFullColumnName( path, columnName ), getChunkSize() ) }; + auto columnBuilderPtr{ + std::make_shared( resolveFullColumnName( path, columnName ), getChunkSize() ) }; auto columnBuilderRawPtr{ columnBuilderPtr.get() }; - auto enumMetaPtr = std::static_pointer_cast( field -> type() ) -> meta(); + auto enumMetaPtr = std::static_pointer_cast( field->type() )->meta(); - ValueHandler res = [ field, columnBuilderRawPtr ]( const Struct *s ) + ValueHandler res = [field, columnBuilderRawPtr]( const Struct * s ) { - if( field -> isSet( s ) ) + if( field->isSet( s ) ) { - columnBuilderRawPtr -> setValue( field -> value( s ).name() ); + columnBuilderRawPtr->setValue( field->value( s ).name() ); } }; if( path == nullptr ) @@ -304,53 +306,47 @@ inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHa return { columnBuilderPtr, res }; } -inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createStructColumnBuilder( - const StructField *structField, - const std::string &columnName, - const std::string *path ) +inline StructParquetOutputHandler::ColumnBuilderResultType +StructParquetOutputHandler::createStructColumnBuilder( const StructField * structField, const std::string & columnName, + const std::string * path ) { std::vector> fields; std::vector childArrayBuilders; std::vector childFieldSetters; - auto structFieldMetaPtr = std::static_pointer_cast( structField -> type() ) -> meta().get(); + auto structFieldMetaPtr = std::static_pointer_cast( structField->type() )->meta().get(); - for( auto &subField:structFieldMetaPtr -> fields() ) + for( auto & subField : structFieldMetaPtr->fields() ) { - std::string fieldSubPath = resolveFullColumnName( path, subField -> fieldname() ); + std::string fieldSubPath = resolveFullColumnName( path, subField->fieldname() ); - auto childBuilderRes = createColumnBuilder( structFieldMetaPtr, - subField -> fieldname(), - subField -> fieldname(), - &fieldSubPath ); + auto childBuilderRes + = createColumnBuilder( structFieldMetaPtr, subField->fieldname(), subField->fieldname(), &fieldSubPath ); childArrayBuilders.push_back( childBuilderRes.m_columnBuilder ); childFieldSetters.push_back( childBuilderRes.m_valueHandler ); fields.push_back( - std::make_shared<::arrow::Field>( subField -> fieldname(), childBuilderRes.m_columnBuilder -> getDataType() ) ); + std::make_shared<::arrow::Field>( subField->fieldname(), childBuilderRes.m_columnBuilder->getDataType() ) ); } - auto subFieldValueSetter = [ childFieldSetters ]( const Struct *s ) + auto subFieldValueSetter = [childFieldSetters]( const Struct * s ) { - for( auto &childFieldSetter:childFieldSetters ) + for( auto & childFieldSetter : childFieldSetters ) { childFieldSetter( s ); } }; auto columnBuilderPtr{ std::make_shared( - resolveFullColumnName( path, columnName ), getChunkSize(), - std::make_shared<::arrow::StructType>( fields ), - childArrayBuilders, - std::move( subFieldValueSetter ) - ) }; + resolveFullColumnName( path, columnName ), getChunkSize(), std::make_shared<::arrow::StructType>( fields ), + childArrayBuilders, std::move( subFieldValueSetter ) ) }; auto columnBuilderRawPtr = columnBuilderPtr.get(); - ValueHandler valueHandler = [ structField, columnBuilderRawPtr ]( const Struct *s ) + ValueHandler valueHandler = [structField, columnBuilderRawPtr]( const Struct * s ) { - if( structField -> isSet( s ) ) + if( structField->isSet( s ) ) { - columnBuilderRawPtr -> setValue( structField -> value( s ).get() ); + columnBuilderRawPtr->setValue( structField->value( s ).get() ); } }; if( path == nullptr ) @@ -366,4 +362,4 @@ void StructParquetOutputAdapter::executeImpl() writeValueFromTs( input() ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.h b/cpp/csp/adapters/parquet/ParquetOutputAdapter.h index 7148d0fa9..562db651a 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.h +++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.h @@ -2,14 +2,14 @@ #define _IN_CSP_ADAPTERS_PARQUET_ParquetOutputAdapter_H #include +#include #include #include #include -#include namespace arrow { - class ArrayBuilder; +class ArrayBuilder; } namespace csp @@ -17,7 +17,7 @@ namespace csp class Struct; class StructField; -} +} // namespace csp namespace csp::adapters::parquet { @@ -26,8 +26,9 @@ class ParquetWriter; class ParquetOutputHandler { public: - ParquetOutputHandler( ParquetWriter &parquetWriter, CspTypePtr &type ) - : m_type( type ), m_parquetWriter( parquetWriter ) + ParquetOutputHandler( ParquetWriter & parquetWriter, CspTypePtr & type ) + : m_type( type ) + , m_parquetWriter( parquetWriter ) { } @@ -35,40 +36,40 @@ class ParquetOutputHandler uint32_t getChunkSize() const; - virtual uint32_t getNumColumns() = 0; - virtual std::shared_ptr getColumnArrayBuilder( unsigned index ) = 0; - virtual void writeValueFromTs( const TimeSeriesProvider *input ) = 0; + virtual uint32_t getNumColumns() = 0; + virtual std::shared_ptr getColumnArrayBuilder( unsigned index ) = 0; + virtual void writeValueFromTs( const TimeSeriesProvider * input ) = 0; protected: - CspTypePtr &m_type; - ParquetWriter &m_parquetWriter; + CspTypePtr & m_type; + ParquetWriter & m_parquetWriter; }; class SingleColumnParquetOutputHandler : public ParquetOutputHandler { public: - SingleColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, + SingleColumnParquetOutputHandler( Engine * engine, ParquetWriter & parquetWriter, CspTypePtr & type, std::string columnName ); - uint32_t getNumColumns() override{ return 1; } + uint32_t getNumColumns() override { return 1; } - std::shared_ptr getColumnArrayBuilder( unsigned index ) override{ return m_columnArrayBuilder; }; - - template< typename T, typename ColumnBuilderType > - void writeValue( const T &value ) + std::shared_ptr getColumnArrayBuilder( unsigned index ) override { - static_cast(this -> m_columnArrayBuilder.get()) -> setValue( value ); - } + return m_columnArrayBuilder; + }; - void writeValueFromTs( const TimeSeriesProvider *input ) override final + template + void writeValue( const T & value ) { - ( *m_valueHandler )( input ); + static_cast( this->m_columnArrayBuilder.get() )->setValue( value ); } + void writeValueFromTs( const TimeSeriesProvider * input ) override final { ( *m_valueHandler )( input ); } + private: - template< typename ColumnBuilder > - void createColumnBuilder( const std::string &columnName ); - void createEnumColumnBuilder( const std::string &columnName, CspEnumMeta::Ptr enumMetaPtr ); + template + void createColumnBuilder( const std::string & columnName ); + void createEnumColumnBuilder( const std::string & columnName, CspEnumMeta::Ptr enumMetaPtr ); protected: using ValueHandler = std::function; @@ -80,30 +81,30 @@ class SingleColumnParquetOutputHandler : public ParquetOutputHandler class ListColumnParquetOutputHandler : public ParquetOutputHandler { public: - ListColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &elemType, const std::string &columnName, - DialectGenericListWriterInterface::Ptr& listWriterInterface ); - - uint32_t getNumColumns() override{ return 1; } + ListColumnParquetOutputHandler( Engine * engine, ParquetWriter & parquetWriter, CspTypePtr & elemType, + const std::string & columnName, + DialectGenericListWriterInterface::Ptr & listWriterInterface ); - std::shared_ptr getColumnArrayBuilder( unsigned index ) override{ return m_columnArrayBuilder; }; + uint32_t getNumColumns() override { return 1; } - template< typename T, typename ColumnBuilderType > - void writeValue( const T &value ) + std::shared_ptr getColumnArrayBuilder( unsigned index ) override { - static_cast(this -> m_columnArrayBuilder.get()) -> setValue( value ); - } + return m_columnArrayBuilder; + }; - void writeValueFromTs( const TimeSeriesProvider *input ) override final + template + void writeValue( const T & value ) { - ( *m_valueHandler )( input ); + static_cast( this->m_columnArrayBuilder.get() )->setValue( value ); } -private: - std::shared_ptr createValueBuilder( const CspTypePtr &elemType, - DialectGenericListWriterInterface::Ptr &listWriterInterface ); + void writeValueFromTs( const TimeSeriesProvider * input ) override final { ( *m_valueHandler )( input ); } +private: + std::shared_ptr + createValueBuilder( const CspTypePtr & elemType, DialectGenericListWriterInterface::Ptr & listWriterInterface ); -protected : +protected: using ValueHandler = std::function; std::unique_ptr m_valueHandler; @@ -113,12 +114,14 @@ protected : class SingleColumnParquetOutputAdapter final : public OutputAdapter, public SingleColumnParquetOutputHandler { public: - SingleColumnParquetOutputAdapter( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, std::string columnName ) - : OutputAdapter( engine ), SingleColumnParquetOutputHandler( engine, parquetWriter, type, columnName ) + SingleColumnParquetOutputAdapter( Engine * engine, ParquetWriter & parquetWriter, CspTypePtr & type, + std::string columnName ) + : OutputAdapter( engine ) + , SingleColumnParquetOutputHandler( engine, parquetWriter, type, columnName ) { } - const char *name() const override{ return "ParquetSingleColumnOutputAdapter"; } + const char * name() const override { return "ParquetSingleColumnOutputAdapter"; } void executeImpl() override; }; @@ -126,31 +129,32 @@ class SingleColumnParquetOutputAdapter final : public OutputAdapter, public Sing class ListColumnParquetOutputAdapter : public OutputAdapter, public ListColumnParquetOutputHandler { public: - ListColumnParquetOutputAdapter( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, std::string columnName, - DialectGenericListWriterInterface::Ptr listWriterInterface ) - : OutputAdapter( engine ), ListColumnParquetOutputHandler( engine, parquetWriter, type, columnName, listWriterInterface ) + ListColumnParquetOutputAdapter( Engine * engine, ParquetWriter & parquetWriter, CspTypePtr & type, + std::string columnName, DialectGenericListWriterInterface::Ptr listWriterInterface ) + : OutputAdapter( engine ) + , ListColumnParquetOutputHandler( engine, parquetWriter, type, columnName, listWriterInterface ) { } - const char *name() const override{ return "ListColumnParquetOutputAdapter"; } + const char * name() const override { return "ListColumnParquetOutputAdapter"; } void executeImpl() override; }; - class StructParquetOutputHandler : public ParquetOutputHandler { public: - StructParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, DictionaryPtr fieldMap ); + StructParquetOutputHandler( Engine * engine, ParquetWriter & parquetWriter, CspTypePtr & type, + DictionaryPtr fieldMap ); - uint32_t getNumColumns() override{ return m_valueHandlers.size(); } + uint32_t getNumColumns() override { return m_valueHandlers.size(); } std::shared_ptr getColumnArrayBuilder( unsigned index ) override { - return m_columnArrayBuilders[ index ]; + return m_columnArrayBuilders[index]; } - void writeValueFromTs( const TimeSeriesProvider *input ) override final; + void writeValueFromTs( const TimeSeriesProvider * input ) override final; private: using ValueHandler = std::function; @@ -160,26 +164,20 @@ class StructParquetOutputHandler : public ParquetOutputHandler ValueHandler m_valueHandler; }; + ColumnBuilderResultType createColumnBuilder( const StructMeta * structMeta, const std::string & columnName, + const std::string & structFieldName, const std::string * path ); - ColumnBuilderResultType createColumnBuilder( const StructMeta *structMeta, - const std::string &columnName, - const std::string &structFieldName, - const std::string *path ); + template + ColumnBuilderResultType createColumnBuilder( const StructField * field, const std::string & columnName, + const std::string * path ); - template< typename ColumnBuilder > - ColumnBuilderResultType createColumnBuilder( const StructField *field, - const std::string &columnName, - const std::string *path ); + ColumnBuilderResultType createEnumColumnBuilder( const StructField * field, const std::string & columnName, + const std::string * path ); - ColumnBuilderResultType createEnumColumnBuilder( const StructField *field, - const std::string &columnName, - const std::string *path ); + ColumnBuilderResultType createStructColumnBuilder( const StructField * structField, const std::string & columnName, + const std::string * path ); - ColumnBuilderResultType createStructColumnBuilder( const StructField *structField, - const std::string &columnName, - const std::string *path ); - - inline std::string resolveFullColumnName( const std::string *path, const std::string &columnName ) + inline std::string resolveFullColumnName( const std::string * path, const std::string & columnName ) { return path == nullptr ? columnName : *path + "." + columnName; } @@ -192,17 +190,19 @@ class StructParquetOutputHandler : public ParquetOutputHandler class StructParquetOutputAdapter final : public OutputAdapter, public StructParquetOutputHandler { public: - StructParquetOutputAdapter( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, DictionaryPtr fieldMap ) - : OutputAdapter( engine ), StructParquetOutputHandler( engine, parquetWriter, type, fieldMap ) + StructParquetOutputAdapter( Engine * engine, ParquetWriter & parquetWriter, CspTypePtr & type, + DictionaryPtr fieldMap ) + : OutputAdapter( engine ) + , StructParquetOutputHandler( engine, parquetWriter, type, fieldMap ) { } - const char *name() const override{ return "ParquetStructOutputAdapter"; } + const char * name() const override { return "ParquetStructOutputAdapter"; } using StructParquetOutputHandler::StructParquetOutputHandler; void executeImpl() override; }; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp index e662d98d1..2cb85a25d 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp +++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp @@ -1,16 +1,18 @@ +#include +#include #include #include -#include -#include #include #include namespace csp::adapters::parquet { - -ParquetOutputAdapterManager::ParquetOutputAdapterManager( csp::Engine *engine, const Dictionary &properties, FileVisitorCallback fileVisitor ) : - AdapterManager( engine ), m_fileVisitor( fileVisitor ), m_outputFilenameAdapter( nullptr ) +ParquetOutputAdapterManager::ParquetOutputAdapterManager( csp::Engine * engine, const Dictionary & properties, + FileVisitorCallback fileVisitor ) + : AdapterManager( engine ) + , m_fileVisitor( fileVisitor ) + , m_outputFilenameAdapter( nullptr ) { m_fileName = properties.get( "file_name" ); m_timestampColumnName = properties.get( "timestamp_column_name" ); @@ -22,28 +24,26 @@ ParquetOutputAdapterManager::ParquetOutputAdapterManager( csp::Engine *engine, c m_parquetWriter = std::make_unique( this, properties ); } -ParquetOutputAdapterManager::~ParquetOutputAdapterManager() -{ -} +ParquetOutputAdapterManager::~ParquetOutputAdapterManager() {} void ParquetOutputAdapterManager::start( DateTime starttime, DateTime endtime ) { - m_parquetWriter -> start(); - for( auto &&writer:m_dictBasketWriters ) + m_parquetWriter->start(); + for( auto && writer : m_dictBasketWriters ) { - writer -> start(); + writer->start(); } } void ParquetOutputAdapterManager::stop() { - bool visitFile = m_fileVisitor && m_parquetWriter -> isFileOpen(); - m_parquetWriter -> stop(); + bool visitFile = m_fileVisitor && m_parquetWriter->isFileOpen(); + m_parquetWriter->stop(); m_parquetWriter = nullptr; - for( auto &&writer:m_dictBasketWriters ) + for( auto && writer : m_dictBasketWriters ) { - writer -> stop(); + writer->stop(); } m_dictBasketWriters.clear(); @@ -56,9 +56,9 @@ DateTime ParquetOutputAdapterManager::processNextSimTimeSlice( DateTime time ) return DateTime::NONE(); } -OutputAdapter *ParquetOutputAdapterManager::getOutputAdapter( CspTypePtr &type, const Dictionary &properties ) +OutputAdapter * ParquetOutputAdapterManager::getOutputAdapter( CspTypePtr & type, const Dictionary & properties ) { - if( type -> type() == CspType::Type::STRUCT ) + if( type->type() == CspType::Type::STRUCT ) { return getStructOutputAdapter( type, properties ); } @@ -68,46 +68,49 @@ OutputAdapter *ParquetOutputAdapterManager::getOutputAdapter( CspTypePtr &type, } } -OutputAdapter *ParquetOutputAdapterManager::getListOutputAdapter( CspTypePtr &elemType, const Dictionary &properties, - const DialectGenericListWriterInterface::Ptr& listWriterInterface) +OutputAdapter * +ParquetOutputAdapterManager::getListOutputAdapter( CspTypePtr & elemType, const Dictionary & properties, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ) { auto columnName = properties.get( "column_name" ); - return m_parquetWriter -> getListOutputAdapter( elemType, columnName, listWriterInterface ); + return m_parquetWriter->getListOutputAdapter( elemType, columnName, listWriterInterface ); } - ParquetDictBasketOutputWriter * -ParquetOutputAdapterManager::createDictOutputBasketWriter( const char *columnName, const CspTypePtr &cspTypePtr ) +ParquetOutputAdapterManager::createDictOutputBasketWriter( const char * columnName, const CspTypePtr & cspTypePtr ) { - auto &&existingAdapterIt = m_dictBasketWriterIndexByName.find( columnName ); + auto && existingAdapterIt = m_dictBasketWriterIndexByName.find( columnName ); CSP_TRUE_OR_THROW_RUNTIME( existingAdapterIt == m_dictBasketWriterIndexByName.end(), "Trying to create output basket writer for " << columnName << " more than once" ); - if( cspTypePtr -> type() == CspType::Type::STRUCT ) + if( cspTypePtr->type() == CspType::Type::STRUCT ) { - m_dictBasketWriters.push_back( std::make_unique( this, columnName, cspTypePtr ) ); + m_dictBasketWriters.push_back( + std::make_unique( this, columnName, cspTypePtr ) ); } else { - m_dictBasketWriters.push_back( std::make_unique( this, columnName, cspTypePtr ) ); + m_dictBasketWriters.push_back( + std::make_unique( this, columnName, cspTypePtr ) ); } - m_dictBasketWriterIndexByName[ columnName ] = m_dictBasketWriters.size() - 1; + m_dictBasketWriterIndexByName[columnName] = m_dictBasketWriters.size() - 1; return m_dictBasketWriters.back().get(); } -OutputAdapter *ParquetOutputAdapterManager::createOutputFileNameAdapter() +OutputAdapter * ParquetOutputAdapterManager::createOutputFileNameAdapter() { - CSP_TRUE_OR_THROW_RUNTIME( m_outputFilenameAdapter == nullptr, "Trying to set output filename adapter more than once" ); - m_outputFilenameAdapter = engine() -> createOwnedObject( *this ); + CSP_TRUE_OR_THROW_RUNTIME( m_outputFilenameAdapter == nullptr, + "Trying to set output filename adapter more than once" ); + m_outputFilenameAdapter = engine()->createOwnedObject( *this ); return m_outputFilenameAdapter; } -void ParquetOutputAdapterManager::changeFileName( const std::string &filename ) +void ParquetOutputAdapterManager::changeFileName( const std::string & filename ) { if( m_parquetWriter ) { - m_parquetWriter -> onFileNameChange( filename ); + m_parquetWriter->onFileNameChange( filename ); } if( m_fileVisitor ) @@ -118,28 +121,27 @@ void ParquetOutputAdapterManager::changeFileName( const std::string &filename ) void ParquetOutputAdapterManager::scheduleEndCycle() { - if( rootEngine() -> scheduleEndCycleListener( m_parquetWriter.get() ) ) + if( rootEngine()->scheduleEndCycleListener( m_parquetWriter.get() ) ) { - for( auto &&basketWriter:m_dictBasketWriters ) + for( auto && basketWriter : m_dictBasketWriters ) { - rootEngine() -> scheduleEndCycleListener( basketWriter.get() ); + rootEngine()->scheduleEndCycleListener( basketWriter.get() ); } } } -OutputAdapter *ParquetOutputAdapterManager::getScalarOutputAdapter( CspTypePtr &type, const Dictionary &properties ) +OutputAdapter * ParquetOutputAdapterManager::getScalarOutputAdapter( CspTypePtr & type, const Dictionary & properties ) { auto columnName = properties.get( "column_name" ); - return m_parquetWriter -> getScalarOutputAdapter( type, columnName ); + return m_parquetWriter->getScalarOutputAdapter( type, columnName ); } -OutputAdapter *ParquetOutputAdapterManager::getStructOutputAdapter( CspTypePtr &type, const Dictionary &properties ) +OutputAdapter * ParquetOutputAdapterManager::getStructOutputAdapter( CspTypePtr & type, const Dictionary & properties ) { auto fieldMap = properties.get( "field_map" ); - return m_parquetWriter -> getStructOutputAdapter( type, fieldMap ); + return m_parquetWriter->getStructOutputAdapter( type, fieldMap ); } - -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h index a2b5da200..aa874e744 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h +++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h @@ -1,6 +1,7 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetOutputAdapterManager_H #define _IN_CSP_ADAPTERS_PARQUET_ParquetOutputAdapterManager_H +#include #include #include #include @@ -9,8 +10,6 @@ #include #include #include -#include - namespace csp::adapters::parquet { @@ -20,52 +19,53 @@ class ParquetOutputFilenameAdapter; class ParquetDictBasketOutputWriter; -//Top level AdapterManager object for all parquet adapters in the engine +// Top level AdapterManager object for all parquet adapters in the engine class ParquetOutputAdapterManager final : public csp::AdapterManager { public: - using FileVisitorCallback = std::function; + using FileVisitorCallback = std::function; - ParquetOutputAdapterManager( csp::Engine *engine, const Dictionary &properties, FileVisitorCallback fileVisitor ); + ParquetOutputAdapterManager( csp::Engine * engine, const Dictionary & properties, FileVisitorCallback fileVisitor ); ~ParquetOutputAdapterManager(); - const char *name() const override{ return "ParquetOutputAdapterManager"; } + const char * name() const override { return "ParquetOutputAdapterManager"; } - const std::string &getFileName() const{ return m_fileName; } + const std::string & getFileName() const { return m_fileName; } - const std::string &getTimestampColumnName() const{ return m_timestampColumnName; } + const std::string & getTimestampColumnName() const { return m_timestampColumnName; } - bool isAllowOverwrite() const{ return m_allowOverwrite; } + bool isAllowOverwrite() const { return m_allowOverwrite; } - uint32_t getBatchSize() const{ return m_batchSize; } + uint32_t getBatchSize() const { return m_batchSize; } - std::string getCompression() const{ return m_compression; } + std::string getCompression() const { return m_compression; } - bool isWriteArrowBinary() const{ return m_writeArrowBinary; } + bool isWriteArrowBinary() const { return m_writeArrowBinary; } - bool isSplitColumnsToFiles() const{ return m_splitColumnsToFiles; } + bool isSplitColumnsToFiles() const { return m_splitColumnsToFiles; } - //start the writer, open file if necessary + // start the writer, open file if necessary void start( DateTime starttime, DateTime endtime ) override; - //stop the writer, write any unwritten data and close file + // stop the writer, write any unwritten data and close file void stop() override; DateTime processNextSimTimeSlice( DateTime time ) override; - OutputAdapter *getOutputAdapter( CspTypePtr &type, const Dictionary &properties ); - OutputAdapter *getListOutputAdapter( CspTypePtr &elemType, const Dictionary &properties, - const DialectGenericListWriterInterface::Ptr& listWriterInterface ); - ParquetDictBasketOutputWriter *createDictOutputBasketWriter( const char *columnName, const CspTypePtr &cspTypePtr); - OutputAdapter *createOutputFileNameAdapter(); + OutputAdapter * getOutputAdapter( CspTypePtr & type, const Dictionary & properties ); + OutputAdapter * getListOutputAdapter( CspTypePtr & elemType, const Dictionary & properties, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ); + ParquetDictBasketOutputWriter * createDictOutputBasketWriter( const char * columnName, + const CspTypePtr & cspTypePtr ); + OutputAdapter * createOutputFileNameAdapter(); - void changeFileName( const std::string &filename ); + void changeFileName( const std::string & filename ); void scheduleEndCycle(); private: - OutputAdapter *getScalarOutputAdapter( CspTypePtr &type, const Dictionary &properties ); - OutputAdapter *getStructOutputAdapter( CspTypePtr &type, const Dictionary &properties ); + OutputAdapter * getScalarOutputAdapter( CspTypePtr & type, const Dictionary & properties ); + OutputAdapter * getStructOutputAdapter( CspTypePtr & type, const Dictionary & properties ); std::string m_fileName; std::string m_timestampColumnName; @@ -78,9 +78,9 @@ class ParquetOutputAdapterManager final : public csp::AdapterManager std::unordered_map m_dictBasketWriterIndexByName; std::vector> m_dictBasketWriters; FileVisitorCallback m_fileVisitor; - ParquetOutputFilenameAdapter *m_outputFilenameAdapter; + ParquetOutputFilenameAdapter * m_outputFilenameAdapter; }; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.cpp b/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.cpp index 968d68f21..0ab071a3c 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.cpp +++ b/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include namespace csp::adapters::parquet @@ -7,7 +7,7 @@ namespace csp::adapters::parquet void ParquetOutputFilenameAdapter::executeImpl() { - m_parquetOutputAdapterManager.changeFileName( input()->lastValueTyped()); + m_parquetOutputAdapterManager.changeFileName( input()->lastValueTyped() ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.h b/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.h index ba0b48a71..6707744ae 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.h +++ b/cpp/csp/adapters/parquet/ParquetOutputFilenameAdapter.h @@ -3,7 +3,6 @@ #include - namespace csp::adapters::parquet { class ParquetOutputAdapterManager; @@ -11,19 +10,20 @@ class ParquetOutputAdapterManager; class ParquetOutputFilenameAdapter : public csp::OutputAdapter { public: - ParquetOutputFilenameAdapter( Engine *engine, ParquetOutputAdapterManager &parquetOutputAdapterManager ) - : csp::OutputAdapter( engine ), m_parquetOutputAdapterManager( parquetOutputAdapterManager ) + ParquetOutputFilenameAdapter( Engine * engine, ParquetOutputAdapterManager & parquetOutputAdapterManager ) + : csp::OutputAdapter( engine ) + , m_parquetOutputAdapterManager( parquetOutputAdapterManager ) { } void executeImpl() override; - const char *name() const override { return "ParquetOutputFilenameAdapter"; } + const char * name() const override { return "ParquetOutputFilenameAdapter"; } protected: - ParquetOutputAdapterManager &m_parquetOutputAdapterManager; + ParquetOutputAdapterManager & m_parquetOutputAdapterManager; }; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ParquetReader.cpp b/cpp/csp/adapters/parquet/ParquetReader.cpp index 81cfd40ec..7267bea4d 100644 --- a/cpp/csp/adapters/parquet/ParquetReader.cpp +++ b/cpp/csp/adapters/parquet/ParquetReader.cpp @@ -1,15 +1,15 @@ -#include +#include +#include +#include #include -#include #include +#include +#include #include -#include #include +#include #include #include -#include -#include -#include namespace { @@ -21,50 +21,50 @@ struct FieldInfo std::size_t getFieldWidth( std::shared_ptr<::arrow::DataType> fieldType ) { - if( fieldType -> id() != ::arrow::Type::STRUCT ) + if( fieldType->id() != ::arrow::Type::STRUCT ) { return 1; } std::size_t res = 0; - for( auto &childField: fieldType -> fields() ) + for( auto & childField : fieldType->fields() ) { - res += getFieldWidth( childField -> type() ); + res += getFieldWidth( childField->type() ); } return res; } -inline std::vector getFieldsInfo( const std::vector> &fields ) +inline std::vector getFieldsInfo( const std::vector> & fields ) { std::size_t curStartIndex = 0; std::vector res; res.reserve( fields.size() ); - for( auto &field:fields ) + for( auto & field : fields ) { - auto width = getFieldWidth( field -> type() ); + auto width = getFieldWidth( field->type() ); res.push_back( { curStartIndex, width } ); curStartIndex += width; } return res; } -} +} // namespace namespace csp::adapters::parquet { -ParquetStructAdapter &ParquetReader::getStructAdapter( const StructAdapterInfo &structAdapterInfo ) +ParquetStructAdapter & ParquetReader::getStructAdapter( const StructAdapterInfo & structAdapterInfo ) { auto it = m_structInfoToAdapterIndex.find( structAdapterInfo ); if( it == m_structInfoToAdapterIndex.end() ) { - m_structInfoToAdapterIndex[ structAdapterInfo ] = m_structAdapters.size(); + m_structInfoToAdapterIndex[structAdapterInfo] = m_structAdapters.size(); m_structAdapters.push_back( std::make_unique( *this, structAdapterInfo ) ); - return *m_structAdapters[ m_structAdapters.size() - 1 ]; + return *m_structAdapters[m_structAdapters.size() - 1]; } else { - return *m_structAdapters[ it -> second ]; + return *m_structAdapters[it->second]; } } @@ -76,20 +76,20 @@ const utils::Symbol * ParquetReader::getCurSymbol() { case CspType::Type::STRING: { - auto &curSymbol = getSymbolColumnAdapter() -> getCurValue(); + auto & curSymbol = getSymbolColumnAdapter()->getCurValue(); CSP_TRUE_OR_THROW_RUNTIME( curSymbol.has_value(), "Parquet file row contains row with no value for symbol column " - << getSymbolColumnName().value() ); + << getSymbolColumnName().value() ); m_curSymbol = curSymbol.value(); break; } case CspType::Type::INT64: { - auto &curSymbol = getSymbolColumnAdapter() -> getCurValue(); + auto & curSymbol = getSymbolColumnAdapter()->getCurValue(); CSP_TRUE_OR_THROW_RUNTIME( curSymbol.has_value(), "Parquet file row contains row with no value for symbol column " - << getSymbolColumnName().value() ); + << getSymbolColumnName().value() ); m_curSymbol = curSymbol.value(); break; } @@ -104,25 +104,25 @@ const utils::Symbol * ParquetReader::getCurSymbol() return nullptr; } - void ParquetReader::setSymbolColumnAdapter( ColumnAdapterReference adapter ) { m_symbolColumn = adapter; try { - m_symbolColumn -> ensureType( CspType::STRING() ); + m_symbolColumn->ensureType( CspType::STRING() ); m_symbolType = CspType::Type::STRING; } catch( const TypeError & ) { try { - m_symbolColumn -> ensureType( CspType::INT64() ); + m_symbolColumn->ensureType( CspType::INT64() ); m_symbolType = CspType::Type::INT64; } catch( const TypeError & ) { - CSP_THROW( TypeError, "Invalid symbol column type. Only string and int64 symbols are currently supported" ); + CSP_THROW( TypeError, + "Invalid symbol column type. Only string and int64 symbols are currently supported" ); } } } @@ -132,26 +132,30 @@ void ParquetReader::validateSymbolType( const utils::Symbol & symbol ) switch( m_symbolType ) { case CspType::Type::STRING: - CSP_TRUE_OR_THROW( std::holds_alternative( symbol ), TypeError, "Provided symbol type does not match symbol column type (string)" ); + CSP_TRUE_OR_THROW( std::holds_alternative( symbol ), TypeError, + "Provided symbol type does not match symbol column type (string)" ); break; case CspType::Type::INT64: - CSP_TRUE_OR_THROW( std::holds_alternative( symbol ), TypeError, "Provided symbol type does not match symbol column type (int64)" ); + CSP_TRUE_OR_THROW( std::holds_alternative( symbol ), TypeError, + "Provided symbol type does not match symbol column type (int64)" ); break; default: CSP_THROW( RuntimeException, "Unexpected symbol type: " << m_symbolType ); } } -SingleTableParquetReader::SingleTableParquetReader( std::vector columns, bool arrowIPC, bool allowMissingColumns, +SingleTableParquetReader::SingleTableParquetReader( std::vector columns, bool arrowIPC, + bool allowMissingColumns, std::optional symbolColumnName ) - : ParquetReader( symbolColumnName, arrowIPC, allowMissingColumns ), m_columns( columns ) + : ParquetReader( symbolColumnName, arrowIPC, allowMissingColumns ) + , m_columns( columns ) { } void SingleTableParquetReader::init() { - if(!openNextFile()) + if( !openNextFile() ) { return; } @@ -166,31 +170,31 @@ void SingleTableParquetReader::setColumnAdaptersFromCurrentTable() m_neededColumnIndices.reserve( m_columns.size() ); m_columnAdapters.reserve( m_columns.size() ); - auto &fields = m_schema -> fields(); - auto fieldsInfo = getFieldsInfo( fields ); + auto & fields = m_schema->fields(); + auto fieldsInfo = getFieldsInfo( fields ); - for( const auto &columnName:m_columns ) + for( const auto & columnName : m_columns ) { - auto index = m_schema -> GetFieldIndex( columnName ); - auto existingRecordIt = m_columnNameToAdapterIndex.find(columnName); - if(existingRecordIt != m_columnNameToAdapterIndex.end()) + auto index = m_schema->GetFieldIndex( columnName ); + auto existingRecordIt = m_columnNameToAdapterIndex.find( columnName ); + if( existingRecordIt != m_columnNameToAdapterIndex.end() ) { - CSP_TRUE_OR_THROW_RUNTIME( existingRecordIt -> second == m_columnAdapters.size(), - "Unexpected index change of column " << columnName << - " was " << existingRecordIt -> second << " became " + CSP_TRUE_OR_THROW_RUNTIME( existingRecordIt->second == m_columnAdapters.size(), + "Unexpected index change of column " << columnName << " was " + << existingRecordIt->second << " became " << m_columnAdapters.size() ); } else { - m_columnNameToAdapterIndex[ columnName ] = m_columnAdapters.size(); + m_columnNameToAdapterIndex[columnName] = m_columnAdapters.size(); } std::unique_ptr columnAdapter; if( index >= 0 ) { - auto &field = fields[ index ]; - columnAdapter = createColumnAdapter( *this, *field, getCurFileOrTableName(), &getStructColumnMeta() ); - auto &fieldInfo = fieldsInfo[ index ]; + auto & field = fields[index]; + columnAdapter = createColumnAdapter( *this, *field, getCurFileOrTableName(), &getStructColumnMeta() ); + auto & fieldInfo = fieldsInfo[index]; for( std::size_t i = 0; i < fieldInfo.m_width; ++i ) { @@ -199,7 +203,8 @@ void SingleTableParquetReader::setColumnAdaptersFromCurrentTable() } else { - CSP_TRUE_OR_THROW_RUNTIME( isAllowMissingColumns(), "Missing column " << columnName << " in file " << getCurFileOrTableName() ); + CSP_TRUE_OR_THROW_RUNTIME( isAllowMissingColumns(), + "Missing column " << columnName << " in file " << getCurFileOrTableName() ); columnAdapter = createMissingColumnAdapter( *this, columnName ); } m_columnAdapters.push_back( std::move( columnAdapter ) ); @@ -207,11 +212,10 @@ void SingleTableParquetReader::setColumnAdaptersFromCurrentTable() if( getSymbolColumnName().has_value() ) { - setSymbolColumnAdapter( ( *this )[ getSymbolColumnName().value() ] ); + setSymbolColumnAdapter( ( *this )[getSymbolColumnName().value()] ); } } - bool SingleTableParquetReader::start() { return readNextRowGroup() && readNextRow(); @@ -229,38 +233,36 @@ bool SingleTableParquetReader::readNextRow() return false; } - if( unlikely( m_curTableNextRow >= m_curTable -> num_rows() ) ) + if( unlikely( m_curTableNextRow >= m_curTable->num_rows() ) ) { if( !readNextRowGroup() ) { return false; } } - for( auto &colAdapter:m_columnAdapters ) + for( auto & colAdapter : m_columnAdapters ) { - colAdapter -> readCurValue(); + colAdapter->readCurValue(); } ++m_curTableNextRow; return true; } - void SingleTableParquetReader::dispatchRow( bool doReadNextRow ) { - dispatchRow( doReadNextRow, getCurSymbol() ); } -void SingleTableParquetReader::dispatchRow( bool doReadNextRow, const utils::Symbol *symbol ) +void SingleTableParquetReader::dispatchRow( bool doReadNextRow, const utils::Symbol * symbol ) { - for( auto &adapter:m_columnAdapters ) + for( auto & adapter : m_columnAdapters ) { - adapter -> dispatchValue( symbol ); + adapter->dispatchValue( symbol ); } - for( auto &adapter:getStructAdapters() ) + for( auto & adapter : getStructAdapters() ) { - adapter -> dispatchValue( symbol ); + adapter->dispatchValue( symbol ); } if( doReadNextRow ) { @@ -282,9 +284,12 @@ void SingleTableParquetReader::clear() m_curTableNextRow = -1; } -SingleFileParquetReader::SingleFileParquetReader( GeneratorPtr generatorPtr, std::vector columns, bool arrowIPC, - bool allowMissingColumns, bool allowMissingFiles, std::optional symbolColumnName ) - : SingleTableParquetReader( columns, arrowIPC, allowMissingColumns, symbolColumnName ), m_generatorPtr( generatorPtr ), m_allowMissingFiles(allowMissingFiles) +SingleFileParquetReader::SingleFileParquetReader( GeneratorPtr generatorPtr, std::vector columns, + bool arrowIPC, bool allowMissingColumns, bool allowMissingFiles, + std::optional symbolColumnName ) + : SingleTableParquetReader( columns, arrowIPC, allowMissingColumns, symbolColumnName ) + , m_generatorPtr( generatorPtr ) + , m_allowMissingFiles( allowMissingFiles ) { init(); } @@ -294,11 +299,11 @@ bool SingleFileParquetReader::openNextFile() std::string fileName; FileReaderWrapperPtr fileReader; std::shared_ptr<::arrow::Schema> fileSchema; - while(true) + while( true ) { - if( m_generatorPtr -> next( fileName ) ) + if( m_generatorPtr->next( fileName ) ) { - if(m_allowMissingFiles && !csp::utils::fileExists(fileName)) + if( m_allowMissingFiles && !csp::utils::fileExists( fileName ) ) { continue; } @@ -310,8 +315,8 @@ bool SingleFileParquetReader::openNextFile() { fileReader = std::make_unique(); } - fileReader -> open( fileName ); - fileReader -> getSchema( fileSchema ); + fileReader->open( fileName ); + fileReader->getSchema( fileSchema ); break; } else @@ -321,12 +326,12 @@ bool SingleFileParquetReader::openNextFile() } } - auto is_new_schema = m_schema && !m_schema -> Equals( *fileSchema ); - m_fileName = fileName; - m_fileReader = std::move( fileReader ); - m_schema = fileSchema; + auto is_new_schema = m_schema && !m_schema->Equals( *fileSchema ); + m_fileName = fileName; + m_fileReader = std::move( fileReader ); + m_schema = fileSchema; - if(is_new_schema) + if( is_new_schema ) { setColumnAdaptersFromCurrentTable(); resubscribeAll(); @@ -345,7 +350,7 @@ bool SingleFileParquetReader::readNextRowGroup() while( !endOfData ) { // Get to first non empty table - while( m_fileReader -> readNextRowGroup( m_neededColumnIndices, m_curTable ) && m_curTable -> num_rows() == 0 ) + while( m_fileReader->readNextRowGroup( m_neededColumnIndices, m_curTable ) && m_curTable->num_rows() == 0 ) { } if( m_curTable != nullptr ) @@ -360,14 +365,14 @@ bool SingleFileParquetReader::readNextRowGroup() return false; } m_curTableNextRow = 0; - auto columns = m_curTable -> columns(); + auto columns = m_curTable->columns(); std::size_t columnIndex = 0; for( std::size_t i = 0; i < m_columnAdapters.size(); ++i ) { - if(!m_columnAdapters[i]->isMissingColumn()) + if( !m_columnAdapters[i]->isMissingColumn() ) { - m_columnAdapters[ i ] -> handleNewBatch( columns[ columnIndex++ ] ); + m_columnAdapters[i]->handleNewBatch( columns[columnIndex++] ); } } return true; @@ -382,8 +387,10 @@ void SingleFileParquetReader::clear() } InMemoryTableParquetReader::InMemoryTableParquetReader( GeneratorPtr generatorPtr, std::vector columns, - bool allowMissingColumns, std::optional symbolColumnName ) - : SingleTableParquetReader( columns, true, allowMissingColumns, symbolColumnName ), m_generatorPtr( generatorPtr ) + bool allowMissingColumns, + std::optional symbolColumnName ) + : SingleTableParquetReader( columns, true, allowMissingColumns, symbolColumnName ) + , m_generatorPtr( generatorPtr ) { init(); } @@ -393,31 +400,32 @@ bool InMemoryTableParquetReader::openNextFile() std::shared_ptr<::arrow::Schema> schema; std::shared_ptr<::arrow::Table> table; - if( !m_generatorPtr -> next( table ) ) + if( !m_generatorPtr->next( table ) ) { clear(); return false; } - CSP_TRUE_OR_THROW_RUNTIME( table -> num_columns() > 0, "Provided in memory arrow table with 0 columns" ); - schema = table -> schema(); + CSP_TRUE_OR_THROW_RUNTIME( table->num_columns() > 0, "Provided in memory arrow table with 0 columns" ); + schema = table->schema(); - int refNumChunks = table -> column( 0 ) -> num_chunks(); + int refNumChunks = table->column( 0 )->num_chunks(); - for( int i = 0; i < table -> num_columns(); ++i ) + for( int i = 0; i < table->num_columns(); ++i ) { - CSP_TRUE_OR_THROW_RUNTIME( table -> column( i ) -> num_chunks() == refNumChunks, + CSP_TRUE_OR_THROW_RUNTIME( table->column( i )->num_chunks() == refNumChunks, "Found in memory table with non aligned chunks, number of chunks in one column is " - << refNumChunks << " vs " << table -> column( i ) -> num_chunks() << " in another table" ); + << refNumChunks << " vs " << table->column( i )->num_chunks() + << " in another table" ); } - auto is_new_schema = m_schema && !m_schema -> Equals( *schema ); + auto is_new_schema = m_schema && !m_schema->Equals( *schema ); m_schema = schema; m_fullTable = table; m_nextChunkIndex = 0; m_curTable = nullptr; - if(is_new_schema) + if( is_new_schema ) { setColumnAdaptersFromCurrentTable(); resubscribeAll(); @@ -436,33 +444,33 @@ bool InMemoryTableParquetReader::readNextRowGroup() while( !endOfData ) { - if(m_nextChunkIndex >= m_fullTable->column(0)->num_chunks()) + if( m_nextChunkIndex >= m_fullTable->column( 0 )->num_chunks() ) { endOfData = !openNextFile(); continue; } std::vector> curTableChunks; - auto refChunkLength = m_fullTable->column(0)->chunk(m_nextChunkIndex)->length(); + auto refChunkLength = m_fullTable->column( 0 )->chunk( m_nextChunkIndex )->length(); std::vector> neededFields; - for(auto colIndex : m_neededColumnIndices) + for( auto colIndex : m_neededColumnIndices ) { - neededFields.push_back(m_fullTable->field(colIndex)); - auto &&curArrayChunk = m_fullTable -> column( colIndex ) -> chunk( m_nextChunkIndex ); - CSP_TRUE_OR_THROW_RUNTIME( curArrayChunk -> length() == refChunkLength, + neededFields.push_back( m_fullTable->field( colIndex ) ); + auto && curArrayChunk = m_fullTable->column( colIndex )->chunk( m_nextChunkIndex ); + CSP_TRUE_OR_THROW_RUNTIME( curArrayChunk->length() == refChunkLength, "Found in memory table with non aligned chunks, for chunk " - << m_nextChunkIndex << "found arrays of lenght " << refChunkLength << " and " - << curArrayChunk -> length() ); - arrow::Result> newChunkedArrayResult = arrow::ChunkedArray::Make( - std::vector>{curArrayChunk}); - STATUS_OK_OR_THROW_RUNTIME(newChunkedArrayResult.status(), "Failed to creae a new chunked array"); - curTableChunks.push_back(newChunkedArrayResult.ValueUnsafe()); + << m_nextChunkIndex << "found arrays of lenght " << refChunkLength << " and " + << curArrayChunk->length() ); + arrow::Result> newChunkedArrayResult + = arrow::ChunkedArray::Make( std::vector>{ curArrayChunk } ); + STATUS_OK_OR_THROW_RUNTIME( newChunkedArrayResult.status(), "Failed to creae a new chunked array" ); + curTableChunks.push_back( newChunkedArrayResult.ValueUnsafe() ); } m_nextChunkIndex += 1; - if(refChunkLength > 0) + if( refChunkLength > 0 ) { - m_curTable = arrow::Table::Make(arrow::schema(neededFields), curTableChunks); + m_curTable = arrow::Table::Make( arrow::schema( neededFields ), curTableChunks ); break; } } @@ -472,14 +480,14 @@ bool InMemoryTableParquetReader::readNextRowGroup() return false; } m_curTableNextRow = 0; - auto columns = m_curTable -> columns(); + auto columns = m_curTable->columns(); std::size_t columnIndex = 0; for( std::size_t i = 0; i < m_columnAdapters.size(); ++i ) { - if(!m_columnAdapters[i]->isMissingColumn()) + if( !m_columnAdapters[i]->isMissingColumn() ) { - m_columnAdapters[ i ] -> handleNewBatch( columns[ columnIndex++ ] ); + m_columnAdapters[i]->handleNewBatch( columns[columnIndex++] ); } } return true; @@ -501,25 +509,22 @@ void InMemoryTableParquetReader::clear() } MultipleFileParquetReader::MultipleFileParquetReader( FileNameGeneratorReplicator::Ptr generatorReplicatorPtr, - std::vector columns, - bool arrowIPC, - bool allowMissingColumns, + std::vector columns, bool arrowIPC, + bool allowMissingColumns, std::optional symbolColumnName ) - : ParquetReader( symbolColumnName, arrowIPC, allowMissingColumns ), m_generatorReplicatorPtr( generatorReplicatorPtr ) + : ParquetReader( symbolColumnName, arrowIPC, allowMissingColumns ) + , m_generatorReplicatorPtr( generatorReplicatorPtr ) { - for( auto &&column : columns ) + for( auto && column : columns ) { - m_columnReaders.push_back( - std::make_unique( - m_generatorReplicatorPtr -> getGeneratorReplica( std::string( "/" ) + '/' + column + ".parquet" ), - std::vector{ column }, - arrowIPC, - allowMissingColumns) ); + m_columnReaders.push_back( std::make_unique( + m_generatorReplicatorPtr->getGeneratorReplica( std::string( "/" ) + '/' + column + ".parquet" ), + std::vector{ column }, arrowIPC, allowMissingColumns ) ); m_columnReaderByName[column] = m_columnReaders.back().get(); } if( symbolColumnName.has_value() ) { - setSymbolColumnAdapter( ( *this )[ symbolColumnName.value() ] ); + setSymbolColumnAdapter( ( *this )[symbolColumnName.value()] ); } } @@ -527,84 +532,81 @@ bool MultipleFileParquetReader::start() { uint32_t successCount = 0; - for( auto &&columnReader : m_columnReaders ) + for( auto && columnReader : m_columnReaders ) { - if( columnReader -> start() ) + if( columnReader->start() ) { ++successCount; } } CSP_TRUE_OR_THROW_RUNTIME( successCount == 0 || successCount == m_columnReaders.size(), - "Expected all or none of the column readers to start, actual:" << successCount << '/' - << m_columnReaders.size() ); + "Expected all or none of the column readers to start, actual:" + << successCount << '/' << m_columnReaders.size() ); return successCount != 0; } - -ColumnAdapterReference MultipleFileParquetReader::operator[]( const std::string &name ) +ColumnAdapterReference MultipleFileParquetReader::operator[]( const std::string & name ) { - auto it = m_columnReaderByName.find(name); - CSP_TRUE_OR_THROW_RUNTIME( it != m_columnReaderByName.end(), - "No column " << name << " found in parquet file" ); - return (*it -> second)[name]; + auto it = m_columnReaderByName.find( name ); + CSP_TRUE_OR_THROW_RUNTIME( it != m_columnReaderByName.end(), "No column " << name << " found in parquet file" ); + return ( *it->second )[name]; } -ParquetColumnAdapter* MultipleFileParquetReader::getCurrentColumnAdapterByIndex( std::size_t index ) +ParquetColumnAdapter * MultipleFileParquetReader::getCurrentColumnAdapterByIndex( std::size_t index ) { CSP_NOT_IMPLEMENTED; } - bool MultipleFileParquetReader::skipRow() { unsigned successfulSkips = 0; - for( auto &&columnReader : m_columnReaders ) + for( auto && columnReader : m_columnReaders ) { - if(columnReader -> skipRow()) + if( columnReader->skipRow() ) { ++successfulSkips; } } - if(unlikely(successfulSkips == 0)) + if( unlikely( successfulSkips == 0 ) ) { return false; } - if(unlikely(successfulSkips!=m_columnReaders.size())) + if( unlikely( successfulSkips != m_columnReaders.size() ) ) { - CSP_THROW(RuntimeException, "Input files are not alligned - some columns have more data than the others"); + CSP_THROW( RuntimeException, "Input files are not alligned - some columns have more data than the others" ); } return true; } void MultipleFileParquetReader::dispatchRow( bool doReadNextRow ) { - // By default dispatchRow of columnReaders will dispatch row and read the next row (clear out the currently read row). - // If we have structAdapters, we can't move on to the next row before we populate the struct adapter fields. So if any - // struct adapters exist, we need to postpone reading of next row for child readers and explicitly call them after dispatching the - // struct values. - auto *symbol = getCurSymbol(); - bool childDoReadNextRow = getStructAdapters().empty() && doReadNextRow && symbol == nullptr; + // By default dispatchRow of columnReaders will dispatch row and read the next row (clear out the currently read + // row). If we have structAdapters, we can't move on to the next row before we populate the struct adapter fields. + // So if any struct adapters exist, we need to postpone reading of next row for child readers and explicitly call + // them after dispatching the struct values. + auto * symbol = getCurSymbol(); + bool childDoReadNextRow = getStructAdapters().empty() && doReadNextRow && symbol == nullptr; - for( auto &&columnReader : m_columnReaders ) + for( auto && columnReader : m_columnReaders ) { - columnReader -> dispatchRow( childDoReadNextRow, symbol ); + columnReader->dispatchRow( childDoReadNextRow, symbol ); } - for( auto &adapter:getStructAdapters() ) + for( auto & adapter : getStructAdapters() ) { - adapter -> dispatchValue( symbol ); + adapter->dispatchValue( symbol ); } if( !childDoReadNextRow && doReadNextRow ) { - for( auto &&columnReader : m_columnReaders ) + for( auto && columnReader : m_columnReaders ) { - columnReader -> readNextRow(); + columnReader->readNextRow(); } } } bool MultipleFileParquetReader::hasData() const { - return !m_columnReaders.empty() && m_columnReaders[ 0 ] -> hasData(); + return !m_columnReaders.empty() && m_columnReaders[0]->hasData(); } void MultipleFileParquetReader::clear() @@ -612,5 +614,4 @@ void MultipleFileParquetReader::clear() m_columnReaders.clear(); } - -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetReader.h b/cpp/csp/adapters/parquet/ParquetReader.h index c10923bc0..f9c8c28f4 100644 --- a/cpp/csp/adapters/parquet/ParquetReader.h +++ b/cpp/csp/adapters/parquet/ParquetReader.h @@ -1,23 +1,23 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetReader_H #define _IN_CSP_ADAPTERS_PARQUET_ParquetReader_H -#include +#include +#include #include #include +#include #include #include -#include #include +#include #include #include -#include -#include #include +#include #include #include -#include #include -#include +#include namespace arrow::io { @@ -37,10 +37,11 @@ class ParquetColumnAdapter; class ParquetStructAdapter; /** - * Wrapper of "GeneratorPtr". Generator is a general a single generator that provides a sequence of filenames (or folder names) from which the - * files should be read. Since we might have multiple file readers that need the same sequence of folders, this class wraps the generator - * and is able to create new generators that will produce exactly the same sequence as the original one. Another "service" that this class - * provides is that it can append suffix to a given generator replica. For example consider input folders sequence of : + * Wrapper of "GeneratorPtr". Generator is a general a single generator that provides a sequence of filenames (or folder + * names) from which the files should be read. Since we might have multiple file readers that need the same sequence of + * folders, this class wraps the generator and is able to create new generators that will produce exactly the same + * sequence as the original one. Another "service" that this class provides is that it can append suffix to a given + * generator replica. For example consider input folders sequence of : * - "folder1" * - "folder2". * We can ask for a suffix of "/column1.parquet" when calling to getGeneratorReplica and the replica will produce: @@ -50,31 +51,28 @@ class ParquetStructAdapter; class FileNameGeneratorReplicator { public: - using Ptr = std::shared_ptr; + using Ptr = std::shared_ptr; using GeneratorPtr = csp::Generator::Ptr; FileNameGeneratorReplicator( GeneratorPtr source ) - : m_generatorPtr( source ) + : m_generatorPtr( source ) { } - void init( csp::DateTime start, csp::DateTime end ) - { - m_generatorPtr -> init( start, end ); - } + void init( csp::DateTime start, csp::DateTime end ) { m_generatorPtr->init( start, end ); } - const std::vector &getFileNames(){ return m_fileNames; } + const std::vector & getFileNames() { return m_fileNames; } void consumeNextGeneratorFile() { std::string nextFile; - if( m_generatorPtr -> next( nextFile ) ) + if( m_generatorPtr->next( nextFile ) ) { m_fileNames.push_back( std::move( nextFile ) ); } } - GeneratorPtr getGeneratorReplica( const std::string &suffix = "" ) + GeneratorPtr getGeneratorReplica( const std::string & suffix = "" ) { return std::make_shared( *this, suffix ); } @@ -83,38 +81,40 @@ class FileNameGeneratorReplicator class ChildGenerator : public csp::Generator { public: - ChildGenerator( FileNameGeneratorReplicator &owner, const std::string &suffix ) - : m_owner( owner ), m_suffix( suffix ), m_nextIndex( 0 ) + ChildGenerator( FileNameGeneratorReplicator & owner, const std::string & suffix ) + : m_owner( owner ) + , m_suffix( suffix ) + , m_nextIndex( 0 ) { } - void init( csp::DateTime, csp::DateTime ){} + void init( csp::DateTime, csp::DateTime ) {} - bool next( std::string &value ) + bool next( std::string & value ) { if( m_nextIndex < 0 ) { return false; } - const std::vector &folders = m_owner.getFileNames(); - if( m_nextIndex >= static_cast(folders.size()) ) + const std::vector & folders = m_owner.getFileNames(); + if( m_nextIndex >= static_cast( folders.size() ) ) { m_owner.consumeNextGeneratorFile(); } - if( m_nextIndex >= static_cast(folders.size() ) ) + if( m_nextIndex >= static_cast( folders.size() ) ) { m_nextIndex = -1; return false; } - value = folders[ m_nextIndex++ ] + m_suffix; + value = folders[m_nextIndex++] + m_suffix; return true; } private: - FileNameGeneratorReplicator &m_owner; - const std::string m_suffix; - int m_nextIndex = 0; + FileNameGeneratorReplicator & m_owner; + const std::string m_suffix; + int m_nextIndex = 0; }; private: @@ -122,28 +122,30 @@ class FileNameGeneratorReplicator std::vector m_fileNames; }; - - class ParquetReader; - -// When we read multiple files with different schemas, column adapters may change, the adapter reference will persist and still be valid. +// When we read multiple files with different schemas, column adapters may change, the adapter reference will persist +// and still be valid. class ColumnAdapterReference { public: - ColumnAdapterReference( ParquetReader *reader = nullptr, std::size_t index = -1 ) : m_reader( reader ), m_index( index ){} + ColumnAdapterReference( ParquetReader * reader = nullptr, std::size_t index = -1 ) + : m_reader( reader ) + , m_index( index ) + { + } - ParquetColumnAdapter &operator*() const; + ParquetColumnAdapter & operator*() const; - ParquetColumnAdapter *get() const{ return &( **this ); }; + ParquetColumnAdapter * get() const { return &( **this ); }; - ParquetColumnAdapter *operator->() const{ return get(); }; + ParquetColumnAdapter * operator->() const { return get(); }; - bool valid() const{ return m_reader != nullptr; } + bool valid() const { return m_reader != nullptr; } private: - ParquetReader *m_reader; - std::size_t m_index; + ParquetReader * m_reader; + std::size_t m_index; }; class ParquetReader @@ -152,18 +154,17 @@ class ParquetReader using StructMetaByColumnName = std::map>; ParquetReader( std::optional symbolColumnName, bool arrowIPC, bool allowMissingColumns ) - : m_symbolColumnName( symbolColumnName ), m_arrowIPC( arrowIPC ), m_allowMissingColumns(allowMissingColumns) + : m_symbolColumnName( symbolColumnName ) + , m_arrowIPC( arrowIPC ) + , m_allowMissingColumns( allowMissingColumns ) { } - virtual ~ParquetReader() = default; - ParquetReader( const ParquetReader &other ) = delete; - ParquetReader &operator=( const ParquetReader &other ) = delete; + virtual ~ParquetReader() = default; + ParquetReader( const ParquetReader & other ) = delete; + ParquetReader & operator=( const ParquetReader & other ) = delete; - inline bool isAllowMissingColumns() const - { - return m_allowMissingColumns; - } + inline bool isAllowMissingColumns() const { return m_allowMissingColumns; } virtual bool start() = 0; @@ -173,7 +174,7 @@ class ParquetReader { for( int i = 0; i < nRows; ++i ) { - if(unlikely(!skipRow())) + if( unlikely( !skipRow() ) ) { return false; } @@ -181,115 +182,111 @@ class ParquetReader return true; } - void dispatchRow() - { - dispatchRow( true ); - } + void dispatchRow() { dispatchRow( true ); } virtual void dispatchRow( bool readNextRow ) = 0; // Check if the reader can't read any data at all (no input files). In that case even the file schema // can't be deduces virtual bool isEmpty() const { return false; } - // Check if the reader has any more data left to read. Note it's different from "isEmpty", isEmpty, will always return - // the same value, while this function will return false only when all data that could be read is already read. This - // function should be called only after start, before that it is not expected to be valid. + // Check if the reader has any more data left to read. Note it's different from "isEmpty", isEmpty, will always + // return the same value, while this function will return false only when all data that could be read is already + // read. This function should be called only after start, before that it is not expected to be valid. virtual bool hasData() const = 0; - ParquetStructAdapter &getStructAdapter( const csp::adapters::utils::StructAdapterInfo &structAdapterInfo ); + ParquetStructAdapter & getStructAdapter( const csp::adapters::utils::StructAdapterInfo & structAdapterInfo ); - virtual ColumnAdapterReference operator[]( const std::string &name ) = 0; + virtual ColumnAdapterReference operator[]( const std::string & name ) = 0; - virtual ParquetColumnAdapter* getCurrentColumnAdapterByIndex( std::size_t index ) = 0; + virtual ParquetColumnAdapter * getCurrentColumnAdapterByIndex( std::size_t index ) = 0; virtual int64_t getCurRow() const = 0; - bool isArrowIPC() const{ return m_arrowIPC; } + bool isArrowIPC() const { return m_arrowIPC; } virtual std::string getCurFileOrTableName() const = 0; - virtual void addSubscriber( const std::string &column, ManagedSimInputAdapter *inputAdapter, - const std::optional &symbol ) + virtual void addSubscriber( const std::string & column, ManagedSimInputAdapter * inputAdapter, + const std::optional & symbol ) { if( symbol ) validateSymbolType( symbol.value() ); - (*this)[column]->addSubscriber( inputAdapter, symbol ); + ( *this )[column]->addSubscriber( inputAdapter, symbol ); } - virtual void addListSubscriber( const std::string &column, ManagedSimInputAdapter *inputAdapter, - const std::optional &symbol, const DialectGenericListReaderInterface::Ptr &listReaderInterface ) + virtual void addListSubscriber( const std::string & column, ManagedSimInputAdapter * inputAdapter, + const std::optional & symbol, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) { if( symbol ) validateSymbolType( symbol.value() ); - auto columnAdapterReference = (*this)[column]; + auto columnAdapterReference = ( *this )[column]; - if(columnAdapterReference->getContainerValueType()->type() == csp::CspType::Type::STRING) + if( columnAdapterReference->getContainerValueType()->type() == csp::CspType::Type::STRING ) { - auto &listAdapter = dynamic_cast &>(*columnAdapterReference); - listAdapter.addSubscriber( inputAdapter, symbol, - listReaderInterface ); + auto & listAdapter + = dynamic_cast &>( *columnAdapterReference ); + listAdapter.addSubscriber( inputAdapter, symbol, listReaderInterface ); } else { - PartialSwitchCspType::invoke( - columnAdapterReference -> getContainerValueType().get(), - [ &columnAdapterReference, &listReaderInterface, &symbol, &inputAdapter ]( auto tag ) - { - using T = typename decltype(tag)::type; - using ArrowArrayType = typename arrow::TypeTraits::ArrowType>::ArrayType; - auto &listAdapter = dynamic_cast &>(*columnAdapterReference); - listAdapter.addSubscriber( inputAdapter, symbol, - listReaderInterface ); - } ); + PartialSwitchCspType:: + invoke( columnAdapterReference->getContainerValueType().get(), + [&columnAdapterReference, &listReaderInterface, &symbol, &inputAdapter]( auto tag ) + { + using T = typename decltype( tag )::type; + using ArrowArrayType = + typename arrow::TypeTraits::ArrowType>::ArrayType; + auto & listAdapter + = dynamic_cast &>( *columnAdapterReference ); + listAdapter.addSubscriber( inputAdapter, symbol, listReaderInterface ); + } ); } } /** - * Add parquet struct adapter that depends on columns from "this" and needs to be updated if the schema of the read files changes. + * Add parquet struct adapter that depends on columns from "this" and needs to be updated if the schema of the read + * files changes. * @param adapter */ - void addDependentStructAdapter(ParquetStructAdapter* adapter) {m_dependentAdapters.insert(adapter);} + void addDependentStructAdapter( ParquetStructAdapter * adapter ) { m_dependentAdapters.insert( adapter ); } /** - * Set a known schema of a struct column. Any future files that will have the column will automatically bind to the given csp struct - * metadata + * Set a known schema of a struct column. Any future files that will have the column will automatically bind to the + * given csp struct metadata * @param columnName * @param metaData */ - void setStructColumnMeta(const std::string& columnName, const std::shared_ptr& metaData) + void setStructColumnMeta( const std::string & columnName, const std::shared_ptr & metaData ) { m_structMetaByColumnName[columnName] = metaData; } /** - * @return The known mappings of struct columns to the corresponding csp metadata. NOTE: the values might not be set for all struct columns, - * currently the values are set only for struct columns that are used as fields inside a wrapping struct (i.e only for columns which are - * user as substructs). + * @return The known mappings of struct columns to the corresponding csp metadata. NOTE: the values might not be set + * for all struct columns, currently the values are set only for struct columns that are used as fields inside a + * wrapping struct (i.e only for columns which are user as substructs). */ - const StructMetaByColumnName& getStructColumnMeta() const - { - return m_structMetaByColumnName; - } + const StructMetaByColumnName & getStructColumnMeta() const { return m_structMetaByColumnName; } + protected: using StructAdapterInfo = csp::adapters::utils::StructAdapterInfo; using StructAdapterContainer = std::vector>; - using StructInfoToIndexMap = std::unordered_map; + using StructInfoToIndexMap = std::unordered_map; - inline const StructAdapterContainer &getStructAdapters() const - { - return m_structAdapters; - } + inline const StructAdapterContainer & getStructAdapters() const { return m_structAdapters; } const utils::Symbol * getCurSymbol(); void setSymbolColumnAdapter( ColumnAdapterReference adapter ); - ColumnAdapterReference &getSymbolColumnAdapter(){ return m_symbolColumn; } + ColumnAdapterReference & getSymbolColumnAdapter() { return m_symbolColumn; } + + const std::optional & getSymbolColumnName() const { return m_symbolColumnName; } + const std::set & getDependentAdapters() const { return m_dependentAdapters; } - const std::optional &getSymbolColumnName() const{ return m_symbolColumnName; } - const std::set& getDependentAdapters() const {return m_dependentAdapters;} private: void validateSymbolType( const utils::Symbol & symbol ); @@ -307,12 +304,12 @@ class ParquetReader inline ParquetColumnAdapter & ColumnAdapterReference::operator*() const { - return *( m_reader -> getCurrentColumnAdapterByIndex( m_index ) ); + return *( m_reader->getCurrentColumnAdapterByIndex( m_index ) ); } struct ColumnSubscriberInfo { - ManagedSimInputAdapter * m_inputAdapter; + ManagedSimInputAdapter * m_inputAdapter; std::optional m_symbol; }; @@ -323,7 +320,7 @@ struct ListColumnSubscriberInfo : public ColumnSubscriberInfo struct ColumnSubscriptionContainer { - std::map> m_scalarColumnSubscriptions; + std::map> m_scalarColumnSubscriptions; std::map> m_listColumnSubscriptions; }; @@ -331,7 +328,7 @@ class SingleTableParquetReader : public ParquetReader { public: using NewBatchCallback = std::function & )>; - using NewRowCallback = std::function; + using NewRowCallback = std::function; SingleTableParquetReader( std::vector columns, bool arrowIPC, bool allowMissingColumns, std::optional symbolColumnName = {} ); @@ -343,73 +340,77 @@ class SingleTableParquetReader : public ParquetReader using ParquetReader::dispatchRow; void dispatchRow( bool doReadNextRow ) override; - void dispatchRow( bool doReadNextRow, const utils::Symbol *symbol ); + void dispatchRow( bool doReadNextRow, const utils::Symbol * symbol ); - bool isEmpty() const override {return m_columnAdapters.empty();} + bool isEmpty() const override { return m_columnAdapters.empty(); } bool hasData() const override; - ColumnAdapterReference operator[]( const std::string &name ) override + ColumnAdapterReference operator[]( const std::string & name ) override { auto it = m_columnNameToAdapterIndex.find( name ); CSP_TRUE_OR_THROW_RUNTIME( it != m_columnNameToAdapterIndex.end(), "No column " << name << " found in parquet file" ); - return ColumnAdapterReference(this, it->second); + return ColumnAdapterReference( this, it->second ); } - ParquetColumnAdapter* getCurrentColumnAdapterByIndex( std::size_t index ) override + ParquetColumnAdapter * getCurrentColumnAdapterByIndex( std::size_t index ) override { return m_columnAdapters[index].get(); } - int64_t getCurRow() const override{ return m_curTableNextRow; } + int64_t getCurRow() const override { return m_curTableNextRow; } - - void addSubscriber( const std::string &column, ManagedSimInputAdapter *inputAdapter, - const std::optional &symbol ) override + void addSubscriber( const std::string & column, ManagedSimInputAdapter * inputAdapter, + const std::optional & symbol ) override { ParquetReader::addSubscriber( column, inputAdapter, symbol ); - m_columnSubscriptionContainer.m_scalarColumnSubscriptions[column].push_back(ColumnSubscriberInfo{inputAdapter, symbol}); + m_columnSubscriptionContainer.m_scalarColumnSubscriptions[column].push_back( + ColumnSubscriberInfo{ inputAdapter, symbol } ); } - virtual void addListSubscriber( const std::string &column, ManagedSimInputAdapter *inputAdapter, - const std::optional &symbol, const DialectGenericListReaderInterface::Ptr &listReaderInterface ) override + virtual void addListSubscriber( const std::string & column, ManagedSimInputAdapter * inputAdapter, + const std::optional & symbol, + const DialectGenericListReaderInterface::Ptr & listReaderInterface ) override { - ParquetReader::addListSubscriber( column, inputAdapter, symbol, listReaderInterface); - m_columnSubscriptionContainer.m_listColumnSubscriptions[column].push_back(ListColumnSubscriberInfo{{inputAdapter, symbol}, listReaderInterface}); + ParquetReader::addListSubscriber( column, inputAdapter, symbol, listReaderInterface ); + m_columnSubscriptionContainer.m_listColumnSubscriptions[column].push_back( + ListColumnSubscriberInfo{ { inputAdapter, symbol }, listReaderInterface } ); } protected: - void init(); - void setColumnAdaptersFromCurrentTable(); - virtual bool openNextFile() = 0; + void init(); + void setColumnAdaptersFromCurrentTable(); + virtual bool openNextFile() = 0; virtual bool readNextRowGroup() = 0; virtual void clear(); - void resubscribeAll() + void resubscribeAll() { - for(auto&& column_vector_pair:m_columnSubscriptionContainer.m_scalarColumnSubscriptions) + for( auto && column_vector_pair : m_columnSubscriptionContainer.m_scalarColumnSubscriptions ) { - for(auto&& record:column_vector_pair.second) + for( auto && record : column_vector_pair.second ) { - // We need to call the base class subscribe since we don't want on resubscription to add columns again to the container - ParquetReader::addSubscriber(column_vector_pair.first, record.m_inputAdapter, record.m_symbol); + // We need to call the base class subscribe since we don't want on resubscription to add columns again + // to the container + ParquetReader::addSubscriber( column_vector_pair.first, record.m_inputAdapter, record.m_symbol ); } } - for(auto&& column_vector_pair:m_columnSubscriptionContainer.m_listColumnSubscriptions) + for( auto && column_vector_pair : m_columnSubscriptionContainer.m_listColumnSubscriptions ) { - for(auto&& record:column_vector_pair.second) + for( auto && record : column_vector_pair.second ) { - // We need to call the base class subscribe since we don't want on resubscription to add columns again to the container - ParquetReader::addListSubscriber(column_vector_pair.first, record.m_inputAdapter, record.m_symbol, record.listReader); + // We need to call the base class subscribe since we don't want on resubscription to add columns again + // to the container + ParquetReader::addListSubscriber( column_vector_pair.first, record.m_inputAdapter, record.m_symbol, + record.listReader ); } } - for(auto&& dependentAdapter: getDependentAdapters()) + for( auto && dependentAdapter : getDependentAdapters() ) { dependentAdapter->onSchemaChange(); } } - protected: std::vector m_columns; std::vector> m_columnAdapters; @@ -427,10 +428,11 @@ class SingleFileParquetReader final : public SingleTableParquetReader public: using GeneratorPtr = csp::Generator::Ptr; - SingleFileParquetReader( GeneratorPtr generatorPtr, std::vector columns, bool arrowIPC, bool allowMissingColumns, - bool allowMissingFiles = false, std::optional symbolColumnName = {} ); + SingleFileParquetReader( GeneratorPtr generatorPtr, std::vector columns, bool arrowIPC, + bool allowMissingColumns, bool allowMissingFiles = false, + std::optional symbolColumnName = {} ); - std::string getCurFileOrTableName() const override{ return m_fileName; } + std::string getCurFileOrTableName() const override { return m_fileName; } protected: bool openNextFile() override; @@ -450,10 +452,9 @@ class InMemoryTableParquetReader final : public SingleTableParquetReader public: using GeneratorPtr = csp::Generator, csp::DateTime, csp::DateTime>::Ptr; - InMemoryTableParquetReader( GeneratorPtr generatorPtr, std::vector columns, - bool allowMissingColumns, + InMemoryTableParquetReader( GeneratorPtr generatorPtr, std::vector columns, bool allowMissingColumns, std::optional symbolColumnName = {} ); - std::string getCurFileOrTableName() const override{ return "IN_MEMORY_TABLE"; } + std::string getCurFileOrTableName() const override { return "IN_MEMORY_TABLE"; } protected: bool openNextFile() override; @@ -471,11 +472,12 @@ class MultipleFileParquetReader final : public ParquetReader { public: using NewBatchCallback = std::function & )>; - using NewRowCallback = std::function; - using GeneratorPtr = csp::Generator::Ptr; + using NewRowCallback = std::function; + using GeneratorPtr = csp::Generator::Ptr; - MultipleFileParquetReader( FileNameGeneratorReplicator::Ptr generatorReplicatorPtr, std::vector columns, bool arrowIPC, - bool allowMissingColumns, std::optional symbolColumnName = {} ); + MultipleFileParquetReader( FileNameGeneratorReplicator::Ptr generatorReplicatorPtr, + std::vector columns, bool arrowIPC, bool allowMissingColumns, + std::optional symbolColumnName = {} ); bool start() override; @@ -486,12 +488,13 @@ class MultipleFileParquetReader final : public ParquetReader bool hasData() const override; - ColumnAdapterReference operator[]( const std::string &name ) override; - ParquetColumnAdapter* getCurrentColumnAdapterByIndex( std::size_t index ) override; + ColumnAdapterReference operator[]( const std::string & name ) override; + ParquetColumnAdapter * getCurrentColumnAdapterByIndex( std::size_t index ) override; + + int64_t getCurRow() const override { CSP_NOT_IMPLEMENTED; } - int64_t getCurRow() const override{ CSP_NOT_IMPLEMENTED; } + std::string getCurFileOrTableName() const override { return "MULTI_FILE_READER"; } - std::string getCurFileOrTableName() const override{ return "MULTI_FILE_READER"; } private: void clear(); @@ -501,7 +504,6 @@ class MultipleFileParquetReader final : public ParquetReader std::unordered_map m_columnReaderByName; }; -} - +} // namespace csp::adapters::parquet #endif //_IN_CSP_ADAPTERS_PARQUET_ParquetIterator_H diff --git a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp index 3fe763a60..6c8e1b39a 100644 --- a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp +++ b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp @@ -1,157 +1,158 @@ -#include #include +#include #include +#include #include #include -#include namespace csp::adapters::parquet { -template< typename ArrowArrayType > -static inline std::unique_ptr createDateColumnAdapter( - ParquetReader &reader, - const std::string& columnName, - const std::shared_ptr &dateType ) +template +static inline std::unique_ptr +createDateColumnAdapter( ParquetReader & reader, const std::string & columnName, + const std::shared_ptr & dateType ) { - switch( dateType -> unit() ) + switch( dateType->unit() ) { case arrow::DateUnit::MILLI: return std::unique_ptr( - new DateColumnAdapter<1000000, ArrowArrayType>( reader, columnName ) ); + new DateColumnAdapter<1000000, ArrowArrayType>( reader, columnName ) ); case arrow::DateUnit::DAY: return std::unique_ptr( - new DateColumnAdapter<1000000000LL * 3600 * 24, ArrowArrayType>( reader, columnName ) ); + new DateColumnAdapter<1000000000LL * 3600 * 24, ArrowArrayType>( reader, columnName ) ); } - CSP_THROW( csp::TypeError, "Unexpected day unit: " << ( int ) dateType -> unit() << " for column " << columnName ); + CSP_THROW( csp::TypeError, "Unexpected day unit: " << (int)dateType->unit() << " for column " << columnName ); } -template< typename ArrowArrayType > -static inline std::unique_ptr createTimeColumnAdapter( - ParquetReader &reader, - const std::string& columnName, - const std::shared_ptr &timeType ) +template +static inline std::unique_ptr +createTimeColumnAdapter( ParquetReader & reader, const std::string & columnName, + const std::shared_ptr & timeType ) { ParquetColumnAdapter * adapter; - switch( timeType -> unit() ) - { - case arrow::TimeUnit::SECOND: adapter = new TimeColumnAdapter<1000000000,ArrowArrayType>( reader, columnName ); break; - case arrow::TimeUnit::MILLI: adapter = new TimeColumnAdapter<1000000,ArrowArrayType>( reader, columnName ); break; - case arrow::TimeUnit::MICRO: adapter = new TimeColumnAdapter<1000,ArrowArrayType>( reader, columnName ); break; - case arrow::TimeUnit::NANO: adapter = new TimeColumnAdapter<1,ArrowArrayType>( reader, columnName ); break; + switch( timeType->unit() ) + { + case arrow::TimeUnit::SECOND: + adapter = new TimeColumnAdapter<1000000000, ArrowArrayType>( reader, columnName ); + break; + case arrow::TimeUnit::MILLI: + adapter = new TimeColumnAdapter<1000000, ArrowArrayType>( reader, columnName ); + break; + case arrow::TimeUnit::MICRO: + adapter = new TimeColumnAdapter<1000, ArrowArrayType>( reader, columnName ); + break; + case arrow::TimeUnit::NANO: + adapter = new TimeColumnAdapter<1, ArrowArrayType>( reader, columnName ); + break; } return std::unique_ptr( adapter ); } -std::unique_ptr createColumnAdapter( - ParquetReader &parquetReader, - const ::arrow::Field &field, - const std::string &fileName, - const std::map>* structMetaByColumnName) +std::unique_ptr +createColumnAdapter( ParquetReader & parquetReader, const ::arrow::Field & field, const std::string & fileName, + const std::map> * structMetaByColumnName ) { - auto typeId = field.type() -> id(); + auto typeId = field.type()->id(); switch( typeId ) { case arrow::Type::BOOL: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::UINT8: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::INT8: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::UINT16: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::INT16: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::UINT32: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::INT32: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::UINT64: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::INT64: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::HALF_FLOAT: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::FLOAT: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::DOUBLE: return std::unique_ptr( - new NativeTypeColumnAdapter( parquetReader, field.name() ) ); + new NativeTypeColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::STRING: return std::unique_ptr( new StringColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::BINARY: return std::unique_ptr( new BytesColumnAdapter( parquetReader, field.name() ) ); case arrow::Type::DATE32: - return createDateColumnAdapter( parquetReader, - field.name(), - std::static_pointer_cast( - field.type() ) ); + return createDateColumnAdapter( + parquetReader, field.name(), std::static_pointer_cast( field.type() ) ); case arrow::Type::DATE64: - return createDateColumnAdapter( parquetReader, - field.name(), - std::static_pointer_cast( - field.type() ) ); + return createDateColumnAdapter( + parquetReader, field.name(), std::static_pointer_cast( field.type() ) ); case arrow::Type::TIME32: - return createTimeColumnAdapter( parquetReader, field.name(), - std::static_pointer_cast( field.type() ) ); + return createTimeColumnAdapter( + parquetReader, field.name(), std::static_pointer_cast( field.type() ) ); case arrow::Type::TIME64: - return createTimeColumnAdapter( parquetReader, field.name(), - std::static_pointer_cast( field.type() ) ); + return createTimeColumnAdapter( + parquetReader, field.name(), std::static_pointer_cast( field.type() ) ); case arrow::Type::TIMESTAMP: { auto timestampType = std::static_pointer_cast( field.type() ); -// CSP_TRUE_OR_THROW_RUNTIME( -// timestampType -> timezone().empty() || timestampType -> timezone() == "UTC" || timestampType -> timezone() == "utc", -// "Unexpected parquet column timezone: " << timestampType -> timezone() ); - switch( timestampType -> unit() ) + // CSP_TRUE_OR_THROW_RUNTIME( + // timestampType -> timezone().empty() || timestampType -> timezone() == "UTC" || + // timestampType -> timezone() == "utc", "Unexpected parquet column timezone: " << + // timestampType -> timezone() ); + switch( timestampType->unit() ) { case arrow::TimeUnit::SECOND: return std::unique_ptr( - new DatetimeColumnAdapter<1000000000>( parquetReader, field.name() ) ); + new DatetimeColumnAdapter<1000000000>( parquetReader, field.name() ) ); case arrow::TimeUnit::MILLI: return std::unique_ptr( - new DatetimeColumnAdapter<1000000>( parquetReader, field.name() ) ); + new DatetimeColumnAdapter<1000000>( parquetReader, field.name() ) ); case arrow::TimeUnit::MICRO: return std::unique_ptr( - new DatetimeColumnAdapter<1000>( parquetReader, field.name() ) ); + new DatetimeColumnAdapter<1000>( parquetReader, field.name() ) ); case arrow::TimeUnit::NANO: return std::unique_ptr( - new DatetimeColumnAdapter<1>( parquetReader, field.name() ) ); + new DatetimeColumnAdapter<1>( parquetReader, field.name() ) ); } } case arrow::Type::DURATION: { auto durationType = std::static_pointer_cast( field.type() ); - switch( durationType -> unit() ) + switch( durationType->unit() ) { case arrow::TimeUnit::SECOND: return std::unique_ptr( - new DurationColumnAdapter<1000000000>( parquetReader, field.name() ) ); + new DurationColumnAdapter<1000000000>( parquetReader, field.name() ) ); case arrow::TimeUnit::MILLI: return std::unique_ptr( - new DurationColumnAdapter<1000000>( parquetReader, field.name() ) ); + new DurationColumnAdapter<1000000>( parquetReader, field.name() ) ); case arrow::TimeUnit::MICRO: return std::unique_ptr( - new DurationColumnAdapter<1000>( parquetReader, field.name() ) ); + new DurationColumnAdapter<1000>( parquetReader, field.name() ) ); case arrow::TimeUnit::NANO: return std::unique_ptr( - new DurationColumnAdapter<1>( parquetReader, field.name() ) ); + new DurationColumnAdapter<1>( parquetReader, field.name() ) ); } } case arrow::Type::FIXED_SIZE_BINARY: @@ -162,24 +163,24 @@ std::unique_ptr createColumnAdapter( { auto dictionaryType = std::static_pointer_cast( field.type() ); - if( dictionaryType -> value_type() -> id() != arrow::Type::STRING ) + if( dictionaryType->value_type()->id() != arrow::Type::STRING ) { CSP_THROW( ParquetColumnTypeError, - "Unsupported dictionary column type " << field.type() -> name() << " in file " << fileName + "Unsupported dictionary column type " << field.type()->name() << " in file " << fileName << " only string values supported" ); } return std::make_unique( parquetReader, field.name() ); } case arrow::Type::STRUCT: { - auto res = std::make_unique( parquetReader, std::static_pointer_cast<::arrow::StructType>( field.type() ), - field.name() ); - if(structMetaByColumnName!= nullptr) + auto res = std::make_unique( + parquetReader, std::static_pointer_cast<::arrow::StructType>( field.type() ), field.name() ); + if( structMetaByColumnName != nullptr ) { - auto metaIt = structMetaByColumnName->find(field.name() ); - if(metaIt != structMetaByColumnName->end()) + auto metaIt = structMetaByColumnName->find( field.name() ); + if( metaIt != structMetaByColumnName->end() ) { - res->setStructMeta(metaIt->second); + res->setStructMeta( metaIt->second ); } } return res; @@ -187,8 +188,8 @@ std::unique_ptr createColumnAdapter( case arrow::Type::LIST: { auto listType = std::static_pointer_cast( field.type() ); - auto valueType = listType -> value_type(); - switch( valueType -> id() ) + auto valueType = listType->value_type(); + switch( valueType->id() ) { case arrow::Type::INT64: return std::make_unique>( parquetReader, field.name() ); @@ -197,58 +198,65 @@ std::unique_ptr createColumnAdapter( case arrow::Type::BOOL: return std::make_unique>( parquetReader, field.name() ); case arrow::Type::STRING: - return std::make_unique>( parquetReader, field.name() ); + return std::make_unique>( parquetReader, + field.name() ); default: - CSP_THROW( TypeError, "Trying to create arrow list array reader for unsupported element type " << valueType -> name() << " for column " << field.name() ); + CSP_THROW( TypeError, + "Trying to create arrow list array reader for unsupported element type " + << valueType->name() << " for column " << field.name() ); } } case arrow::Type::LARGE_STRING: default: CSP_THROW( ParquetColumnTypeError, - "Unsupported column type " << field.type() -> name() << " for column " << field.name() << " in file " << fileName ); + "Unsupported column type " << field.type()->name() << " for column " << field.name() + << " in file " << fileName ); } } -std::unique_ptr createMissingColumnAdapter(ParquetReader &parquetReader,const std::string& columnName) +std::unique_ptr createMissingColumnAdapter( ParquetReader & parquetReader, + const std::string & columnName ) { return std::make_unique( parquetReader, columnName ); } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void BaseTypedColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, - std::optional symbol ) +template +void BaseTypedColumnAdapter::addSubscriber( + ManagedSimInputAdapter * inputAdapter, std::optional symbol ) { try { - auto callback = CompatibleTypeSwitch::invoke( inputAdapter -> type(), [ inputAdapter ]( auto tag ) - { - return std::function( [ inputAdapter ]( const ValueType *val ) - { - if( val ) - { - inputAdapter -> pushTick( *val ); - } - else - { - inputAdapter -> pushNullTick(); - } - } ); - - } ); + auto callback = CompatibleTypeSwitch::invoke( + inputAdapter->type(), + [inputAdapter]( auto tag ) + { + return std::function( + [inputAdapter]( const ValueType * val ) + { + if( val ) + { + inputAdapter->pushTick( *val ); + } + else + { + inputAdapter->pushNullTick(); + } + } ); + } ); m_dispatcher.addSubscriber( callback, symbol ); } - catch( UnsupportedSwitchType &e ) + catch( UnsupportedSwitchType & e ) { - CSP_THROW( TypeError, "Unexpected column type for column " << getColumnName() << " , expected " - << inputAdapter -> type() -> type().asCString() << - " got " << ArrowArrayType::TypeClass::type_name() ); + CSP_THROW( TypeError, + "Unexpected column type for column " << getColumnName() << " , expected " + << inputAdapter->type()->type().asCString() << " got " + << ArrowArrayType::TypeClass::type_name() ); } } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void BaseTypedColumnAdapter::dispatchValue( const utils::Symbol *symbol ) +template +void BaseTypedColumnAdapter::dispatchValue( const utils::Symbol * symbol ) { if( m_curValue.has_value() ) { @@ -260,217 +268,223 @@ void BaseTypedColumnAdapter::dispat } } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > +template void BaseTypedColumnAdapter::ensureType( CspType::Ptr cspType ) { try { - CompatibleTypeSwitch::invoke( cspType.get(), [ cspType, this ]( auto tag ) - { - // No need to check native types here, they won't be assignable so it will raise exception. - if( cspType -> type() > CspType::Type::MAX_NATIVE_TYPE ) + CompatibleTypeSwitch::invoke( + cspType.get(), + [cspType, this]( auto tag ) { - CSP_TRUE_OR_THROW( ( std::is_same::value ), TypeError, - "Unexpected column type for column " << getColumnName() << " , expected " << cspType -> type().asCString() << - " got " << ArrowArrayType::TypeClass::type_name() ); - } - } ); + // No need to check native types here, they won't be assignable so it will raise exception. + if( cspType->type() > CspType::Type::MAX_NATIVE_TYPE ) + { + CSP_TRUE_OR_THROW( ( std::is_same::value ), TypeError, + "Unexpected column type for column " << getColumnName() << " , expected " + << cspType->type().asCString() << " got " + << ArrowArrayType::TypeClass::type_name() ); + } + } ); } - catch( UnsupportedSwitchType &e ) + catch( UnsupportedSwitchType & e ) { - CSP_THROW( TypeError, "Unexpected column type for column " << getColumnName() << " , expected " << cspType -> type().asCString() << - " got " << ArrowArrayType::TypeClass::type_name() ); + CSP_THROW( TypeError, + "Unexpected column type for column " << getColumnName() << " , expected " + << cspType->type().asCString() << " got " + << ArrowArrayType::TypeClass::type_name() ); } - } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void *BaseTypedColumnAdapter::getCurValueUntyped() +template +void * BaseTypedColumnAdapter::getCurValueUntyped() { return &m_curValue; } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void -BaseTypedColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> &data ) +template +void BaseTypedColumnAdapter::handleNewBatch( + const std::shared_ptr<::arrow::ChunkedArray> & data ) { - CSP_TRUE_OR_THROW_RUNTIME( data -> num_chunks() == 1, - "Unexpected number of chunks in column" << data -> num_chunks() ); - m_curChunkArray = std::static_pointer_cast( data -> chunk( 0 ) ); + CSP_TRUE_OR_THROW_RUNTIME( data->num_chunks() == 1, "Unexpected number of chunks in column" << data->num_chunks() ); + m_curChunkArray = std::static_pointer_cast( data->chunk( 0 ) ); } -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT > -void -BaseTypedColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::Array> &data ) +template +void BaseTypedColumnAdapter::handleNewBatch( + const std::shared_ptr<::arrow::Array> & data ) { m_curChunkArray = std::static_pointer_cast( data ); } -ParquetStructAdapter::ParquetStructAdapter( ParquetReader &parquetReader, StructAdapterInfo adapterInfo ) - : m_parquetReader( parquetReader ), - m_structMeta( std::static_pointer_cast( adapterInfo.type() ) -> meta() ) +ParquetStructAdapter::ParquetStructAdapter( ParquetReader & parquetReader, StructAdapterInfo adapterInfo ) + : m_parquetReader( parquetReader ) + , m_structMeta( std::static_pointer_cast( adapterInfo.type() )->meta() ) { - m_resetFunc = [ this, adapterInfo ]() + m_resetFunc = [this, adapterInfo]() { m_fieldSetters.clear(); - for( auto it = adapterInfo.fieldMap() -> begin(); it != adapterInfo.fieldMap() -> end(); ++it ) + for( auto it = adapterInfo.fieldMap()->begin(); it != adapterInfo.fieldMap()->end(); ++it ) { - createFieldSetter( it.value(), *m_parquetReader[ it.key() ] ); + createFieldSetter( it.value(), *m_parquetReader[it.key()] ); } }; m_resetFunc(); - // Note the columns for single struct can come from multiple different readers, so we have to iterate over all columns. - // MultipleFileParquetReader uses the columns from child SingleFileParquetReader readers. - for( auto it = adapterInfo.fieldMap() -> begin(); it != adapterInfo.fieldMap() -> end(); ++it ) + // Note the columns for single struct can come from multiple different readers, so we have to iterate over all + // columns. MultipleFileParquetReader uses the columns from child SingleFileParquetReader readers. + for( auto it = adapterInfo.fieldMap()->begin(); it != adapterInfo.fieldMap()->end(); ++it ) { - auto&& parquetColumnName = it.key(); - auto&& structFieldName = it.value(); - auto& columnReader = m_parquetReader[ parquetColumnName ]->getReader(); - columnReader.addDependentStructAdapter(this); - auto &&fieldPtr = m_structMeta -> field( structFieldName ); - if(fieldPtr -> type() -> type() == CspType::TypeTraits::STRUCT) + auto && parquetColumnName = it.key(); + auto && structFieldName = it.value(); + auto & columnReader = m_parquetReader[parquetColumnName]->getReader(); + columnReader.addDependentStructAdapter( this ); + auto && fieldPtr = m_structMeta->field( structFieldName ); + if( fieldPtr->type()->type() == CspType::TypeTraits::STRUCT ) { - auto &fieldMeta = std::static_pointer_cast( fieldPtr -> type() ) -> meta(); - columnReader.setStructColumnMeta(parquetColumnName, fieldMeta); + auto & fieldMeta = std::static_pointer_cast( fieldPtr->type() )->meta(); + columnReader.setStructColumnMeta( parquetColumnName, fieldMeta ); } } } -ParquetStructAdapter::ParquetStructAdapter( ParquetReader &parquetReader, - std::shared_ptr<::arrow::StructType> arrowType, - const std::shared_ptr &structMeta, - const std::vector> &columnAdapters ) - : m_parquetReader( parquetReader ), - m_structMeta( structMeta ) +ParquetStructAdapter::ParquetStructAdapter( ParquetReader & parquetReader, + std::shared_ptr<::arrow::StructType> arrowType, + const std::shared_ptr & structMeta, + const std::vector> & columnAdapters ) + : m_parquetReader( parquetReader ) + , m_structMeta( structMeta ) { - m_resetFunc = []() - { - CSP_THROW( RuntimeException, "Internal error, trying to reset single column struct adapter" ); - }; + m_resetFunc + = []() { CSP_THROW( RuntimeException, "Internal error, trying to reset single column struct adapter" ); }; - auto &arrowColumns = arrowType -> fields(); + auto & arrowColumns = arrowType->fields(); - CSP_TRUE_OR_THROW_RUNTIME( arrowColumns.size() == columnAdapters.size(), "Found mismatch between arrow and csp schema" ); + CSP_TRUE_OR_THROW_RUNTIME( arrowColumns.size() == columnAdapters.size(), + "Found mismatch between arrow and csp schema" ); for( std::size_t i = 0; i < arrowColumns.size(); ++i ) { - auto &arrowCol = *arrowColumns[ i ]; - createFieldSetter( arrowCol.name(), *columnAdapters[ i ] ); + auto & arrowCol = *arrowColumns[i]; + createFieldSetter( arrowCol.name(), *columnAdapters[i] ); } } -void ParquetStructAdapter::createFieldSetter( const std::string &fieldName, - ParquetColumnAdapter &columnAdapter ) +void ParquetStructAdapter::createFieldSetter( const std::string & fieldName, ParquetColumnAdapter & columnAdapter ) { - if(columnAdapter.isMissingColumn()) + if( columnAdapter.isMissingColumn() ) { return; } - auto &&fieldPtr = m_structMeta -> field( fieldName ); - CSP_TRUE_OR_THROW_RUNTIME( fieldPtr != nullptr, - "No field " << fieldName << " in struct " << m_structMeta -> name() ); + auto && fieldPtr = m_structMeta->field( fieldName ); + CSP_TRUE_OR_THROW_RUNTIME( fieldPtr != nullptr, "No field " << fieldName << " in struct " << m_structMeta->name() ); FieldSetter fieldSetter; - if( fieldPtr -> type() -> type() == CspType::TypeTraits::ENUM ) + if( fieldPtr->type()->type() == CspType::TypeTraits::ENUM ) { columnAdapter.ensureType( CspType::STRING() ); - auto enumMetaPtr = std::static_pointer_cast( fieldPtr -> type() ) -> meta(); + auto enumMetaPtr = std::static_pointer_cast( fieldPtr->type() )->meta(); using ColumnType = CspType::Type::toCType::type; - fieldSetter = [ &columnAdapter, fieldPtr, enumMetaPtr ]( StructPtr &s ) + fieldSetter = [&columnAdapter, fieldPtr, enumMetaPtr]( StructPtr & s ) { auto curValue = columnAdapter.getCurValue(); if( curValue.has_value() ) { - fieldPtr -> setValue( s.get(), enumMetaPtr -> fromString( curValue.value().c_str() ) ); + fieldPtr->setValue( s.get(), enumMetaPtr->fromString( curValue.value().c_str() ) ); } }; } else { using TypeSwitch = PrimitiveCspTypeSwitch::Extend; - columnAdapter.ensureType( fieldPtr -> type() ); - if( fieldPtr -> type() -> type() == CspType::Type::STRUCT ) + columnAdapter.ensureType( fieldPtr->type() ); + if( fieldPtr->type()->type() == CspType::Type::STRUCT ) { - auto &fieldMeta = std::static_pointer_cast( fieldPtr -> type() ) -> meta(); - static_cast(columnAdapter).initFromStructMeta( fieldMeta ); + auto & fieldMeta = std::static_pointer_cast( fieldPtr->type() )->meta(); + static_cast( columnAdapter ).initFromStructMeta( fieldMeta ); } if( columnAdapter.isNativeType() ) { fieldSetter = NativeCspTypeSwitch::invoke( - ( columnAdapter.getNativeCspType().get() ), [ &columnAdapter, &fieldPtr ]( auto columnTag ) - { - using ColType = typename decltype(columnTag)::type; - return ConstructibleTypeSwitch::invoke( - fieldPtr -> type().get(), - [ &columnAdapter, &fieldPtr ]( auto tag ) + ( columnAdapter.getNativeCspType().get() ), + [&columnAdapter, &fieldPtr]( auto columnTag ) + { + using ColType = typename decltype( columnTag )::type; + return ConstructibleTypeSwitch::invoke( + fieldPtr->type().get(), + [&columnAdapter, &fieldPtr]( auto tag ) + { + using FieldType = typename decltype( tag )::type; + return FieldSetter( + [&columnAdapter, fieldPtr]( StructPtr & s ) { - using FieldType = typename decltype(tag)::type; - return FieldSetter( [ &columnAdapter, fieldPtr ]( StructPtr &s ) - { - auto curValue = columnAdapter.getCurValue(); - if( curValue.has_value() ) - { - fieldPtr -> setValue( s.get(), csp::cast( - curValue.value() ) ); - } - } ); + auto curValue = columnAdapter.getCurValue(); + if( curValue.has_value() ) + { + fieldPtr->setValue( s.get(), + csp::cast( curValue.value() ) ); + } } ); - } + } ); + } ); } else { - fieldSetter = TypeSwitch::invoke( fieldPtr -> type().get(), [ &columnAdapter, fieldPtr ]( auto tag ) - { - using FieldType = typename decltype(tag)::type; - return FieldSetter( [ &columnAdapter, fieldPtr ]( StructPtr &s ) - { - auto curValue = columnAdapter.getCurValue(); - if( curValue.has_value() ) - { - fieldPtr -> setValue( s.get(), curValue.value() ); - } - } ); - } ); + fieldSetter + = TypeSwitch::invoke( fieldPtr->type().get(), + [&columnAdapter, fieldPtr]( auto tag ) + { + using FieldType = typename decltype( tag )::type; + return FieldSetter( + [&columnAdapter, fieldPtr]( StructPtr & s ) + { + auto curValue = columnAdapter.getCurValue(); + if( curValue.has_value() ) + { + fieldPtr->setValue( s.get(), curValue.value() ); + } + } ); + } ); } } m_fieldSetters.push_back( fieldSetter ); } -void ParquetStructAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol ) +void ParquetStructAdapter::addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol ) { - CSP_TRUE_OR_THROW( inputAdapter -> type() -> type() == CspType::Type::STRUCT, TypeError, - "Subscribing unexpected type " << inputAdapter -> type() -> type() << " as struct for column " ); - auto meta = static_cast( inputAdapter -> type()) -> meta(); + CSP_TRUE_OR_THROW( inputAdapter->type()->type() == CspType::Type::STRUCT, TypeError, + "Subscribing unexpected type " << inputAdapter->type()->type() << " as struct for column " ); + auto meta = static_cast( inputAdapter->type() )->meta(); CSP_TRUE_OR_THROW( meta == m_structMeta, TypeError, - "Subscribing " << meta -> name() << " where " << m_structMeta -> name() << " is expected" ); - - m_valueDispatcher.addSubscriber( [ inputAdapter ]( StructPtr *s ) - { - if( s ) - { - inputAdapter -> pushTick( *s ); - } - else - { - inputAdapter -> pushNullTick(); - } - }, - symbol ); + "Subscribing " << meta->name() << " where " << m_structMeta->name() << " is expected" ); + m_valueDispatcher.addSubscriber( + [inputAdapter]( StructPtr * s ) + { + if( s ) + { + inputAdapter->pushTick( *s ); + } + else + { + inputAdapter->pushNullTick(); + } + }, + symbol ); } -void ParquetStructAdapter::addSubscriber( ValueDispatcher::SubscriberType subscriber, std::optional symbol ) +void ParquetStructAdapter::addSubscriber( ValueDispatcher::SubscriberType subscriber, + std::optional symbol ) { m_valueDispatcher.addSubscriber( subscriber, symbol ); } -void ParquetStructAdapter::dispatchValue( const utils::Symbol *symbol, bool isNull ) +void ParquetStructAdapter::dispatchValue( const utils::Symbol * symbol, bool isNull ) { - if(unlikely(m_needsReset)) + if( unlikely( m_needsReset ) ) { m_resetFunc(); m_needsReset = false; @@ -484,16 +498,16 @@ void ParquetStructAdapter::dispatchValue( const utils::Symbol *symbol, bool isNu return; } - StructPtr s; - StructPtr *dispatchedValue; + StructPtr s; + StructPtr * dispatchedValue; if( isNull ) { dispatchedValue = nullptr; } else { - s = StructPtr{ m_structMeta -> create() }; - for( auto &fieldSetter : m_fieldSetters ) + s = StructPtr{ m_structMeta->create() }; + for( auto & fieldSetter : m_fieldSetters ) { fieldSetter( s ); } @@ -510,98 +524,98 @@ void ParquetStructAdapter::dispatchValue( const utils::Symbol *symbol, bool isNu } } -template< typename ValueType, typename ArrowArrayType > +template void NativeTypeColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::cast( this -> m_curChunkArray -> Value( curRow ) ); + this->m_curValue = csp::cast( this->m_curChunkArray->Value( curRow ) ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -template< int64_t UNIT > +template void DatetimeColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::DateTime::fromNanoseconds( - m_curChunkArray -> Value( m_parquetReader.getCurRow() ) * UNIT ); + this->m_curValue + = csp::DateTime::fromNanoseconds( m_curChunkArray->Value( m_parquetReader.getCurRow() ) * UNIT ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -template< int64_t UNIT > +template void DurationColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::TimeDelta::fromNanoseconds( - m_curChunkArray -> Value( m_parquetReader.getCurRow() ) * UNIT ); + this->m_curValue + = csp::TimeDelta::fromNanoseconds( m_curChunkArray->Value( m_parquetReader.getCurRow() ) * UNIT ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -template< int64_t UNIT, typename ArrowDateArray > +template void DateColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::DateTime::fromNanoseconds( - this -> m_curChunkArray -> Value( this -> m_parquetReader.getCurRow() ) * UNIT ).date(); + this->m_curValue + = csp::DateTime::fromNanoseconds( this->m_curChunkArray->Value( this->m_parquetReader.getCurRow() ) * UNIT ) + .date(); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -template< int64_t UNIT, typename ArrowTimeArray > +template void TimeColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = csp::Time::fromNanoseconds( - this -> m_curChunkArray -> Value( this -> m_parquetReader.getCurRow() ) * UNIT ); + this->m_curValue + = csp::Time::fromNanoseconds( this->m_curChunkArray->Value( this->m_parquetReader.getCurRow() ) * UNIT ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -void StringColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol ) +void StringColumnAdapter::addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol ) { - if( inputAdapter -> type() -> type() == CspType::TypeTraits::ENUM ) + if( inputAdapter->type()->type() == CspType::TypeTraits::ENUM ) { - auto enumMetaPtr = static_cast( inputAdapter -> type()) -> meta(); - auto callback = std::function( - [ inputAdapter, enumMetaPtr ]( const std::string *val ) + auto enumMetaPtr = static_cast( inputAdapter->type() )->meta(); + auto callback = std::function( + [inputAdapter, enumMetaPtr]( const std::string * val ) + { + if( val ) { - if( val ) - { - inputAdapter -> pushTick( enumMetaPtr -> fromString( val -> c_str() ) ); - } - else - { - inputAdapter -> pushNullTick(); - } - } ); + inputAdapter->pushTick( enumMetaPtr->fromString( val->c_str() ) ); + } + else + { + inputAdapter->pushNullTick(); + } + } ); m_dispatcher.addSubscriber( callback, symbol ); } else @@ -612,73 +626,73 @@ void StringColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, s void StringColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = this -> m_curChunkArray -> GetString( curRow ); + this->m_curValue = this->m_curChunkArray->GetString( curRow ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } void BytesColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = this -> m_curChunkArray -> GetString( curRow ); + this->m_curValue = this->m_curChunkArray->GetString( curRow ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } - void FixedSizeBinaryColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - this -> m_curValue = this -> m_curChunkArray -> GetString( curRow ); + this->m_curValue = this->m_curChunkArray->GetString( curRow ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } void DictionaryColumnAdapter::readCurValue() { - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - auto index = this -> m_curChunkArray -> GetValueIndex( curRow ); - auto dict = static_cast(this -> m_curChunkArray -> dictionary().get()); - this -> m_curValue = dict -> GetString( index ); + auto index = this->m_curChunkArray->GetValueIndex( curRow ); + auto dict = static_cast( this->m_curChunkArray->dictionary().get() ); + this->m_curValue = dict->GetString( index ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } - namespace { template struct ArrayValidValueProvider { template - static T getValue(const V& array, int index) + static T getValue( const V & array, int index ) { - if(!array->IsValid(index)) + if( !array->IsValid( index ) ) { - CSP_THROW(ValueError, "Can't read empty value to array from arrow array of type " << arrow::StringArray::TypeClass::type_name()); + CSP_THROW( ValueError, + "Can't read empty value to array from arrow array of type " + << arrow::StringArray::TypeClass::type_name() ); } - return array->GetView(index); + return array->GetView( index ); } }; @@ -686,207 +700,209 @@ namespace struct ArrayValidValueProvider { template - static double getValue(const V& array, int index) + static double getValue( const V & array, int index ) { - if(!array->IsValid(index)) + if( !array->IsValid( index ) ) { return std::numeric_limits::quiet_NaN(); } - return std::move(array->GetView(index)); + return std::move( array->GetView( index ) ); } }; -} +} // namespace -template< typename ValueArrayType, typename ValueType > -void ListColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol ) +template +void ListColumnAdapter::addSubscriber( ManagedSimInputAdapter * inputAdapter, + std::optional symbol ) { CSP_THROW( NotImplemented, "Trying to subscribe to list column indirectly, via struct field?" ); } -template< typename ValueArrayType, typename ValueType > -void ListColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, - std::optional symbol, - const DialectGenericListReaderInterface::Ptr &listReader ) +template +void ListColumnAdapter::addSubscriber( + ManagedSimInputAdapter * inputAdapter, std::optional symbol, + const DialectGenericListReaderInterface::Ptr & listReader ) { - CSP_TRUE_OR_THROW_RUNTIME( m_listReader == nullptr, - "Trying to subscribe list column in parquet reader more than once, this is not supported" ); + CSP_TRUE_OR_THROW_RUNTIME( + m_listReader == nullptr, + "Trying to subscribe list column in parquet reader more than once, this is not supported" ); CSP_TRUE_OR_THROW_RUNTIME( listReader != nullptr, "Trying to subscribe list column in parquet reader with null listReader" ); Base::addSubscriber( inputAdapter, symbol ); m_listReader = std::dynamic_pointer_cast>( listReader ); CSP_TRUE_OR_THROW_RUNTIME( m_listReader != nullptr, - "Subscribed to parquet column " << getColumnName() << " with type " - << "NumpyArray[" << listReader -> getValueType() -> type().asString() - << "] while " - << " column type in file is NumpyArray[" - << getContainerValueType() -> type().asString() << "]" - << " in file " << m_parquetReader.getCurFileOrTableName() ); + "Subscribed to parquet column " + << getColumnName() << " with type " + << "NumpyArray[" << listReader->getValueType()->type().asString() << "] while " + << " column type in file is NumpyArray[" + << getContainerValueType()->type().asString() << "]" + << " in file " << m_parquetReader.getCurFileOrTableName() ); } -template< typename ValueArrayType, typename ValueType > +template void ListColumnAdapter::readCurValue() { CSP_TRUE_OR_THROW_RUNTIME( m_listReader != nullptr, "Trying to read list value from parquet file but not list reader interface is set" ); - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - auto values = this -> m_curChunkArray -> value_slice( curRow ); + auto values = this->m_curChunkArray->value_slice( curRow ); auto typedValues = std::dynamic_pointer_cast( values ); - auto arrayValue = m_listReader -> create( typedValues -> length() ); - auto* internalBuffer = m_listReader -> getRawDataBuffer( arrayValue ); + auto arrayValue = m_listReader->create( typedValues->length() ); + auto * internalBuffer = m_listReader->getRawDataBuffer( arrayValue ); if( internalBuffer != nullptr ) { - for( int64_t i = 0; i < typedValues -> length(); ++i ) + for( int64_t i = 0; i < typedValues->length(); ++i ) { - *(internalBuffer++) = ArrayValidValueProvider::getValue(typedValues, i); + *( internalBuffer++ ) = ArrayValidValueProvider::getValue( typedValues, i ); } } else { - for( int64_t i = 0; i < typedValues -> length(); ++i ) + for( int64_t i = 0; i < typedValues->length(); ++i ) { - m_listReader -> setValue( arrayValue, i, ArrayValidValueProvider::getValue(typedValues, i) ); + m_listReader->setValue( arrayValue, i, ArrayValidValueProvider::getValue( typedValues, i ) ); } } - this -> m_curValue = std::move( arrayValue ); + this->m_curValue = std::move( arrayValue ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } - template<> void ListColumnAdapter::readCurValue() { CSP_TRUE_OR_THROW_RUNTIME( m_listReader != nullptr, "Trying to read list value from parquet file but not list reader interface is set" ); - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - auto values = this -> m_curChunkArray -> value_slice( curRow ); + auto values = this->m_curChunkArray->value_slice( curRow ); auto typedValues = std::dynamic_pointer_cast( values ); uint32_t maxStringLength = 0; - for( int64_t i = 0; i < typedValues -> length(); ++i ) + for( int64_t i = 0; i < typedValues->length(); ++i ) { - maxStringLength = std::max( ( uint32_t ) ArrayValidValueProvider::getValue(typedValues, i).length(), maxStringLength ); + maxStringLength + = std::max( (uint32_t)ArrayValidValueProvider::getValue( typedValues, i ).length(), + maxStringLength ); } - auto arrayValue = m_listReader -> create( typedValues -> length(), maxStringLength ); + auto arrayValue = m_listReader->create( typedValues->length(), maxStringLength ); - for( int64_t i = 0; i < typedValues -> length(); ++i ) + for( int64_t i = 0; i < typedValues->length(); ++i ) { - m_listReader -> setValue( arrayValue, i, std::string(typedValues -> GetView( i ))); + m_listReader->setValue( arrayValue, i, std::string( typedValues->GetView( i ) ) ); } - this -> m_curValue = std::move( arrayValue ); + this->m_curValue = std::move( arrayValue ); } else { - this -> m_curValue.reset(); + this->m_curValue.reset(); } } -// We need to force the compiler to instantiate the relevant ListColumnAdapter classes since the implementation is not in header -template -class ListColumnAdapter; +// We need to force the compiler to instantiate the relevant ListColumnAdapter classes since the implementation is not +// in header +template class ListColumnAdapter; -template -class ListColumnAdapter; +template class ListColumnAdapter; -template -class ListColumnAdapter; +template class ListColumnAdapter; -template -class ListColumnAdapter; +template class ListColumnAdapter; -void StructColumnAdapter::addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol ) +void StructColumnAdapter::addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol ) { - CSP_TRUE_OR_THROW_RUNTIME( inputAdapter -> type() -> type() == CspType::TypeTraits::STRUCT, - "Trying to subscribe with non struct type " << inputAdapter -> type() -> type().asString() ); - auto structMeta = static_cast( inputAdapter -> type()) -> meta(); + CSP_TRUE_OR_THROW_RUNTIME( inputAdapter->type()->type() == CspType::TypeTraits::STRUCT, + "Trying to subscribe with non struct type " << inputAdapter->type()->type().asString() ); + auto structMeta = static_cast( inputAdapter->type() )->meta(); initFromStructMeta( structMeta ); BASE::addSubscriber( inputAdapter, symbol ); } void StructColumnAdapter::readCurValue() { - // TODO: In the future we should make optimization here and skip creating structs for symbols for which there's no subscription - auto curRow = this -> m_parquetReader.getCurRow(); - if( this -> m_curChunkArray -> IsValid( curRow ) ) + // TODO: In the future we should make optimization here and skip creating structs for symbols for which there's no + // subscription + auto curRow = this->m_parquetReader.getCurRow(); + if( this->m_curChunkArray->IsValid( curRow ) ) { - for( auto &childColumnAdapter: m_childColumnAdapters ) + for( auto & childColumnAdapter : m_childColumnAdapters ) { - childColumnAdapter -> readCurValue(); + childColumnAdapter->readCurValue(); } - m_structAdapter -> dispatchValue( nullptr ); + m_structAdapter->dispatchValue( nullptr ); } else { - m_structAdapter -> dispatchValue( nullptr, true ); - this -> m_curValue.reset(); + m_structAdapter->dispatchValue( nullptr, true ); + this->m_curValue.reset(); } } -void StructColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> &data ) +void StructColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> & data ) { BASE::handleNewBatch( data ); - auto &childArrays = m_curChunkArray -> fields(); + auto & childArrays = m_curChunkArray->fields(); CSP_TRUE_OR_THROW_RUNTIME( childArrays.size() == m_childColumnAdapters.size(), - "Expected " << m_childColumnAdapters.size() << " child arrays, got " << childArrays.size() ); + "Expected " << m_childColumnAdapters.size() << " child arrays, got " + << childArrays.size() ); for( std::size_t i = 0; i < childArrays.size(); ++i ) { - m_childColumnAdapters[ i ] -> handleNewBatch( childArrays[ i ] ); + m_childColumnAdapters[i]->handleNewBatch( childArrays[i] ); } } -void StructColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::Array> &data ) +void StructColumnAdapter::handleNewBatch( const std::shared_ptr<::arrow::Array> & data ) { BASE::handleNewBatch( data ); - auto &childArrays = m_curChunkArray -> fields(); + auto & childArrays = m_curChunkArray->fields(); CSP_TRUE_OR_THROW_RUNTIME( childArrays.size() == m_childColumnAdapters.size(), - "Expected " << m_childColumnAdapters.size() << " child arrays, got " << childArrays.size() ); + "Expected " << m_childColumnAdapters.size() << " child arrays, got " + << childArrays.size() ); for( std::size_t i = 0; i < childArrays.size(); ++i ) { - m_childColumnAdapters[ i ] -> handleNewBatch( childArrays[ i ] ); + m_childColumnAdapters[i]->handleNewBatch( childArrays[i] ); } } -void StructColumnAdapter::initFromStructMeta( const std::shared_ptr &structMeta ) +void StructColumnAdapter::initFromStructMeta( const std::shared_ptr & structMeta ) { if( m_structAdapter ) { - CSP_TRUE_OR_THROW_RUNTIME( m_structAdapter -> getStructMeta() == structMeta, + CSP_TRUE_OR_THROW_RUNTIME( m_structAdapter->getStructMeta() == structMeta, "Trying to subscribe to structure field with struct " - << structMeta -> name() << " and " << m_structAdapter -> getStructMeta() -> name() ); + << structMeta->name() << " and " << m_structAdapter->getStructMeta()->name() ); return; } - m_childColumnAdapters.reserve( m_arrowType -> num_fields() ); - for( auto &field:m_arrowType -> fields() ) + m_childColumnAdapters.reserve( m_arrowType->num_fields() ); + for( auto & field : m_arrowType->fields() ) { m_childColumnAdapters.push_back( createColumnAdapter( m_parquetReader, *field, "" ) ); } - m_structAdapter = std::make_unique( m_parquetReader, - m_arrowType, - structMeta, - m_childColumnAdapters ); - m_structAdapter -> addSubscriber( [ this ]( StructPtr *s ) - { - if( s ) - { - this -> m_curValue = *s; - } - } ); + m_structAdapter + = std::make_unique( m_parquetReader, m_arrowType, structMeta, m_childColumnAdapters ); + m_structAdapter->addSubscriber( + [this]( StructPtr * s ) + { + if( s ) + { + this->m_curValue = *s; + } + } ); } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.h b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.h index 5d8ad8580..337674593 100644 --- a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.h +++ b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.h @@ -1,6 +1,7 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetReaderColumnAdapter_H #define _IN_CSP_ADAPTERS_PARQUET_ParquetReaderColumnAdapter_H +#include #include #include #include @@ -8,17 +9,16 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include -#include namespace arrow::io { @@ -35,47 +35,57 @@ class ParquetStructAdapter; class ParquetColumnAdapter { public: - ParquetColumnAdapter( ParquetReader &parquetReader, const std::string& columnName ) : m_parquetReader( parquetReader ), m_columnName(columnName){} + ParquetColumnAdapter( ParquetReader & parquetReader, const std::string & columnName ) + : m_parquetReader( parquetReader ) + , m_columnName( columnName ) + { + } - virtual ~ParquetColumnAdapter(){} + virtual ~ParquetColumnAdapter() {} - const std::string& getColumnName() const {return m_columnName;} + const std::string & getColumnName() const { return m_columnName; } - virtual bool isListType() const {return false;}; - virtual CspTypePtr getContainerValueType() const {CSP_THROW(TypeError, "Trying to get list value on non container type");} + virtual bool isListType() const { return false; }; + virtual CspTypePtr getContainerValueType() const + { + CSP_THROW( TypeError, "Trying to get list value on non container type" ); + } - virtual void addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol = {} ) = 0; + virtual void addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol = {} ) = 0; - virtual void dispatchValue( const utils::Symbol *symbol ) = 0; + virtual void dispatchValue( const utils::Symbol * symbol ) = 0; virtual void ensureType( CspType::Ptr cspType ) = 0; virtual void readCurValue() = 0; - ParquetReader &getReader(){ return m_parquetReader; } + ParquetReader & getReader() { return m_parquetReader; } - const ParquetReader &getReader() const{ return m_parquetReader; } + const ParquetReader & getReader() const { return m_parquetReader; } - virtual bool isNativeType() const {return getNativeCspType() && getNativeCspType()->type() < CspType::TypeTraits::MAX_NATIVE_TYPE;} - virtual bool isMissingColumn() const { return false;} - virtual CspTypePtr getNativeCspType() const =0; + virtual bool isNativeType() const + { + return getNativeCspType() && getNativeCspType()->type() < CspType::TypeTraits::MAX_NATIVE_TYPE; + } + virtual bool isMissingColumn() const { return false; } + virtual CspTypePtr getNativeCspType() const = 0; - template< typename T > - std::optional &getCurValue() + template + std::optional & getCurValue() { - return *static_cast *>(getCurValueUntyped()); + return *static_cast *>( getCurValueUntyped() ); } - virtual void handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> &data ) = 0; - virtual void handleNewBatch( const std::shared_ptr<::arrow::Array> &data ) = 0; + virtual void handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> & data ) = 0; + virtual void handleNewBatch( const std::shared_ptr<::arrow::Array> & data ) = 0; protected: friend class ParquetReader; - virtual void *getCurValueUntyped() = 0; + virtual void * getCurValueUntyped() = 0; protected: - ParquetReader &m_parquetReader; + ParquetReader & m_parquetReader; const std::string m_columnName; }; @@ -83,30 +93,29 @@ class ParquetStructAdapter final { public: using ValueDispatcher = csp::adapters::utils::ValueDispatcher; - using FieldSetter = std::function; + using FieldSetter = std::function; - ParquetStructAdapter( ParquetReader &parquetReader, - csp::adapters::utils::StructAdapterInfo adapterInfo ); - ParquetStructAdapter( ParquetReader &parquetReader, - std::shared_ptr<::arrow::StructType> arrowType, - const std::shared_ptr &structMeta, - const std::vector> &columnAdapters ); + ParquetStructAdapter( ParquetReader & parquetReader, csp::adapters::utils::StructAdapterInfo adapterInfo ); + ParquetStructAdapter( ParquetReader & parquetReader, std::shared_ptr<::arrow::StructType> arrowType, + const std::shared_ptr & structMeta, + const std::vector> & columnAdapters ); - void addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol = {} ); + void addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol = {} ); void addSubscriber( ValueDispatcher::SubscriberType subscriber, std::optional symbol = {} ); - void dispatchValue( const utils::Symbol *symbol, bool isNull = false ); + void dispatchValue( const utils::Symbol * symbol, bool isNull = false ); - const std::shared_ptr &getStructMeta(){ return m_structMeta; } + const std::shared_ptr & getStructMeta() { return m_structMeta; } + + void onSchemaChange() { m_needsReset = true; } - void onSchemaChange(){ m_needsReset = true; } private: - void createFieldSetter( const std::string &fieldName, ParquetColumnAdapter &columnAdapter ); + void createFieldSetter( const std::string & fieldName, ParquetColumnAdapter & columnAdapter ); private: using StructAdapterInfo = csp::adapters::utils::StructAdapterInfo; - ParquetReader &m_parquetReader; + ParquetReader & m_parquetReader; std::shared_ptr m_structMeta; ValueDispatcher m_valueDispatcher; std::vector m_fieldSetters; @@ -119,9 +128,10 @@ class MissingColumnAdapter : public ParquetColumnAdapter public: using ParquetColumnAdapter::ParquetColumnAdapter; - virtual void addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol = {} ) override {}; + virtual void addSubscriber( ManagedSimInputAdapter * inputAdapter, + std::optional symbol = {} ) override {}; - virtual void dispatchValue( const utils::Symbol *symbol ) override {}; + virtual void dispatchValue( const utils::Symbol * symbol ) override {}; virtual void ensureType( CspType::Ptr cspType ) override {}; @@ -132,107 +142,112 @@ class MissingColumnAdapter : public ParquetColumnAdapter CSP_THROW( csp::RuntimeException, "Trying to check type of a missing column " << getColumnName() ); } - bool isMissingColumn() const override{ return true; } + bool isMissingColumn() const override { return true; } virtual CspTypePtr getNativeCspType() const override { CSP_THROW( csp::RuntimeException, "Trying to get native type of a missing column " << getColumnName() ); } - virtual void handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> &data ) override + virtual void handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> & data ) override { CSP_THROW( csp::RuntimeException, "Trying to handle new batch for a missing column " << getColumnName() ); } - virtual void handleNewBatch( const std::shared_ptr<::arrow::Array> &data ) override + virtual void handleNewBatch( const std::shared_ptr<::arrow::Array> & data ) override { CSP_THROW( csp::RuntimeException, "Trying to handle new batch for a missing column " << getColumnName() ); } protected: - void *getCurValueUntyped() override + void * getCurValueUntyped() override { CSP_THROW( csp::RuntimeException, "Trying to get value of a missing column " << getColumnName() ); } }; -template< typename ValueType, typename ArrowArrayType, typename ValueDispatcherT = csp::adapters::utils::ValueDispatcher> +template> class BaseTypedColumnAdapter : public ParquetColumnAdapter { public: - BaseTypedColumnAdapter( ParquetReader &parquetReader, const std::string& columnName ) - : ParquetColumnAdapter( parquetReader, columnName ) + BaseTypedColumnAdapter( ParquetReader & parquetReader, const std::string & columnName ) + : ParquetColumnAdapter( parquetReader, columnName ) { } - void addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol = {} ) override; + void addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol = {} ) override; - void dispatchValue( const utils::Symbol *symbol ) override; + void dispatchValue( const utils::Symbol * symbol ) override; void ensureType( CspType::Ptr cspType ) override; protected: - void *getCurValueUntyped() override; + void * getCurValueUntyped() override; - void handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> &data ) override; - void handleNewBatch( const std::shared_ptr<::arrow::Array> &data ) override; + void handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> & data ) override; + void handleNewBatch( const std::shared_ptr<::arrow::Array> & data ) override; protected: using CompatibleTypeSwitch = ConstructibleTypeSwitch; - using ValueDispatcher = ValueDispatcherT; + using ValueDispatcher = ValueDispatcherT; ValueDispatcher m_dispatcher; std::shared_ptr m_curChunkArray; std::optional m_curValue; }; -template< typename ValueType, typename ArrowArrayType > +template class NativeTypeColumnAdapter : public BaseTypedColumnAdapter { public: using BaseTypedColumnAdapter::BaseTypedColumnAdapter; - virtual CspTypePtr getNativeCspType() const override {return CspType::fromCType::type();} + virtual CspTypePtr getNativeCspType() const override { return CspType::fromCType::type(); } protected: void readCurValue() override; }; -template< int64_t UNIT > +template class DatetimeColumnAdapter : public BaseTypedColumnAdapter { public: using BaseTypedColumnAdapter::BaseTypedColumnAdapter; - virtual CspTypePtr getNativeCspType() const override {return CspType::fromCType::type();} + virtual CspTypePtr getNativeCspType() const override { return CspType::fromCType::type(); } + protected: void readCurValue() override; }; -template< int64_t UNIT > +template class DurationColumnAdapter : public BaseTypedColumnAdapter { public: using BaseTypedColumnAdapter::BaseTypedColumnAdapter; - virtual CspTypePtr getNativeCspType() const override {return CspType::fromCType::type();} + virtual CspTypePtr getNativeCspType() const override { return CspType::fromCType::type(); } + protected: void readCurValue() override; }; -template< int64_t UNIT, typename ArrowDateArray > +template class DateColumnAdapter : public BaseTypedColumnAdapter { public: using BaseTypedColumnAdapter::BaseTypedColumnAdapter; - virtual CspTypePtr getNativeCspType() const override {return CspType::fromCType::type();} + virtual CspTypePtr getNativeCspType() const override { return CspType::fromCType::type(); } + protected: void readCurValue() override; }; -template< int64_t UNIT, typename ArrowTimeArray > +template class TimeColumnAdapter : public BaseTypedColumnAdapter { public: using BaseTypedColumnAdapter::BaseTypedColumnAdapter; - virtual CspTypePtr getNativeCspType() const override {return CspType::fromCType::type();} + virtual CspTypePtr getNativeCspType() const override { return CspType::fromCType::type(); } + protected: void readCurValue() override; }; @@ -241,8 +256,9 @@ class StringColumnAdapter : public BaseTypedColumnAdapter symbol = {} ) override; - virtual CspTypePtr getNativeCspType() const override {return nullptr;} + void addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol = {} ) override; + virtual CspTypePtr getNativeCspType() const override { return nullptr; } + protected: void readCurValue() override; }; @@ -251,7 +267,8 @@ class BytesColumnAdapter : public BaseTypedColumnAdapter +template class ListColumnAdapter : public BaseTypedColumnAdapter { public: using Base = BaseTypedColumnAdapter; using BaseTypedColumnAdapter::BaseTypedColumnAdapter; - void addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol = {} ) override; - void addSubscriber( ManagedSimInputAdapter *inputAdapter, - std::optional symbol, const DialectGenericListReaderInterface::Ptr &listReader); - CspTypePtr getNativeCspType() const override {return nullptr;} + void addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol = {} ) override; + void addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol, + const DialectGenericListReaderInterface::Ptr & listReader ); + CspTypePtr getNativeCspType() const override { return nullptr; } + + bool isListType() const override { return true; }; + CspTypePtr getContainerValueType() const override { return CspType::fromCType::type(); } - bool isListType() const override{ return true; }; - CspTypePtr getContainerValueType() const override{ return CspType::fromCType::type(); } protected: void readCurValue() override; + private: - // For now we allow only one subscription for list columns. In the future we might allow multiple subscriptions using different readers. - // For example we could subscritbe to the same column as a list and as array, we would need to do more book keepting in this case. + // For now we allow only one subscription for list columns. In the future we might allow multiple subscriptions + // using different readers. For example we could subscritbe to the same column as a list and as array, we would need + // to do more book keepting in this case. typename TypedDialectGenericListReaderInterface::Ptr m_listReader = nullptr; }; -class StructColumnAdapter : public BaseTypedColumnAdapter> +class StructColumnAdapter + : public BaseTypedColumnAdapter> { public: - using BASE = BaseTypedColumnAdapter>; + using BASE + = BaseTypedColumnAdapter>; - StructColumnAdapter( ParquetReader &parquetReader, const std::shared_ptr<::arrow::StructType> &arrowType, - const std::string& columnName) - : BASE( parquetReader, columnName ), m_arrowType( arrowType ) + StructColumnAdapter( ParquetReader & parquetReader, const std::shared_ptr<::arrow::StructType> & arrowType, + const std::string & columnName ) + : BASE( parquetReader, columnName ) + , m_arrowType( arrowType ) { } - virtual CspTypePtr getNativeCspType() const override {return nullptr;} + virtual CspTypePtr getNativeCspType() const override { return nullptr; } + + void setStructMeta( const std::shared_ptr & structMeta ) { initFromStructMeta( structMeta ); } + void addSubscriber( ManagedSimInputAdapter * inputAdapter, std::optional symbol = {} ) override; - void setStructMeta( const std::shared_ptr &structMeta ){ initFromStructMeta( structMeta ); } - void addSubscriber( ManagedSimInputAdapter *inputAdapter, std::optional symbol = {} ) override; protected: void readCurValue() override; - void handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> &data ) override; - void handleNewBatch( const std::shared_ptr<::arrow::Array> &data ) override; + void handleNewBatch( const std::shared_ptr<::arrow::ChunkedArray> & data ) override; + void handleNewBatch( const std::shared_ptr<::arrow::Array> & data ) override; + private: - void initFromStructMeta( const std::shared_ptr &structMeta ); + void initFromStructMeta( const std::shared_ptr & structMeta ); private: friend class ParquetStructAdapter; @@ -325,15 +352,13 @@ class StructColumnAdapter : public BaseTypedColumnAdapter> m_childColumnAdapters; }; -std::unique_ptr createColumnAdapter( - ParquetReader &parquetReader, - const ::arrow::Field& field, - const std::string& fileName, - const std::map>* structMetaByColumnName = nullptr); - -std::unique_ptr createMissingColumnAdapter(ParquetReader &parquetReader,const std::string& columnName); +std::unique_ptr +createColumnAdapter( ParquetReader & parquetReader, const ::arrow::Field & field, const std::string & fileName, + const std::map> * structMetaByColumnName = nullptr ); -} +std::unique_ptr createMissingColumnAdapter( ParquetReader & parquetReader, + const std::string & columnName ); +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/parquet/ParquetStatusUtils.h b/cpp/csp/adapters/parquet/ParquetStatusUtils.h index b8353c36a..5f9b2317c 100644 --- a/cpp/csp/adapters/parquet/ParquetStatusUtils.h +++ b/cpp/csp/adapters/parquet/ParquetStatusUtils.h @@ -3,11 +3,11 @@ #include -#define STATUS_OK_OR_THROW_RUNTIME( EXPR, MESSAGE ) \ - do \ - { \ - arrow::Status st = EXPR; \ - CSP_TRUE_OR_THROW_RUNTIME( st.ok(), MESSAGE << ':' << st.ToString()); \ +#define STATUS_OK_OR_THROW_RUNTIME( EXPR, MESSAGE ) \ + do \ + { \ + arrow::Status st = EXPR; \ + CSP_TRUE_OR_THROW_RUNTIME( st.ok(), MESSAGE << ':' << st.ToString() ); \ } while( 0 ) #endif diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp index e40d1889f..50143ac77 100644 --- a/cpp/csp/adapters/parquet/ParquetWriter.cpp +++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp @@ -1,57 +1,67 @@ -#include -#include -#include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include +#include #include namespace csp::adapters::parquet { -ParquetWriter::ParquetWriter( ParquetOutputAdapterManager *mgr, std::optional writeTimestampColumn ) - : m_adapterMgr( *mgr ), m_engine( mgr -> engine() ), m_curChunkSize( 0 ), m_writeTimestampColumn( writeTimestampColumn ) -{} +ParquetWriter::ParquetWriter( ParquetOutputAdapterManager * mgr, std::optional writeTimestampColumn ) + : m_adapterMgr( *mgr ) + , m_engine( mgr->engine() ) + , m_curChunkSize( 0 ) + , m_writeTimestampColumn( writeTimestampColumn ) +{ +} -ParquetWriter::ParquetWriter( ParquetOutputAdapterManager *mgr, const Dictionary & properties ) : ParquetWriter( mgr, std::optional{} ) +ParquetWriter::ParquetWriter( ParquetOutputAdapterManager * mgr, const Dictionary & properties ) + : ParquetWriter( mgr, std::optional{} ) { if( properties.exists( "file_metadata" ) ) { auto file_meta = properties.get( "file_metadata" ); m_fileMetaData = std::make_shared(); - for( auto it = file_meta -> begin(); it != file_meta -> end(); ++it ) + for( auto it = file_meta->begin(); it != file_meta->end(); ++it ) { const std::string * value = std::get_if( &it.getUntypedValue() ); if( !value ) CSP_THROW( TypeError, "parquet metadata can only have string values" ); - m_fileMetaData -> Append( it.key(), *value ); + m_fileMetaData->Append( it.key(), *value ); } } if( properties.exists( "column_metadata" ) ) { auto column_metadata = properties.get( "column_metadata" ); - for( auto colIt = column_metadata -> begin(); colIt != column_metadata -> end(); ++colIt ) + for( auto colIt = column_metadata->begin(); colIt != column_metadata->end(); ++colIt ) { const DictionaryPtr * cmeta = std::get_if( &colIt.getUntypedValue() ); if( !cmeta ) - CSP_THROW( TypeError, "parquet column metadata expects dictionary entry per column, got unrecognized type for column '" << colIt.key() << "'" ); + CSP_THROW( + TypeError, + "parquet column metadata expects dictionary entry per column, got unrecognized type for column '" + << colIt.key() << "'" ); auto kv_metadata = std::make_shared(); - for( auto it = (*cmeta) -> begin(); it != (*cmeta) -> end(); ++it ) + for( auto it = ( *cmeta )->begin(); it != ( *cmeta )->end(); ++it ) { const std::string * value = std::get_if( &it.getUntypedValue() ); if( !value ) - CSP_THROW( TypeError, "parquet column metadata can only have string values, got non-string value for metadata on column '" << colIt.key() << "'" ); - kv_metadata -> Append( it.key(), *value ); + CSP_THROW( TypeError, + "parquet column metadata can only have string values, got non-string value for metadata " + "on column '" + << colIt.key() << "'" ); + kv_metadata->Append( it.key(), *value ); } - m_columnMetaData[ colIt.key() ] = kv_metadata; + m_columnMetaData[colIt.key()] = kv_metadata; } } } @@ -64,33 +74,35 @@ ParquetWriter::~ParquetWriter() } } -SingleColumnParquetOutputAdapter *ParquetWriter::getScalarOutputAdapter( CspTypePtr &type, const std::string &columnName ) +SingleColumnParquetOutputAdapter * ParquetWriter::getScalarOutputAdapter( CspTypePtr & type, + const std::string & columnName ) { - auto res = static_cast(getScalarOutputHandler( type, columnName )); + auto res = static_cast( getScalarOutputHandler( type, columnName ) ); return res; } -StructParquetOutputAdapter *ParquetWriter::getStructOutputAdapter( CspTypePtr &type, csp::DictionaryPtr fieldMap ) +StructParquetOutputAdapter * ParquetWriter::getStructOutputAdapter( CspTypePtr & type, csp::DictionaryPtr fieldMap ) { - auto res = static_cast(getStructOutputHandler( type, fieldMap )); + auto res = static_cast( getStructOutputHandler( type, fieldMap ) ); return res; } -SingleColumnParquetOutputHandler *ParquetWriter::getScalarOutputHandler( CspTypePtr &type, const std::string &columnName ) +SingleColumnParquetOutputHandler * ParquetWriter::getScalarOutputHandler( CspTypePtr & type, + const std::string & columnName ) { CSP_TRUE_OR_THROW_RUNTIME( m_publishedColumnNames.emplace( columnName ).second, "Trying to publish column " << columnName << " more than once" ); - //create and register adapter + // create and register adapter auto adapter = createScalarOutputHandler( type, columnName ); m_adapters.emplace_back( adapter ); return adapter; } -StructParquetOutputHandler *ParquetWriter::getStructOutputHandler( CspTypePtr &type, csp::DictionaryPtr fieldMap ) +StructParquetOutputHandler * ParquetWriter::getStructOutputHandler( CspTypePtr & type, csp::DictionaryPtr fieldMap ) { - for( auto it = fieldMap -> begin(); it != fieldMap -> end(); ++it ) + for( auto it = fieldMap->begin(); it != fieldMap->end(); ++it ) { - auto &&columnName = it.value(); + auto && columnName = it.value(); CSP_TRUE_OR_THROW_RUNTIME( m_publishedColumnNames.emplace( columnName ).second, "Trying to publish column " << columnName << " more than once" ); } @@ -101,26 +113,28 @@ StructParquetOutputHandler *ParquetWriter::getStructOutputHandler( CspTypePtr &t return adapter; } -ListColumnParquetOutputHandler *ParquetWriter::getListOutputHandler( CspTypePtr &elemType, const std::string &columnName, - const DialectGenericListWriterInterface::Ptr &listWriterInterface ) +ListColumnParquetOutputHandler * +ParquetWriter::getListOutputHandler( CspTypePtr & elemType, const std::string & columnName, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ) { CSP_TRUE_OR_THROW_RUNTIME( m_publishedColumnNames.emplace( columnName ).second, "Trying to publish column " << columnName << " more than once" ); - //create and register adapter - auto adapter = createListOutputHandler( elemType, columnName, listWriterInterface); + // create and register adapter + auto adapter = createListOutputHandler( elemType, columnName, listWriterInterface ); m_adapters.emplace_back( adapter ); return adapter; } -ListColumnParquetOutputAdapter *ParquetWriter::getListOutputAdapter( - CspTypePtr &elemType, const std::string &columnName, - const DialectGenericListWriterInterface::Ptr &listWriterInterface ) +ListColumnParquetOutputAdapter * +ParquetWriter::getListOutputAdapter( CspTypePtr & elemType, const std::string & columnName, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ) { - auto res = static_cast(getListOutputHandler( elemType, columnName, listWriterInterface )); + auto res = static_cast( + getListOutputHandler( elemType, columnName, listWriterInterface ) ); return res; } -PushInputAdapter *ParquetWriter::getStatusAdapter() +PushInputAdapter * ParquetWriter::getStatusAdapter() { return nullptr; } @@ -131,43 +145,44 @@ void ParquetWriter::start() if( !m_writeTimestampColumn.has_value() && !m_adapterMgr.getTimestampColumnName().empty() ) { m_writeTimestampColumn = true; - m_columnBuilders.push_back( std::make_shared( m_adapterMgr.getTimestampColumnName(), getChunkSize() ) ); + m_columnBuilders.push_back( + std::make_shared( m_adapterMgr.getTimestampColumnName(), getChunkSize() ) ); std::shared_ptr colMetaData; auto colMetaIt = m_columnMetaData.find( m_adapterMgr.getTimestampColumnName() ); if( colMetaIt != m_columnMetaData.end() ) { - colMetaData = colMetaIt -> second; + colMetaData = colMetaIt->second; m_columnMetaData.erase( colMetaIt ); } - arrowFields.push_back( - arrow::field( m_adapterMgr.getTimestampColumnName(), m_columnBuilders.back() -> getDataType(), colMetaData ) ); + arrowFields.push_back( arrow::field( m_adapterMgr.getTimestampColumnName(), + m_columnBuilders.back()->getDataType(), colMetaData ) ); } else { m_writeTimestampColumn = false; } - for( auto &&adapter:m_adapters ) + for( auto && adapter : m_adapters ) { - for( unsigned i = 0; i < adapter -> getNumColumns(); ++i ) + for( unsigned i = 0; i < adapter->getNumColumns(); ++i ) { - m_columnBuilders.push_back( adapter -> getColumnArrayBuilder( i ) ); + m_columnBuilders.push_back( adapter->getColumnArrayBuilder( i ) ); std::shared_ptr colMetaData; - auto colMetaIt = m_columnMetaData.find( m_columnBuilders.back() -> getColumnName() ); + auto colMetaIt = m_columnMetaData.find( m_columnBuilders.back()->getColumnName() ); if( colMetaIt != m_columnMetaData.end() ) { - colMetaData = colMetaIt -> second; + colMetaData = colMetaIt->second; m_columnMetaData.erase( colMetaIt ); } - arrowFields.push_back( arrow::field( m_columnBuilders.back() -> getColumnName(), - m_columnBuilders.back() -> getDataType(), - colMetaData ) ); + arrowFields.push_back( arrow::field( m_columnBuilders.back()->getColumnName(), + m_columnBuilders.back()->getDataType(), colMetaData ) ); } } if( !m_columnMetaData.empty() ) - CSP_THROW( ValueError, "parquet column metadata has unmapped column: '" << m_columnMetaData.begin() -> first << "'" ); + CSP_THROW( ValueError, + "parquet column metadata has unmapped column: '" << m_columnMetaData.begin()->first << "'" ); initFileWriterContainer( arrow::schema( arrowFields, m_fileMetaData ) ); } @@ -179,7 +194,7 @@ void ParquetWriter::stop() { writeCurChunkToFile(); } - m_fileWriterWrapperContainer -> close(); + m_fileWriterWrapperContainer->close(); m_fileWriterWrapperContainer = nullptr; } } @@ -194,12 +209,12 @@ void ParquetWriter::onEndCycle() if( m_writeTimestampColumn.value() ) { // Set the timestamp value it's always the first - now = m_adapterMgr.rootEngine() -> now(); - static_cast(m_columnBuilders[ 0 ].get()) -> setValue( now ); + now = m_adapterMgr.rootEngine()->now(); + static_cast( m_columnBuilders[0].get() )->setValue( now ); } - for( auto &&columnBuilder:m_columnBuilders ) + for( auto && columnBuilder : m_columnBuilders ) { - columnBuilder -> handleRowFinished(); + columnBuilder->handleRowFinished(); } if( ++m_curChunkSize >= getChunkSize() ) { @@ -210,55 +225,55 @@ void ParquetWriter::onEndCycle() bool ParquetWriter::isFileOpen() const { - return m_fileWriterWrapperContainer != nullptr && m_fileWriterWrapperContainer -> isOpen(); + return m_fileWriterWrapperContainer != nullptr && m_fileWriterWrapperContainer->isOpen(); } -void ParquetWriter::onFileNameChange( const std::string &fileName ) +void ParquetWriter::onFileNameChange( const std::string & fileName ) { - CSP_TRUE_OR_THROW_RUNTIME( m_fileWriterWrapperContainer, "Trying to set file name when file writer already closed" ); + CSP_TRUE_OR_THROW_RUNTIME( m_fileWriterWrapperContainer, + "Trying to set file name when file writer already closed" ); writeCurChunkToFile(); - m_fileWriterWrapperContainer -> close(); + m_fileWriterWrapperContainer->close(); if( !fileName.empty() ) { - m_fileWriterWrapperContainer - -> open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); + m_fileWriterWrapperContainer->open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); } } -SingleColumnParquetOutputHandler *ParquetWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name ) +SingleColumnParquetOutputHandler * ParquetWriter::createScalarOutputHandler( CspTypePtr type, const std::string & name ) { - return m_engine -> createOwnedObject( *this, type, name ); + return m_engine->createOwnedObject( *this, type, name ); } -ListColumnParquetOutputHandler *ParquetWriter::createListOutputHandler( CspTypePtr &elemType, const std::string &columnName, - DialectGenericListWriterInterface::Ptr listWriterInterface ) +ListColumnParquetOutputHandler * +ParquetWriter::createListOutputHandler( CspTypePtr & elemType, const std::string & columnName, + DialectGenericListWriterInterface::Ptr listWriterInterface ) { - return m_engine -> createOwnedObject( *this, elemType, columnName, listWriterInterface ); + return m_engine->createOwnedObject( *this, elemType, columnName, + listWriterInterface ); } - -StructParquetOutputHandler *ParquetWriter::createStructOutputHandler( CspTypePtr type, - const DictionaryPtr &fieldMap ) +StructParquetOutputHandler * ParquetWriter::createStructOutputHandler( CspTypePtr type, const DictionaryPtr & fieldMap ) { - return m_engine -> createOwnedObject( *this, type, fieldMap ); + return m_engine->createOwnedObject( *this, type, fieldMap ); } void ParquetWriter::initFileWriterContainer( std::shared_ptr schema ) { if( m_adapterMgr.isSplitColumnsToFiles() ) { - m_fileWriterWrapperContainer = std::make_unique( schema, - m_adapterMgr.isWriteArrowBinary() ); + m_fileWriterWrapperContainer + = std::make_unique( schema, m_adapterMgr.isWriteArrowBinary() ); } else { - m_fileWriterWrapperContainer = std::make_unique( schema, - m_adapterMgr.isWriteArrowBinary() ); + m_fileWriterWrapperContainer + = std::make_unique( schema, m_adapterMgr.isWriteArrowBinary() ); } if( !m_adapterMgr.getFileName().empty() ) { - m_fileWriterWrapperContainer -> open( m_adapterMgr.getFileName(), - m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() ); + m_fileWriterWrapperContainer->open( m_adapterMgr.getFileName(), m_adapterMgr.getCompression(), + m_adapterMgr.isAllowOverwrite() ); } } @@ -268,16 +283,16 @@ void ParquetWriter::writeCurChunkToFile() { if( unlikely( !isFileOpen() ) ) { - if(m_curChunkSize != 0 ) + if( m_curChunkSize != 0 ) { - CSP_THROW( csp::RuntimeException, "Trying to write to parquet/arrow file, when no file name was provided" ); - + CSP_THROW( csp::RuntimeException, + "Trying to write to parquet/arrow file, when no file name was provided" ); } return; } - m_fileWriterWrapperContainer -> writeData( m_columnBuilders ); + m_fileWriterWrapperContainer->writeData( m_columnBuilders ); m_curChunkSize = 0; } } -} +} // namespace csp::adapters::parquet diff --git a/cpp/csp/adapters/parquet/ParquetWriter.h b/cpp/csp/adapters/parquet/ParquetWriter.h index ebf89a1e6..ca26878dd 100644 --- a/cpp/csp/adapters/parquet/ParquetWriter.h +++ b/cpp/csp/adapters/parquet/ParquetWriter.h @@ -5,11 +5,14 @@ #include #include #include -#include #include +#include #include -namespace arrow{ class KeyValueMetadata; } +namespace arrow +{ +class KeyValueMetadata; +} namespace csp::adapters::parquet { @@ -27,72 +30,73 @@ class FileWriterWrapper; class FileWriterWrapperContainer; - class ParquetWriter : public EndCycleListener { public: - ParquetWriter( ParquetOutputAdapterManager *mgr, std::optional writeTimestampColumn = {} ); - ParquetWriter( ParquetOutputAdapterManager *mgr, const Dictionary & properties ); + ParquetWriter( ParquetOutputAdapterManager * mgr, std::optional writeTimestampColumn = {} ); + ParquetWriter( ParquetOutputAdapterManager * mgr, const Dictionary & properties ); ~ParquetWriter(); - SingleColumnParquetOutputAdapter *getScalarOutputAdapter( CspTypePtr &type, const std::string &columnName ); - StructParquetOutputAdapter *getStructOutputAdapter( CspTypePtr &type, csp::DictionaryPtr fieldMap ); - ListColumnParquetOutputAdapter *getListOutputAdapter( CspTypePtr &elemType, const std::string &columnName, - const DialectGenericListWriterInterface::Ptr &listWriterInterface ); + SingleColumnParquetOutputAdapter * getScalarOutputAdapter( CspTypePtr & type, const std::string & columnName ); + StructParquetOutputAdapter * getStructOutputAdapter( CspTypePtr & type, csp::DictionaryPtr fieldMap ); + ListColumnParquetOutputAdapter * + getListOutputAdapter( CspTypePtr & elemType, const std::string & columnName, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ); - SingleColumnParquetOutputHandler *getScalarOutputHandler( CspTypePtr &type, const std::string &columnName ); - StructParquetOutputHandler *getStructOutputHandler( CspTypePtr &type, csp::DictionaryPtr fieldMap ); - ListColumnParquetOutputHandler *getListOutputHandler( CspTypePtr &elemType, const std::string &columnName, - const DialectGenericListWriterInterface::Ptr &listWriterInterface ); + SingleColumnParquetOutputHandler * getScalarOutputHandler( CspTypePtr & type, const std::string & columnName ); + StructParquetOutputHandler * getStructOutputHandler( CspTypePtr & type, csp::DictionaryPtr fieldMap ); + ListColumnParquetOutputHandler * + getListOutputHandler( CspTypePtr & elemType, const std::string & columnName, + const DialectGenericListWriterInterface::Ptr & listWriterInterface ); - PushInputAdapter *getStatusAdapter(); + PushInputAdapter * getStatusAdapter(); virtual void start(); virtual void stop(); void onEndCycle() override; - std::uint32_t getChunkSize() const{ return m_adapterMgr.getBatchSize(); } + std::uint32_t getChunkSize() const { return m_adapterMgr.getBatchSize(); } - virtual void scheduleEndCycleEvent() - { - m_adapterMgr.scheduleEndCycle(); - } + virtual void scheduleEndCycleEvent() { m_adapterMgr.scheduleEndCycle(); } bool isFileOpen() const; - virtual void onFileNameChange(const std::string& fileName); + virtual void onFileNameChange( const std::string & fileName ); + protected: - virtual SingleColumnParquetOutputHandler *createScalarOutputHandler( CspTypePtr type, const std::string& name ); - virtual ListColumnParquetOutputHandler *createListOutputHandler( CspTypePtr &elemType, const std::string &columnName, - DialectGenericListWriterInterface::Ptr listWriterInterface ); - virtual StructParquetOutputHandler *createStructOutputHandler( CspTypePtr type, const DictionaryPtr &fieldMap ); + virtual SingleColumnParquetOutputHandler * createScalarOutputHandler( CspTypePtr type, const std::string & name ); + virtual ListColumnParquetOutputHandler * + createListOutputHandler( CspTypePtr & elemType, const std::string & columnName, + DialectGenericListWriterInterface::Ptr listWriterInterface ); + virtual StructParquetOutputHandler * createStructOutputHandler( CspTypePtr type, const DictionaryPtr & fieldMap ); private: void initFileWriterContainer( std::shared_ptr schema ); void writeCurChunkToFile(); + protected: - using Adapters = std::vector; + using Adapters = std::vector; using PublishedColumnNames = std::unordered_set; - ParquetOutputAdapterManager &m_adapterMgr; - Engine *m_engine; + ParquetOutputAdapterManager & m_adapterMgr; + Engine * m_engine; + private: Adapters m_adapters; PublishedColumnNames m_publishedColumnNames; - std::unique_ptr m_fileWriterWrapperContainer; std::vector> m_columnBuilders; std::uint32_t m_curChunkSize; std::optional m_writeTimestampColumn; - std::shared_ptr m_fileMetaData; - std::unordered_map> m_columnMetaData; + std::shared_ptr m_fileMetaData; + std::unordered_map> m_columnMetaData; }; -} +} // namespace csp::adapters::parquet #endif diff --git a/cpp/csp/adapters/utils/FileUtils.h b/cpp/csp/adapters/utils/FileUtils.h index 59a6dd260..2b60524e3 100644 --- a/cpp/csp/adapters/utils/FileUtils.h +++ b/cpp/csp/adapters/utils/FileUtils.h @@ -13,22 +13,22 @@ inline std::string dirname( const std::string & path ) return ""; } -//files and directories are treated equally +// files and directories are treated equally inline bool fileExists( const std::string & fileOrDir ) { - return std::filesystem::exists(fileOrDir); + return std::filesystem::exists( fileOrDir ); } -inline void mkdir( const std::string & path, mode_t mode = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH ) +inline void mkdir( const std::string & path, mode_t mode = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH ) { size_t pos = 0; do { - pos = path.find( '/', pos + 1 ); + pos = path.find( '/', pos + 1 ); std::string subpath = path.substr( 0, pos ); - if( !fileExists( subpath) && ( ::mkdir( subpath.c_str(), mode ) == -1 && errno != EEXIST ) ) + if( !fileExists( subpath ) && ( ::mkdir( subpath.c_str(), mode ) == -1 && errno != EEXIST ) ) CSP_THROW( IOError, "Failed to create path " << subpath << ": " << strerror( errno ) ); } while( pos != std::string::npos ); } -} +} // namespace csp::utils diff --git a/cpp/csp/adapters/utils/JSONMessageStructConverter.cpp b/cpp/csp/adapters/utils/JSONMessageStructConverter.cpp index 8e1217c68..f36f49db3 100644 --- a/cpp/csp/adapters/utils/JSONMessageStructConverter.cpp +++ b/cpp/csp/adapters/utils/JSONMessageStructConverter.cpp @@ -7,26 +7,15 @@ namespace csp::adapters::utils { -using SupportedCspTypeSwitch = PartialSwitchCspType; - -using SupportedArrayCspTypeSwitch = PartialSwitchCspType; +using SupportedCspTypeSwitch + = PartialSwitchCspType; +using SupportedArrayCspTypeSwitch + = PartialSwitchCspType; template<> bool JSONMessageStructConverter::convertJSON( const char * fieldname, const rapidjson::Value & jValue, bool * ) @@ -65,7 +54,8 @@ double JSONMessageStructConverter::convertJSON( const char * fieldname, const ra } template<> -std::string JSONMessageStructConverter::convertJSON( const char * fieldname, const rapidjson::Value & jValue, std::string * ) +std::string JSONMessageStructConverter::convertJSON( const char * fieldname, const rapidjson::Value & jValue, + std::string * ) { if( jValue.IsString() ) return jValue.GetString(); @@ -82,10 +72,18 @@ DateTime JSONMessageStructConverter::convertJSON( const char * fieldname, const DateTime dt; switch( m_datetimeType ) { - case DateTimeWireType::UINT64_NANOS: dt = DateTime::fromNanoseconds( raw ); break; - case DateTimeWireType::UINT64_MICROS: dt = DateTime::fromMicroseconds( raw ); break; - case DateTimeWireType::UINT64_MILLIS: dt = DateTime::fromMilliseconds( raw ); break; - case DateTimeWireType::UINT64_SECONDS: dt = DateTime::fromSeconds( raw ); break; + case DateTimeWireType::UINT64_NANOS: + dt = DateTime::fromNanoseconds( raw ); + break; + case DateTimeWireType::UINT64_MICROS: + dt = DateTime::fromMicroseconds( raw ); + break; + case DateTimeWireType::UINT64_MILLIS: + dt = DateTime::fromMilliseconds( raw ); + break; + case DateTimeWireType::UINT64_SECONDS: + dt = DateTime::fromSeconds( raw ); + break; case DateTimeWireType::UNKNOWN: case DateTimeWireType::NUM_TYPES: @@ -99,39 +97,44 @@ DateTime JSONMessageStructConverter::convertJSON( const char * fieldname, const } template<> -CspEnum JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType & type, const FieldEntry &, const rapidjson::Value & jValue, CspEnum * ) +CspEnum JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType & type, const FieldEntry &, + const rapidjson::Value & jValue, CspEnum * ) { if( !jValue.IsString() ) CSP_THROW( TypeError, "expected ENUM type for json field " << fieldname ); auto & cspEnumType = static_cast( type ); - return cspEnumType.meta() -> fromString( jValue.GetString() ); + return cspEnumType.meta()->fromString( jValue.GetString() ); } template<> -StructPtr JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType &, const FieldEntry & entry, const rapidjson::Value & jValue, StructPtr * ) +StructPtr JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType &, const FieldEntry & entry, + const rapidjson::Value & jValue, StructPtr * ) { if( !jValue.IsObject() ) CSP_THROW( TypeError, "expected Nested object type for json field " << fieldname ); - const CspStructType & sType = static_cast( *entry.sField -> type() ); - auto struct_ = sType.meta() -> create(); - auto & fields = *entry.nestedFields; + const CspStructType & sType = static_cast( *entry.sField->type() ); + auto struct_ = sType.meta()->create(); + auto & fields = *entry.nestedFields; for( auto jit = jValue.MemberBegin(); jit != jValue.MemberEnd(); ++jit ) { - auto sIt = fields.find( jit -> name.GetString() ); + auto sIt = fields.find( jit->name.GetString() ); if( sIt == fields.end() ) continue; - auto & nestedEntry = sIt -> second; + auto & nestedEntry = sIt->second; SupportedCspTypeSwitch::invoke( - nestedEntry.sField -> type().get(), [this,&jit,&nestedEntry,&struct_]( auto tag ) + nestedEntry.sField->type().get(), + [this, &jit, &nestedEntry, &struct_]( auto tag ) { - using T = typename decltype(tag)::type; - auto & jValue = jit -> value; + using T = typename decltype( tag )::type; + auto & jValue = jit->value; - nestedEntry.sField -> setValue( struct_.get(), convertJSON( jit -> name.GetString(), *nestedEntry.sField -> type(), nestedEntry, jValue, static_cast( nullptr ) ) ); + nestedEntry.sField->setValue( struct_.get(), + convertJSON( jit->name.GetString(), *nestedEntry.sField->type(), + nestedEntry, jValue, static_cast( nullptr ) ) ); } ); } @@ -139,7 +142,9 @@ StructPtr JSONMessageStructConverter::convertJSON( const char * fieldname, const } template -std::vector JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType & type, const FieldEntry &, const rapidjson::Value & jValue, std::vector * x ) +std::vector JSONMessageStructConverter::convertJSON( const char * fieldname, const CspType & type, + const FieldEntry &, const rapidjson::Value & jValue, + std::vector * x ) { if( !jValue.IsArray() ) CSP_THROW( TypeError, "expected ARRAY type for json field " << fieldname ); @@ -153,33 +158,34 @@ std::vector JSONMessageStructConverter::convertJSON( const char * fiel out.reserve( jArray.Size() ); for( auto & v : jArray ) { - //note that we dont pass FieldEntry to convert here, this doesnt support arrays of structs - out.emplace_back( convertJSON( fieldname, elemType, {}, v, ( ElemT * ) nullptr) ); + // note that we dont pass FieldEntry to convert here, this doesnt support arrays of structs + out.emplace_back( convertJSON( fieldname, elemType, {}, v, (ElemT *)nullptr ) ); } return out; } -JSONMessageStructConverter::JSONMessageStructConverter( const CspTypePtr & type, - const Dictionary & properties ) : MessageStructConverter( type, properties ) +JSONMessageStructConverter::JSONMessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) + : MessageStructConverter( type, properties ) { - if( type -> type() != CspType::Type::STRUCT ) - CSP_THROW( TypeError, "JSONMessageStructConverter expects type struct got " << type -> type() ); + if( type->type() != CspType::Type::STRUCT ) + CSP_THROW( TypeError, "JSONMessageStructConverter expects type struct got " << type->type() ); const Dictionary & fieldMap = *properties.get( "field_map" ); - m_datetimeType = DateTimeWireType( properties.get( "datetime_type" ) ); - m_fields = buildFields( static_cast( *type ), fieldMap ); + m_datetimeType = DateTimeWireType( properties.get( "datetime_type" ) ); + m_fields = buildFields( static_cast( *type ), fieldMap ); } -JSONMessageStructConverter::Fields JSONMessageStructConverter::buildFields( const CspStructType & type, const Dictionary & fieldMap ) +JSONMessageStructConverter::Fields JSONMessageStructConverter::buildFields( const CspStructType & type, + const Dictionary & fieldMap ) { JSONMessageStructConverter::Fields out; for( auto it = fieldMap.begin(); it != fieldMap.end(); ++it ) { - auto & fieldName = it.key(); + auto & fieldName = it.key(); - std::string structField; + std::string structField; DictionaryPtr nestedFieldMap; if( it.hasValue() ) @@ -187,27 +193,35 @@ JSONMessageStructConverter::Fields JSONMessageStructConverter::buildFields( cons else { if( !it.hasValue() ) - CSP_THROW( TypeError, "fieldMap expected string or dict for field " << fieldName << " on struct " << type.meta() -> name() ); + CSP_THROW( TypeError, + "fieldMap expected string or dict for field " << fieldName << " on struct " + << type.meta()->name() ); auto nestedDict = it.value(); - if( nestedDict -> size() != 1 ) - CSP_THROW( ValueError, "Expected nested fieldmap for incoming JSON field " << fieldName << " to have a single key : map entry" ); - structField = nestedDict -> begin().key(); - nestedFieldMap = nestedDict -> begin().value(); + if( nestedDict->size() != 1 ) + CSP_THROW( ValueError, + "Expected nested fieldmap for incoming JSON field " << fieldName + << " to have a single key : map entry" ); + structField = nestedDict->begin().key(); + nestedFieldMap = nestedDict->begin().value(); } - auto sField = type.meta() -> field( structField ); + auto sField = type.meta()->field( structField ); if( !sField ) - CSP_THROW( ValueError, "field " << structField << " is not a valid field on struct type " << type.meta() -> name() ); + CSP_THROW( ValueError, + "field " << structField << " is not a valid field on struct type " << type.meta()->name() ); std::shared_ptr nestedFields; - if( sField -> type() -> type() == CspType::Type::STRUCT ) + if( sField->type()->type() == CspType::Type::STRUCT ) { if( !nestedFieldMap ) - CSP_THROW( ValueError, "invalid field_map entry for nested struct field " << sField -> fieldname() << " on struct type " << type.meta() -> name() ); - nestedFields = std::make_shared( buildFields( static_cast( *sField -> type() ), *nestedFieldMap ) ); + CSP_THROW( ValueError, + "invalid field_map entry for nested struct field " + << sField->fieldname() << " on struct type " << type.meta()->name() ); + nestedFields = std::make_shared( + buildFields( static_cast( *sField->type() ), *nestedFieldMap ) ); } - //keep strings around to keep const char * keys alive + // keep strings around to keep const char * keys alive m_jsonkeys.emplace_back( fieldName ); out.emplace( m_jsonkeys.back().c_str(), FieldEntry{ sField, nestedFields } ); } @@ -216,32 +230,36 @@ JSONMessageStructConverter::Fields JSONMessageStructConverter::buildFields( cons csp::StructPtr JSONMessageStructConverter::asStruct( void * bytes, size_t size ) { - const char * rawmsg = (const char *) bytes; + const char * rawmsg = (const char *)bytes; - rapidjson::Document document; + rapidjson::Document document; rapidjson::ParseResult ok = document.Parse( rawmsg, size ); if( !ok ) - CSP_THROW( ValueError, "Failed to parse message as JSON: " << rapidjson::GetParseError_En( ok.Code() ) << " on msg: " << std::string( rawmsg, size ) ); + CSP_THROW( ValueError, + "Failed to parse message as JSON: " << rapidjson::GetParseError_En( ok.Code() ) + << " on msg: " << std::string( rawmsg, size ) ); - StructPtr data = m_structMeta -> create(); + StructPtr data = m_structMeta->create(); for( auto jit = document.MemberBegin(); jit != document.MemberEnd(); ++jit ) { - auto sIt = m_fields.find( jit -> name.GetString() ); + auto sIt = m_fields.find( jit->name.GetString() ); if( sIt == m_fields.end() ) continue; - auto & entry = sIt -> second; + auto & entry = sIt->second; SupportedCspTypeSwitch::invoke( - entry.sField -> type().get(), [this,&jit,&entry,&data]( auto tag ) + entry.sField->type().get(), + [this, &jit, &entry, &data]( auto tag ) { - using T = typename decltype(tag)::type; - auto & jValue = jit -> value; - entry.sField -> setValue( data.get(), convertJSON( jit -> name.GetString(), *entry.sField -> type(), entry, jValue, static_cast( nullptr ) ) ); - } - ); + using T = typename decltype( tag )::type; + auto & jValue = jit->value; + entry.sField->setValue( data.get(), + convertJSON( jit->name.GetString(), *entry.sField->type(), entry, jValue, + static_cast( nullptr ) ) ); + } ); } return data; } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/JSONMessageStructConverter.h b/cpp/csp/adapters/utils/JSONMessageStructConverter.h index ce64c505c..7ad7afc95 100644 --- a/cpp/csp/adapters/utils/JSONMessageStructConverter.h +++ b/cpp/csp/adapters/utils/JSONMessageStructConverter.h @@ -4,15 +4,15 @@ #include #include #include -#include #include +#include #include #include namespace csp::adapters::utils { -class JSONMessageStructConverter: public MessageStructConverter +class JSONMessageStructConverter : public MessageStructConverter { public: JSONMessageStructConverter( const CspTypePtr & type, const Dictionary & properties ); @@ -25,38 +25,39 @@ class JSONMessageStructConverter: public MessageStructConverter } private: - - //map of json field -> struct field ptr - //we keep a hash since rapidjson Document field lookups are O(n)! + // map of json field -> struct field ptr + // we keep a hash since rapidjson Document field lookups are O(n)! struct FieldEntry { StructFieldPtr sField; - std::shared_ptr> nestedFields; + std::shared_ptr> + nestedFields; }; - using Fields = std::unordered_map; + using Fields = std::unordered_map; Fields buildFields( const CspStructType & type, const Dictionary & fieldMap ); - //T* only used for vector overloads + // T* only used for vector overloads template T convertJSON( const char * fieldname, const rapidjson::Value & v, T * ); template - T convertJSON( const char * fieldname, const CspType & type, const FieldEntry & entry, const rapidjson::Value & v, T * foo ) + T convertJSON( const char * fieldname, const CspType & type, const FieldEntry & entry, const rapidjson::Value & v, + T * foo ) { return convertJSON( fieldname, v, foo ); } template - std::vector convertJSON( const char * fieldname, const CspType & type, const FieldEntry & entry, const rapidjson::Value & v, std::vector * ); + std::vector convertJSON( const char * fieldname, const CspType & type, const FieldEntry & entry, + const rapidjson::Value & v, std::vector * ); - - Fields m_fields; - DateTimeWireType m_datetimeType; - std::list m_jsonkeys; //intentionally stored as list so they dont invalidate on push + Fields m_fields; + DateTimeWireType m_datetimeType; + std::list m_jsonkeys; // intentionally stored as list so they dont invalidate on push }; -} +} // namespace csp::adapters::utils #endif //_IN_CSP_ADAPTERS_ACTIVEMQ_JSONMESSAGESTRUCTCONVERTER_H diff --git a/cpp/csp/adapters/utils/JSONMessageWriter.h b/cpp/csp/adapters/utils/JSONMessageWriter.h index 688b70cc6..5e324a77c 100644 --- a/cpp/csp/adapters/utils/JSONMessageWriter.h +++ b/cpp/csp/adapters/utils/JSONMessageWriter.h @@ -14,29 +14,16 @@ namespace csp::adapters::utils class JSONMessageWriter : public MessageWriter { public: - using SupportedCspTypeSwitch = PartialSwitchCspType; - - using SupportedArrayCspTypeSwitch = PartialSwitchCspType; + using SupportedCspTypeSwitch + = PartialSwitchCspType; + + using SupportedArrayCspTypeSwitch + = PartialSwitchCspType; JSONMessageWriter( const Dictionary & properties ) { @@ -47,19 +34,19 @@ class JSONMessageWriter : public MessageWriter template void setField( const std::string & field, const T & value, const CspType & type, const FieldEntry & entry ); - std::pair finalize() override + std::pair finalize() override { - using Writer = rapidjson::Writer,rapidjson::UTF8<>, - rapidjson::CrtAllocator,rapidjson::kWriteNanAndInfFlag>; + using Writer = rapidjson::Writer, rapidjson::UTF8<>, + rapidjson::CrtAllocator, rapidjson::kWriteNanAndInfFlag>; m_stringBuffer.Clear(); Writer writer( m_stringBuffer ); m_doc.Accept( writer ); - //reset document - //Note we have to explicitly clear the memory pool to avoid leaking! + // reset document + // Note we have to explicitly clear the memory pool to avoid leaking! m_doc.GetAllocator().Clear(); m_doc.SetObject(); - return {m_stringBuffer.GetString(),m_stringBuffer.GetSize()}; + return { m_stringBuffer.GetString(), m_stringBuffer.GetSize() }; } private: @@ -70,7 +57,7 @@ class JSONMessageWriter : public MessageWriter template inline auto convertValue( const T & value ) - { + { return value; } @@ -80,7 +67,6 @@ class JSONMessageWriter : public MessageWriter return convertValue( value ); } - template auto convertValue( const std::vector & value, const CspType & type, const FieldEntry & entry ); @@ -92,13 +78,13 @@ class JSONMessageWriter : public MessageWriter template<> inline auto JSONMessageWriter::convertValue( const std::string & value ) { - return rapidjson::StringRef( value.c_str() ); + return rapidjson::StringRef( value.c_str() ); } template<> inline auto JSONMessageWriter::convertValue( const csp::Date & value ) { - return rapidjson::Value( value.asYYYYMMDD().c_str(), m_doc.GetAllocator() ); + return rapidjson::Value( value.asYYYYMMDD().c_str(), m_doc.GetAllocator() ); } template<> @@ -107,46 +93,49 @@ inline auto JSONMessageWriter::convertValue( const csp::DateTime & value ) switch( m_datetimeWireType ) { case utils::DateTimeWireType::UINT64_NANOS: - return ( uint64_t ) value.asNanoseconds(); + return (uint64_t)value.asNanoseconds(); case utils::DateTimeWireType::UINT64_MICROS: - return ( uint64_t ) value.asMicroseconds(); + return (uint64_t)value.asMicroseconds(); case utils::DateTimeWireType::UINT64_MILLIS: - return ( uint64_t ) value.asMilliseconds(); + return (uint64_t)value.asMilliseconds(); case utils::DateTimeWireType::UINT64_SECONDS: - return ( uint64_t ) value.asSeconds(); + return (uint64_t)value.asSeconds(); default: - CSP_THROW( NotImplemented, "datetime wire type " << m_datetimeWireType << " not supported for json msg publishing" ); + CSP_THROW( NotImplemented, + "datetime wire type " << m_datetimeWireType << " not supported for json msg publishing" ); } } template<> inline auto JSONMessageWriter::convertValue( const csp::TimeDelta & value ) { - return rapidjson::Value( value.asNanoseconds() ); + return rapidjson::Value( value.asNanoseconds() ); } template<> -inline auto JSONMessageWriter::convertValue( const csp::CspEnum & value, const CspType & type, const FieldEntry & entry ) +inline auto JSONMessageWriter::convertValue( const csp::CspEnum & value, const CspType & type, + const FieldEntry & entry ) { return rapidjson::StringRef( value.name().c_str() ); } template -inline auto JSONMessageWriter::convertValue( const std::vector & value, const CspType & type, const FieldEntry & entry ) +inline auto JSONMessageWriter::convertValue( const std::vector & value, const CspType & type, + const FieldEntry & entry ) { - auto & allocator = m_doc.GetAllocator(); + auto & allocator = m_doc.GetAllocator(); rapidjson::Value array( rapidjson::kArrayType ); - size_t sz = value.size(); + size_t sz = value.size(); const CspType & elemType = *static_cast( type ).elemType(); using ElemT = typename CspType::Type::toCArrayElemType::type; - //iterating by index for vector support + // iterating by index for vector support for( size_t index = 0; index < sz; ++index ) { - //Note this passes an empty FieldEntry / wont work on vector of structs + // Note this passes an empty FieldEntry / wont work on vector of structs array.PushBack( convertValue( value[index], elemType, {} ), allocator ); } return array; @@ -158,28 +147,30 @@ inline auto JSONMessageWriter::convertValue( const StructPtr & struct_, const Cs rapidjson::Value jValue( rapidjson::kObjectType ); for( auto & nestedEntry : *entry.nestedFields ) { - if( !nestedEntry.sField -> isSet( struct_.get() ) ) + if( !nestedEntry.sField->isSet( struct_.get() ) ) continue; - SupportedCspTypeSwitch::template invoke( - nestedEntry.sField -> type().get(), - [ & ]( auto tag ) + nestedEntry.sField->type().get(), + [&]( auto tag ) { - using T = typename decltype(tag)::type; - jValue.AddMember( rapidjson::StringRef( nestedEntry.outField.c_str() ), convertValue( nestedEntry.sField -> value( struct_.get() ), *nestedEntry.sField -> type(), nestedEntry ), m_doc.GetAllocator() ); + using T = typename decltype( tag )::type; + jValue.AddMember( rapidjson::StringRef( nestedEntry.outField.c_str() ), + convertValue( nestedEntry.sField->value( struct_.get() ), + *nestedEntry.sField->type(), nestedEntry ), + m_doc.GetAllocator() ); } ); }; return jValue; } - template -inline void JSONMessageWriter::setField( const std::string & field, const T & value, const CspType & type, const FieldEntry & entry ) +inline void JSONMessageWriter::setField( const std::string & field, const T & value, const CspType & type, + const FieldEntry & entry ) { m_doc.AddMember( rapidjson::StringRef( field.c_str() ), convertValue( value, type, entry ), m_doc.GetAllocator() ); } -} +} // namespace csp::adapters::utils #endif diff --git a/cpp/csp/adapters/utils/MessageEnums.cpp b/cpp/csp/adapters/utils/MessageEnums.cpp index 903f0bee8..31155541a 100644 --- a/cpp/csp/adapters/utils/MessageEnums.cpp +++ b/cpp/csp/adapters/utils/MessageEnums.cpp @@ -3,12 +3,7 @@ namespace csp { -INIT_CSP_ENUM( csp::adapters::utils::DateTimeWireType, - "UNKNOWN", - "UINT64_NANOS", - "UINT64_MICROS", - "UINT64_MILLIS", - "UINT64_SECONDS" -); +INIT_CSP_ENUM( csp::adapters::utils::DateTimeWireType, "UNKNOWN", "UINT64_NANOS", "UINT64_MICROS", "UINT64_MILLIS", + "UINT64_SECONDS" ); } diff --git a/cpp/csp/adapters/utils/MessageEnums.h b/cpp/csp/adapters/utils/MessageEnums.h index defa7640f..974539fdf 100644 --- a/cpp/csp/adapters/utils/MessageEnums.h +++ b/cpp/csp/adapters/utils/MessageEnums.h @@ -10,7 +10,7 @@ struct DateTimeWireTypeTraits { enum _enum : unsigned char { - UNKNOWN = 0, + UNKNOWN = 0, UINT64_NANOS = 1, UINT64_MICROS = 2, UINT64_MILLIS = 3, @@ -25,6 +25,6 @@ struct DateTimeWireTypeTraits using DateTimeWireType = csp::Enum; -}; +}; // namespace csp::adapters::utils #endif diff --git a/cpp/csp/adapters/utils/MessageStructConverter.cpp b/cpp/csp/adapters/utils/MessageStructConverter.cpp index 66e223d2c..e096fd264 100644 --- a/cpp/csp/adapters/utils/MessageStructConverter.cpp +++ b/cpp/csp/adapters/utils/MessageStructConverter.cpp @@ -1,28 +1,30 @@ -#include #include +#include #include namespace csp::adapters::utils { -MessageStructConverter::MessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) : m_type( type ) +MessageStructConverter::MessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) + : m_type( type ) { - if( type -> type() == CspType::Type::STRUCT ) - m_structMeta = std::static_pointer_cast( type ) -> meta(); + if( type->type() == CspType::Type::STRUCT ) + m_structMeta = std::static_pointer_cast( type )->meta(); } MessageStructConverterCache::MessageStructConverterCache() { registerConverter( "RAW_BYTES", &RawBytesMessageStructConverter::create ); - registerConverter( "JSON", &JSONMessageStructConverter::create ); + registerConverter( "JSON", &JSONMessageStructConverter::create ); } bool MessageStructConverterCache::registerConverter( std::string protocol, Creator creator ) { if( m_creators.find( protocol ) != m_creators.end() ) - CSP_THROW( RuntimeException, "Attempted to register creator for MessageStructConverter type " << protocol << " more than once" ); + CSP_THROW( RuntimeException, + "Attempted to register creator for MessageStructConverter type " << protocol << " more than once" ); - m_creators[ protocol ] = creator; + m_creators[protocol] = creator; return true; } @@ -38,16 +40,16 @@ MessageStructConverterPtr MessageStructConverterCache::create( const CspTypePtr auto rv = m_cache.emplace( CacheKey{ type.get(), properties }, nullptr ); if( !rv.second ) - return rv.first -> second; + return rv.first->second; - auto protocol = properties.get( "protocol" ); + auto protocol = properties.get( "protocol" ); auto creatorIt = m_creators.find( protocol ); if( creatorIt == m_creators.end() ) CSP_THROW( ValueError, "MessageStructConverter for type " << protocol << " is not defined" ); - auto result = std::shared_ptr( creatorIt -> second( type, properties ) ); - rv.first -> second = result; - return rv.first -> second; + auto result = std::shared_ptr( creatorIt->second( type, properties ) ); + rv.first->second = result; + return rv.first->second; } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/MessageStructConverter.h b/cpp/csp/adapters/utils/MessageStructConverter.h index 268c94d37..353d34bfa 100644 --- a/cpp/csp/adapters/utils/MessageStructConverter.h +++ b/cpp/csp/adapters/utils/MessageStructConverter.h @@ -19,7 +19,7 @@ class MessageStructConverter public: MessageStructConverter( const CspTypePtr & type, const Dictionary & properties ); virtual ~MessageStructConverter() {} - + virtual csp::StructPtr asStruct( void * bytes, size_t size ) = 0; StructMetaPtr structMeta() { return m_structMeta; } @@ -29,16 +29,16 @@ class MessageStructConverter StructMetaPtr m_structMeta; private: - using FieldEntry = std::pair; + using FieldEntry = std::pair; using Fields = std::vector; Fields m_propertyFields; }; -using MessageStructConverterPtr=std::shared_ptr; +using MessageStructConverterPtr = std::shared_ptr; -//This ensures we dont recreate converters unnecessarily for say subscription by symbol with the same -//conversion onformation +// This ensures we dont recreate converters unnecessarily for say subscription by symbol with the same +// conversion onformation class MessageStructConverterCache { public: @@ -48,19 +48,19 @@ class MessageStructConverterCache MessageStructConverterPtr create( const CspTypePtr &, const Dictionary & properties ); - using Creator = std::function; + using Creator = std::function; bool registerConverter( std::string protocol, Creator creator ); private: - using CacheKey = std::pair; - using Cache = std::unordered_map; + using CacheKey = std::pair; + using Cache = std::unordered_map; - std::unordered_map m_creators; - std::mutex m_cacheMutex; - Cache m_cache; + std::unordered_map m_creators; + std::mutex m_cacheMutex; + Cache m_cache; }; -} +} // namespace csp::adapters::utils #endif diff --git a/cpp/csp/adapters/utils/MessageWriter.cpp b/cpp/csp/adapters/utils/MessageWriter.cpp index 8d8c6296c..8dea054c8 100644 --- a/cpp/csp/adapters/utils/MessageWriter.cpp +++ b/cpp/csp/adapters/utils/MessageWriter.cpp @@ -1,20 +1,19 @@ -#include #include +#include namespace csp::adapters::utils { -MessageWriter::~MessageWriter() -{ -} +MessageWriter::~MessageWriter() {} -OutputDataMapper::OutputDataMapper( const CspTypePtr & type, const Dictionary & fieldMap ) : m_type( type ) +OutputDataMapper::OutputDataMapper( const CspTypePtr & type, const Dictionary & fieldMap ) + : m_type( type ) { m_hasFields = !fieldMap.empty(); - bool isStruct = type -> type() == CspType::Type::STRUCT; + bool isStruct = type->type() == CspType::Type::STRUCT; if( isStruct ) - m_structMeta = std::static_pointer_cast( type ) -> meta(); + m_structMeta = std::static_pointer_cast( type )->meta(); if( isStruct ) m_messageFields = populateStructFields( m_structMeta, fieldMap ); @@ -27,43 +26,55 @@ OutputDataMapper::OutputDataMapper( const CspTypePtr & type, const Dictionary & } } -OutputDataMapper::Fields OutputDataMapper::populateStructFields( const StructMetaPtr & structMeta, const Dictionary & fieldMap ) +OutputDataMapper::Fields OutputDataMapper::populateStructFields( const StructMetaPtr & structMeta, + const Dictionary & fieldMap ) { Fields out; for( auto it = fieldMap.begin(); it != fieldMap.end(); ++it ) { auto & structField = it.key(); - auto sField = structMeta -> field( structField ); + auto sField = structMeta->field( structField ); if( !sField ) - CSP_THROW( ValueError, "field " << structField << " is not a valid field on struct type " << structMeta -> name() ); + CSP_THROW( ValueError, + "field " << structField << " is not a valid field on struct type " << structMeta->name() ); if( it.hasValue() ) { - if( sField -> type() -> type() == CspType::Type::STRUCT ) - CSP_THROW( TypeError, "Expected non-struct type for fieldmap on struct field " << structField << " on struct " << structMeta -> name() ); + if( sField->type()->type() == CspType::Type::STRUCT ) + CSP_THROW( TypeError, + "Expected non-struct type for fieldmap on struct field " << structField << " on struct " + << structMeta->name() ); - auto & fieldName = it.value(); + auto & fieldName = it.value(); out.emplace_back( FieldEntry{ fieldName, sField, {} } ); } else { if( !it.hasValue() ) - CSP_THROW( TypeError, "fieldMap expected string or dict for field \"" << structField << "\" on struct \"" << structMeta -> name() << "\""); + CSP_THROW( TypeError, + "fieldMap expected string or dict for field \"" << structField << "\" on struct \"" + << structMeta->name() << "\"" ); auto nestedEntry = it.value(); - if( nestedEntry -> size() != 1 || !nestedEntry -> begin().hasValue() ) - CSP_THROW( ValueError, "Expected nested fieldmap for outgoing field \"" << structField << "\" on struct \"" << structMeta -> name() << - "\" to have a dict of single key : nested_fieldmap entry" ); - - if( sField -> type() -> type() != CspType::Type::STRUCT ) - CSP_THROW( TypeError, "Expected structed type for nested struct field \"" << structField << "\" on struct \"" << structMeta -> name() << "\" got: \"" << sField -> type() -> type() << "\"" ); - - //sub-struct - auto nestedStructType = std::static_pointer_cast( sField -> type() ); - auto fieldName = nestedEntry -> begin().key(); - auto nestedFieldMap = nestedEntry -> begin().value(); - - auto nestedFields = std::make_shared( populateStructFields( nestedStructType -> meta(), *nestedFieldMap ) ); + if( nestedEntry->size() != 1 || !nestedEntry->begin().hasValue() ) + CSP_THROW( ValueError, + "Expected nested fieldmap for outgoing field \"" + << structField << "\" on struct \"" << structMeta->name() + << "\" to have a dict of single key : nested_fieldmap entry" ); + + if( sField->type()->type() != CspType::Type::STRUCT ) + CSP_THROW( TypeError, + "Expected structed type for nested struct field \"" << structField << "\" on struct \"" + << structMeta->name() << "\" got: \"" + << sField->type()->type() << "\"" ); + + // sub-struct + auto nestedStructType = std::static_pointer_cast( sField->type() ); + auto fieldName = nestedEntry->begin().key(); + auto nestedFieldMap = nestedEntry->begin().value(); + + auto nestedFields + = std::make_shared( populateStructFields( nestedStructType->meta(), *nestedFieldMap ) ); out.emplace_back( FieldEntry{ fieldName, sField, nestedFields } ); } } @@ -71,9 +82,7 @@ OutputDataMapper::Fields OutputDataMapper::populateStructFields( const StructMet return out; } -OutputDataMapperCache::OutputDataMapperCache() -{ -} +OutputDataMapperCache::OutputDataMapperCache() {} OutputDataMapperCache & OutputDataMapperCache::instance() { @@ -87,11 +96,11 @@ OutputDataMapperPtr OutputDataMapperCache::create( const CspTypePtr & type, cons auto rv = m_cache.emplace( CacheKey{ type.get(), fieldMap }, nullptr ); if( !rv.second ) - return rv.first -> second; + return rv.first->second; - auto helper = std::make_shared( type, fieldMap ); - rv.first -> second = helper; - return rv.first -> second; + auto helper = std::make_shared( type, fieldMap ); + rv.first->second = helper; + return rv.first->second; } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/MessageWriter.h b/cpp/csp/adapters/utils/MessageWriter.h index 79794157f..6b29a6837 100644 --- a/cpp/csp/adapters/utils/MessageWriter.h +++ b/cpp/csp/adapters/utils/MessageWriter.h @@ -10,13 +10,14 @@ #include #include #include -#include #include +#include namespace csp::adapters::utils { -//This is used to map data from an output adapter- > message writer ( can have multiple ouput adapters writing to same message ) +// This is used to map data from an output adapter- > message writer ( can have multiple ouput adapters writing to same +// message ) class OutputDataMapper { public: @@ -29,9 +30,9 @@ class OutputDataMapper struct FieldEntry { - std::string outField; - StructFieldPtr sField; - std::shared_ptr> nestedFields; //for nested structs + std::string outField; + StructFieldPtr sField; + std::shared_ptr> nestedFields; // for nested structs }; using Fields = std::vector; @@ -39,26 +40,26 @@ class OutputDataMapper private: Fields populateStructFields( const StructMetaPtr & structMeta, const Dictionary & field_map ); - //struct outputs + // struct outputs template void applyStruct( MessageWriterT & writer, const Struct * struct_ ) const; - //non-struct outputs - template + // non-struct outputs + template void applyField( MessageWriterT & writer, const T & value ) const; - CspTypePtr m_type; - bool m_hasFields; + CspTypePtr m_type; + bool m_hasFields; - //Struct specific + // Struct specific StructMetaPtr m_structMeta; Fields m_messageFields; - //non-struct specific + // non-struct specific std::string m_messageFieldName; }; -using OutputDataMapperPtr=std::shared_ptr; +using OutputDataMapperPtr = std::shared_ptr; // Derived types will be used to create and write timeseries data -> target message protocol ( ie JSON, proto ) // and convert it to bytes for the output adapter @@ -70,8 +71,8 @@ class MessageWriter MessageWriter() {} virtual ~MessageWriter(); - //returns the finalized message as bytes - virtual std::pair finalize() = 0; + // returns the finalized message as bytes + virtual std::pair finalize() = 0; void processTick( const OutputDataMapper & dataMapper, const TimeSeriesProvider * sourcets ) { @@ -83,7 +84,7 @@ class MessageWriter virtual void processTickImpl( const OutputDataMapper & dataMapper, const TimeSeriesProvider * sourcets ) = 0; }; -using MessageWriterPtr=std::shared_ptr; +using MessageWriterPtr = std::shared_ptr; template inline void OutputDataMapper::apply( MessageWriterT & writer, const TimeSeriesProvider * ts ) const @@ -91,22 +92,20 @@ inline void OutputDataMapper::apply( MessageWriterT & writer, const TimeSeriesPr if( !m_hasFields ) return; - if( ts -> type() -> type() == CspType::Type::STRUCT ) - applyStruct( writer, ts -> lastValueTyped().get() ); + if( ts->type()->type() == CspType::Type::STRUCT ) + applyStruct( writer, ts->lastValueTyped().get() ); else { - MessageWriterT::SupportedCspTypeSwitch::template invoke( ts -> type(), - [&]( auto tag ) - { - applyField( writer, ts -> lastValueTyped() ); - } ); + MessageWriterT::SupportedCspTypeSwitch::template invoke( + ts->type(), + [&]( auto tag ) { applyField( writer, ts->lastValueTyped() ); } ); } } -template +template inline void OutputDataMapper::applyField( MessageWriterT & writer, const T & value ) const { - CSP_ASSERT( m_type -> type() != CspType::Type::STRUCT ); + CSP_ASSERT( m_type->type() != CspType::Type::STRUCT ); if( !m_messageFieldName.empty() ) writer.setField( m_messageFieldName, value, *m_type, {} ); @@ -115,43 +114,43 @@ inline void OutputDataMapper::applyField( MessageWriterT & writer, const T & val template inline void OutputDataMapper::applyStruct( MessageWriterT & writer, const Struct * struct_ ) const { - CSP_ASSERT( m_type -> type() == CspType::Type::STRUCT ); + CSP_ASSERT( m_type->type() == CspType::Type::STRUCT ); for( auto & entry : m_messageFields ) { - if( !entry.sField -> isSet( struct_ ) ) + if( !entry.sField->isSet( struct_ ) ) continue; using SwitchT = typename MessageWriterT::SupportedCspTypeSwitch; SwitchT::template invoke( - entry.sField -> type().get(), - [ & ]( auto tag ) + entry.sField->type().get(), + [&]( auto tag ) { - using T = typename decltype(tag)::type; - writer.setField( entry.outField, entry.sField -> value( struct_ ), *entry.sField -> type(), entry ); + using T = typename decltype( tag )::type; + writer.setField( entry.outField, entry.sField->value( struct_ ), *entry.sField->type(), entry ); } ); }; } -//This ensures we dont recreate duplicate writers unnecessarily +// This ensures we dont recreate duplicate writers unnecessarily class OutputDataMapperCache { public: OutputDataMapperCache(); - + static OutputDataMapperCache & instance(); OutputDataMapperPtr create( const CspTypePtr &, const Dictionary & fieldMap ); private: - using CacheKey = std::pair; - using Cache = std::unordered_map; + using CacheKey = std::pair; + using Cache = std::unordered_map; std::mutex m_cacheMutex; Cache m_cache; }; -} +} // namespace csp::adapters::utils #endif diff --git a/cpp/csp/adapters/utils/RawBytesMessageStructConverter.cpp b/cpp/csp/adapters/utils/RawBytesMessageStructConverter.cpp index 4eaa87e3b..89a7dbb98 100644 --- a/cpp/csp/adapters/utils/RawBytesMessageStructConverter.cpp +++ b/cpp/csp/adapters/utils/RawBytesMessageStructConverter.cpp @@ -3,9 +3,9 @@ namespace csp::adapters::utils { -RawBytesMessageStructConverter::RawBytesMessageStructConverter( const CspTypePtr & type, - const Dictionary & properties ) : MessageStructConverter( type, properties ), - m_targetField( nullptr ) +RawBytesMessageStructConverter::RawBytesMessageStructConverter( const CspTypePtr & type, const Dictionary & properties ) + : MessageStructConverter( type, properties ) + , m_targetField( nullptr ) { const Dictionary & fieldMap = *properties.get( "field_map" ); if( fieldMap.size() > 1 ) @@ -13,31 +13,35 @@ RawBytesMessageStructConverter::RawBytesMessageStructConverter( const CspTypePtr if( fieldMap.size() == 1 ) { - if( type -> type() != CspType::Type::STRUCT ) - CSP_THROW( ValueError, "field_map provided on non-struct type " << type -> type() << " in adapter" ); + if( type->type() != CspType::Type::STRUCT ) + CSP_THROW( ValueError, "field_map provided on non-struct type " << type->type() << " in adapter" ); if( !fieldMap.exists( "" ) ) - CSP_THROW( ValueError, "RawBytesMessageStructConverter expects one entry in fieldMap with empty string as source key" ); + CSP_THROW( ValueError, + "RawBytesMessageStructConverter expects one entry in fieldMap with empty string as source key" ); auto targetKey = fieldMap.get( "" ); - auto sField = m_structMeta -> field( targetKey ); - if( !sField || sField -> type() -> type() != CspType::Type::STRING ) - CSP_THROW( TypeError, "field " << targetKey << " on struct " << m_structMeta -> name() << ( sField ? "is not string type" : "does not exist" ) ); + auto sField = m_structMeta->field( targetKey ); + if( !sField || sField->type()->type() != CspType::Type::STRING ) + CSP_THROW( TypeError, + "field " << targetKey << " on struct " << m_structMeta->name() + << ( sField ? "is not string type" : "does not exist" ) ); m_targetField = static_cast( sField.get() ); } - else if( type -> type() != CspType::Type::STRING ) - CSP_THROW( TypeError, "TestMessageStructConverter expected type of STRING for empty field_map got " << type -> type() ); + else if( type->type() != CspType::Type::STRING ) + CSP_THROW( TypeError, + "TestMessageStructConverter expected type of STRING for empty field_map got " << type->type() ); } csp::StructPtr RawBytesMessageStructConverter::asStruct( void * bytes, size_t size ) { - if( m_type -> type() == CspType::Type::STRUCT ) + if( m_type->type() == CspType::Type::STRUCT ) { - StructPtr data = m_structMeta -> create(); - m_targetField -> setValue( data.get(), std::string( ( const char * ) bytes, size ) ); + StructPtr data = m_structMeta->create(); + m_targetField->setValue( data.get(), std::string( (const char *)bytes, size ) ); return data; } else - abort(); //TBD doesnt fit asStruct API + abort(); // TBD doesnt fit asStruct API } -} +} // namespace csp::adapters::utils diff --git a/cpp/csp/adapters/utils/RawBytesMessageStructConverter.h b/cpp/csp/adapters/utils/RawBytesMessageStructConverter.h index 06e95d624..976fdda3a 100644 --- a/cpp/csp/adapters/utils/RawBytesMessageStructConverter.h +++ b/cpp/csp/adapters/utils/RawBytesMessageStructConverter.h @@ -10,7 +10,6 @@ namespace csp::adapters::utils class RawBytesMessageStructConverter : public MessageStructConverter { public: - RawBytesMessageStructConverter( const CspTypePtr & type, const Dictionary & properties ); csp::StructPtr asStruct( void * bytes, size_t size ) override; @@ -24,6 +23,6 @@ class RawBytesMessageStructConverter : public MessageStructConverter const StringStructField * m_targetField; }; -} +} // namespace csp::adapters::utils #endif diff --git a/cpp/csp/adapters/utils/StructAdapterInfo.h b/cpp/csp/adapters/utils/StructAdapterInfo.h index 64dfe5497..82d3d3b5b 100644 --- a/cpp/csp/adapters/utils/StructAdapterInfo.h +++ b/cpp/csp/adapters/utils/StructAdapterInfo.h @@ -1,11 +1,10 @@ #ifndef _IN_CSP_ADAPTERS_PARQUET_StructAdapterInfo_H #define _IN_CSP_ADAPTERS_PARQUET_StructAdapterInfo_H -#include #include +#include #include - namespace csp::adapters::utils { @@ -13,60 +12,53 @@ class StructAdapterInfo { public: StructAdapterInfo( CspTypePtr type, DictionaryPtr fieldMap ) - : m_type( type ), m_fieldMap( fieldMap ), m_hash( 0x9e3779b9 ) + : m_type( type ) + , m_fieldMap( fieldMap ) + , m_hash( 0x9e3779b9 ) { std::hash stringHasher; if( m_fieldMap ) { - for( auto it = m_fieldMap -> begin(); it != m_fieldMap -> end(); ++it ) + for( auto it = m_fieldMap->begin(); it != m_fieldMap->end(); ++it ) { - m_hash ^= stringHasher( it.key()); - m_hash ^= stringHasher( it.value()); + m_hash ^= stringHasher( it.key() ); + m_hash ^= stringHasher( it.value() ); } } } - CspTypePtr type() const - { - return m_type; - } + CspTypePtr type() const { return m_type; } - DictionaryPtr fieldMap() const - { - return m_fieldMap; - } + DictionaryPtr fieldMap() const { return m_fieldMap; } - std::size_t hash() const - { - return m_hash; - } + std::size_t hash() const { return m_hash; } - bool operator==( const StructAdapterInfo &other ) const + bool operator==( const StructAdapterInfo & other ) const { - auto meta = std::static_pointer_cast( m_type ) -> meta(); - auto otherMeta = std::static_pointer_cast( other.m_type ) -> meta(); + auto meta = std::static_pointer_cast( m_type )->meta(); + auto otherMeta = std::static_pointer_cast( other.m_type )->meta(); if( meta != otherMeta ) { return false; } - if( bool( m_fieldMap ) != bool( other.m_fieldMap )) + if( bool( m_fieldMap ) != bool( other.m_fieldMap ) ) { return false; } if( m_fieldMap ) { - if( m_fieldMap -> size() != other.m_fieldMap -> size()) + if( m_fieldMap->size() != other.m_fieldMap->size() ) { return false; } - for( auto it = m_fieldMap -> begin(); it != m_fieldMap -> end(); ++it ) + for( auto it = m_fieldMap->begin(); it != m_fieldMap->end(); ++it ) { - if( !other.m_fieldMap -> exists( it.key())) + if( !other.m_fieldMap->exists( it.key() ) ) { return false; } - if( it.value() != other.m_fieldMap -> get( it.key())) + if( it.value() != other.m_fieldMap->get( it.key() ) ) { return false; } @@ -83,17 +75,16 @@ class StructAdapterInfo std::size_t m_hash; }; -} +} // namespace csp::adapters::utils namespace std { template<> struct hash { - size_t - operator()( const csp::adapters::utils::StructAdapterInfo &o ) const noexcept { return o.hash(); } + size_t operator()( const csp::adapters::utils::StructAdapterInfo & o ) const noexcept { return o.hash(); } }; -} +} // namespace std #endif diff --git a/cpp/csp/adapters/utils/ValueDispatcher.h b/cpp/csp/adapters/utils/ValueDispatcher.h index 4a1451f62..994a2bf05 100644 --- a/cpp/csp/adapters/utils/ValueDispatcher.h +++ b/cpp/csp/adapters/utils/ValueDispatcher.h @@ -4,9 +4,9 @@ #include #include #include +#include #include #include -#include namespace csp { @@ -16,27 +16,26 @@ class PushBatch; namespace csp::adapters::utils { -using Symbol=std::variant; +using Symbol = std::variant; -template< typename V, typename ...SubscriberArgs > +template class ValueDispatcher final { public: - using ValueType = std::remove_reference_t; - using SubscriberType = std::function; + using ValueType = std::remove_reference_t; + using SubscriberType = std::function; using SubscriberContainer = std::vector; void addSubscriber( SubscriberType subscriber, std::optional symbol = {} ) { - if( symbol.has_value()) + if( symbol.has_value() ) { - auto it = m_subscriberBySymbol.find( symbol.value()); - if( it == m_subscriberBySymbol.end()) + auto it = m_subscriberBySymbol.find( symbol.value() ); + if( it == m_subscriberBySymbol.end() ) { - it = m_subscriberBySymbol.emplace( symbol.value(), std::vector()).first; + it = m_subscriberBySymbol.emplace( symbol.value(), std::vector() ).first; } it->second.push_back( subscriber ); - } else { @@ -44,9 +43,9 @@ class ValueDispatcher final } } - SubscriberContainer *getSubscribers() + SubscriberContainer * getSubscribers() { - if( m_subscribers.empty()) + if( m_subscribers.empty() ) { return nullptr; } @@ -56,10 +55,10 @@ class ValueDispatcher final } } - SubscriberContainer *getSubscribersForSymbol( const Symbol &symbol ) + SubscriberContainer * getSubscribersForSymbol( const Symbol & symbol ) { auto it = m_subscriberBySymbol.find( symbol ); - if( it != m_subscriberBySymbol.end()) + if( it != m_subscriberBySymbol.end() ) { return &it->second; } @@ -69,21 +68,17 @@ class ValueDispatcher final } } - void dispatch( ValueType *v, SubscriberContainer &subscriberContainer, SubscriberArgs... extraArgs ) + void dispatch( ValueType * v, SubscriberContainer & subscriberContainer, SubscriberArgs... extraArgs ) { - for( auto &subscriber: subscriberContainer ) + for( auto & subscriber : subscriberContainer ) { subscriber( v, extraArgs... ); } } + void dispatch( ValueType * v, SubscriberArgs... extraArgs ) { dispatch( v, m_subscribers, extraArgs... ); } - void dispatch( ValueType *v, SubscriberArgs... extraArgs ) - { - dispatch( v, m_subscribers, extraArgs... ); - } - - void dispatch( ValueType *v, const Symbol *symbol, SubscriberArgs... extraArgs ) + void dispatch( ValueType * v, const Symbol * symbol, SubscriberArgs... extraArgs ) { dispatch( v, m_subscribers, extraArgs... ); if( symbol ) @@ -101,8 +96,8 @@ class ValueDispatcher final std::unordered_map m_subscriberBySymbol; }; -template< typename V > +template using PushBatchValueDispatcher = ValueDispatcher; -} +} // namespace csp::adapters::utils #endif diff --git a/cpp/csp/adapters/websocket/ClientAdapterManager.cpp b/cpp/csp/adapters/websocket/ClientAdapterManager.cpp index 423f2a234..d776ce31f 100644 --- a/cpp/csp/adapters/websocket/ClientAdapterManager.cpp +++ b/cpp/csp/adapters/websocket/ClientAdapterManager.cpp @@ -1,122 +1,126 @@ #include -namespace csp { +namespace csp +{ -INIT_CSP_ENUM( adapters::websocket::ClientStatusType, - "ACTIVE", - "GENERIC_ERROR", - "CONNECTION_FAILED", - "CLOSED", - "MESSAGE_SEND_FAIL", -); +INIT_CSP_ENUM( adapters::websocket::ClientStatusType, "ACTIVE", "GENERIC_ERROR", "CONNECTION_FAILED", "CLOSED", + "MESSAGE_SEND_FAIL", ); } // With TLS -namespace csp::adapters::websocket { - -ClientAdapterManager::ClientAdapterManager( Engine* engine, const Dictionary & properties ) -: AdapterManager( engine ), - m_active( false ), - m_shouldRun( false ), - m_endpoint( std::make_unique( properties ) ), - m_inputAdapter( nullptr ), - m_outputAdapter( nullptr ), - m_updateAdapter( nullptr ), - m_thread( nullptr ), - m_properties( properties ) -{ }; - -ClientAdapterManager::~ClientAdapterManager() -{ }; +namespace csp::adapters::websocket +{ + +ClientAdapterManager::ClientAdapterManager( Engine * engine, const Dictionary & properties ) + : AdapterManager( engine ) + , m_active( false ) + , m_shouldRun( false ) + , m_endpoint( std::make_unique( properties ) ) + , m_inputAdapter( nullptr ) + , m_outputAdapter( nullptr ) + , m_updateAdapter( nullptr ) + , m_thread( nullptr ) + , m_properties( properties ) {}; + +ClientAdapterManager::~ClientAdapterManager() {}; void ClientAdapterManager::start( DateTime starttime, DateTime endtime ) { AdapterManager::start( starttime, endtime ); m_shouldRun = true; - m_endpoint -> setOnOpen( - [ this ]() { + m_endpoint->setOnOpen( + [this]() + { m_active = true; pushStatus( StatusLevel::INFO, ClientStatusType::ACTIVE, "Connected successfully" ); - } - ); - m_endpoint -> setOnFail( - [ this ]( const std::string& reason ) { + } ); + m_endpoint->setOnFail( + [this]( const std::string & reason ) + { std::stringstream ss; ss << "Connection Failure: " << reason; m_active = false; pushStatus( StatusLevel::ERROR, ClientStatusType::CONNECTION_FAILED, ss.str() ); - } - ); - if( m_inputAdapter ) { - m_endpoint -> setOnMessage( - [ this ]( void* c, size_t t ) { - PushBatch batch( m_engine -> rootEngine() ); - m_inputAdapter -> processMessage( c, t, &batch ); - } - ); - } else { + } ); + if( m_inputAdapter ) + { + m_endpoint->setOnMessage( + [this]( void * c, size_t t ) + { + PushBatch batch( m_engine->rootEngine() ); + m_inputAdapter->processMessage( c, t, &batch ); + } ); + } + else + { // if a user doesn't call WebsocketAdapterManager.subscribe, no inputadapter will be created // but we still need something to avoid on_message_cb not being set in the endpoint. - m_endpoint -> setOnMessage( []( void* c, size_t t ){} ); + m_endpoint->setOnMessage( []( void * c, size_t t ) {} ); } - m_endpoint -> setOnClose( - [ this ]() { + m_endpoint->setOnClose( + [this]() + { m_active = false; pushStatus( StatusLevel::INFO, ClientStatusType::CLOSED, "Connection closed" ); - } - ); - m_endpoint -> setOnSendFail( - [ this ]( const std::string& s ) { + } ); + m_endpoint->setOnSendFail( + [this]( const std::string & s ) + { std::stringstream ss; ss << "Failed to send: " << s; pushStatus( StatusLevel::ERROR, ClientStatusType::MESSAGE_SEND_FAIL, ss.str() ); - } - ); + } ); - m_thread = std::make_unique( [ this ]() { - while( m_shouldRun ) + m_thread = std::make_unique( + [this]() { - m_endpoint -> run(); - m_active = false; - if( m_shouldRun ) sleep( m_properties.get( "reconnect_interval" ) ); - } - }); + while( m_shouldRun ) + { + m_endpoint->run(); + m_active = false; + if( m_shouldRun ) + sleep( m_properties.get( "reconnect_interval" ) ); + } + } ); }; -void ClientAdapterManager::stop() { +void ClientAdapterManager::stop() +{ AdapterManager::stop(); - m_shouldRun=false; - if( m_active ) m_endpoint->stop(); - if( m_thread ) m_thread->join(); + m_shouldRun = false; + if( m_active ) + m_endpoint->stop(); + if( m_thread ) + m_thread->join(); }; -PushInputAdapter* ClientAdapterManager::getInputAdapter(CspTypePtr & type, PushMode pushMode, const Dictionary & properties) +PushInputAdapter * ClientAdapterManager::getInputAdapter( CspTypePtr & type, PushMode pushMode, + const Dictionary & properties ) { - if (m_inputAdapter == nullptr) + if( m_inputAdapter == nullptr ) { - m_inputAdapter = m_engine -> createOwnedObject( + m_inputAdapter = m_engine->createOwnedObject( // m_engine, - type, - pushMode, - properties - ); + type, pushMode, properties ); } return m_inputAdapter; }; -OutputAdapter* ClientAdapterManager::getOutputAdapter() +OutputAdapter * ClientAdapterManager::getOutputAdapter() { - if (m_outputAdapter == nullptr) m_outputAdapter = m_engine -> createOwnedObject(*m_endpoint); + if( m_outputAdapter == nullptr ) + m_outputAdapter = m_engine->createOwnedObject( *m_endpoint ); return m_outputAdapter; } OutputAdapter * ClientAdapterManager::getHeaderUpdateAdapter() { - if (m_updateAdapter == nullptr) m_updateAdapter = m_engine -> createOwnedObject( m_endpoint -> getProperties() ); + if( m_updateAdapter == nullptr ) + m_updateAdapter = m_engine->createOwnedObject( m_endpoint->getProperties() ); return m_updateAdapter; } @@ -127,4 +131,4 @@ DateTime ClientAdapterManager::processNextSimTimeSlice( DateTime time ) return DateTime::NONE(); } -} +} // namespace csp::adapters::websocket diff --git a/cpp/csp/adapters/websocket/ClientAdapterManager.h b/cpp/csp/adapters/websocket/ClientAdapterManager.h index 62577d769..ffc9b2ce9 100644 --- a/cpp/csp/adapters/websocket/ClientAdapterManager.h +++ b/cpp/csp/adapters/websocket/ClientAdapterManager.h @@ -1,23 +1,23 @@ #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_ADAPTERMGR_H #define _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_ADAPTERMGR_H -#include +#include +#include #include #include -#include +#include #include #include +#include #include #include #include -#include -#include -#include #include #include +#include - -namespace csp::adapters::websocket { +namespace csp::adapters::websocket +{ using namespace csp; @@ -25,10 +25,10 @@ struct WebsocketClientStatusTypeTraits { enum _enum : unsigned char { - ACTIVE = 0, - GENERIC_ERROR = 1, + ACTIVE = 0, + GENERIC_ERROR = 1, CONNECTION_FAILED = 2, - CLOSED = 3, + CLOSED = 3, MESSAGE_SEND_FAIL = 4, NUM_TYPES @@ -43,12 +43,8 @@ using ClientStatusType = Enum; class ClientAdapterManager final : public AdapterManager { - public: - ClientAdapterManager( - Engine * engine, - const Dictionary & properties - ); + ClientAdapterManager( Engine * engine, const Dictionary & properties ); ~ClientAdapterManager(); const char * name() const override { return "WebsocketClientAdapterManager"; } @@ -58,24 +54,24 @@ class ClientAdapterManager final : public AdapterManager void stop() override; PushInputAdapter * getInputAdapter( CspTypePtr & type, PushMode pushMode, const Dictionary & properties ); - OutputAdapter * getOutputAdapter(); - OutputAdapter * getHeaderUpdateAdapter(); + OutputAdapter * getOutputAdapter(); + OutputAdapter * getHeaderUpdateAdapter(); DateTime processNextSimTimeSlice( DateTime time ) override; private: // need some client info - - bool m_active; - bool m_shouldRun; + + bool m_active; + bool m_shouldRun; std::unique_ptr m_endpoint; - ClientInputAdapter* m_inputAdapter; - ClientOutputAdapter* m_outputAdapter; - ClientHeaderUpdateOutputAdapter* m_updateAdapter; - std::unique_ptr m_thread; - Dictionary m_properties; + ClientInputAdapter * m_inputAdapter; + ClientOutputAdapter * m_outputAdapter; + ClientHeaderUpdateOutputAdapter * m_updateAdapter; + std::unique_ptr m_thread; + Dictionary m_properties; }; -} +} // namespace csp::adapters::websocket #endif \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.cpp b/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.cpp index 995bd7314..e950e680e 100644 --- a/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.cpp +++ b/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.cpp @@ -1,20 +1,20 @@ #include -namespace csp::adapters::websocket { +namespace csp::adapters::websocket +{ -ClientHeaderUpdateOutputAdapter::ClientHeaderUpdateOutputAdapter( - Engine * engine, - Dictionary& properties -) : OutputAdapter( engine ), m_properties( properties ) -{ }; +ClientHeaderUpdateOutputAdapter::ClientHeaderUpdateOutputAdapter( Engine * engine, Dictionary & properties ) + : OutputAdapter( engine ) + , m_properties( properties ) {}; void ClientHeaderUpdateOutputAdapter::executeImpl() { - DictionaryPtr headers = m_properties.get("headers"); - for( auto& update : input() -> lastValueTyped>() ) - { - if( update -> key_isSet() && update -> value_isSet() ) headers->update( update->key(), update->value() ); + DictionaryPtr headers = m_properties.get( "headers" ); + for( auto & update : input()->lastValueTyped>() ) + { + if( update->key_isSet() && update->value_isSet() ) + headers->update( update->key(), update->value() ); } }; -} \ No newline at end of file +} // namespace csp::adapters::websocket \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.h b/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.h index d2c898a1e..ba979d11e 100644 --- a/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.h +++ b/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.h @@ -1,33 +1,28 @@ #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_HEADERUPDATEADAPTER_H #define _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_HEADERUPDATEADAPTER_H -#include -#include #include #include +#include +#include namespace csp::adapters::websocket { using namespace csp::autogen; -class ClientHeaderUpdateOutputAdapter final: public OutputAdapter +class ClientHeaderUpdateOutputAdapter final : public OutputAdapter { public: - ClientHeaderUpdateOutputAdapter( - Engine * engine, - Dictionary& properties - ); + ClientHeaderUpdateOutputAdapter( Engine * engine, Dictionary & properties ); void executeImpl() override; const char * name() const override { return "WebsocketClientHeaderUpdateAdapter"; } private: - Dictionary& m_properties; - + Dictionary & m_properties; }; -} - +} // namespace csp::adapters::websocket #endif \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientInputAdapter.cpp b/cpp/csp/adapters/websocket/ClientInputAdapter.cpp index e4b0b7ff7..b2767f307 100644 --- a/cpp/csp/adapters/websocket/ClientInputAdapter.cpp +++ b/cpp/csp/adapters/websocket/ClientInputAdapter.cpp @@ -3,40 +3,36 @@ namespace csp::adapters::websocket { -ClientInputAdapter::ClientInputAdapter( - Engine * engine, - CspTypePtr & type, - PushMode pushMode, - const Dictionary & properties -) : PushInputAdapter(engine, type, pushMode) +ClientInputAdapter::ClientInputAdapter( Engine * engine, CspTypePtr & type, PushMode pushMode, + const Dictionary & properties ) + : PushInputAdapter( engine, type, pushMode ) { - if( type -> type() != CspType::Type::STRUCT && - type -> type() != CspType::Type::STRING ) - CSP_THROW( RuntimeException, "Unsupported type: " << type -> type() ); + if( type->type() != CspType::Type::STRUCT && type->type() != CspType::Type::STRING ) + CSP_THROW( RuntimeException, "Unsupported type: " << type->type() ); if( properties.exists( "meta_field_map" ) ) { const Dictionary & metaFieldMap = *properties.get( "meta_field_map" ); - if( !metaFieldMap.empty() && type -> type() != CspType::Type::STRUCT ) + if( !metaFieldMap.empty() && type->type() != CspType::Type::STRUCT ) CSP_THROW( ValueError, "meta_field_map is not supported on non-struct types" ); } m_converter = adapters::utils::MessageStructConverterCache::instance().create( type, properties ); }; -void ClientInputAdapter::processMessage( void* c, size_t t, PushBatch* batch ) +void ClientInputAdapter::processMessage( void * c, size_t t, PushBatch * batch ) { - if( dataType() -> type() == CspType::Type::STRUCT ) + if( dataType()->type() == CspType::Type::STRUCT ) { - auto tick = m_converter -> asStruct( c, t ); - pushTick( std::move(tick), batch ); - } else if ( dataType() -> type() == CspType::Type::STRING ) + auto tick = m_converter->asStruct( c, t ); + pushTick( std::move( tick ), batch ); + } + else if( dataType()->type() == CspType::Type::STRING ) { - pushTick( std::string((char const*)c, t), batch ); + pushTick( std::string( (char const *)c, t ), batch ); } - } -} \ No newline at end of file +} // namespace csp::adapters::websocket \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientInputAdapter.h b/cpp/csp/adapters/websocket/ClientInputAdapter.h index bf3cb295f..2e2548a7a 100644 --- a/cpp/csp/adapters/websocket/ClientInputAdapter.h +++ b/cpp/csp/adapters/websocket/ClientInputAdapter.h @@ -1,32 +1,25 @@ #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_INPUTADAPTER_H #define _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_INPUTADAPTER_H -#include #include +#include #include #include namespace csp::adapters::websocket { - -class ClientInputAdapter final: public PushInputAdapter { +class ClientInputAdapter final : public PushInputAdapter +{ public: - ClientInputAdapter( - Engine * engine, - CspTypePtr & type, - PushMode pushMode, - const Dictionary & properties - ); + ClientInputAdapter( Engine * engine, CspTypePtr & type, PushMode pushMode, const Dictionary & properties ); - void processMessage( void* c, size_t t, PushBatch* batch ); + void processMessage( void * c, size_t t, PushBatch * batch ); private: adapters::utils::MessageStructConverterPtr m_converter; - }; -} - +} // namespace csp::adapters::websocket #endif // _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_INPUTADAPTER_H \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp b/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp index 3ef3c91ac..43881fede 100644 --- a/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp +++ b/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp @@ -1,17 +1,16 @@ #include -namespace csp::adapters::websocket { +namespace csp::adapters::websocket +{ -ClientOutputAdapter::ClientOutputAdapter( - Engine * engine, - WebsocketEndpoint& endpoint -) : OutputAdapter( engine ), m_endpoint( endpoint ) -{ }; +ClientOutputAdapter::ClientOutputAdapter( Engine * engine, WebsocketEndpoint & endpoint ) + : OutputAdapter( engine ) + , m_endpoint( endpoint ) {}; void ClientOutputAdapter::executeImpl() { - const std::string & value = input() -> lastValueTyped(); + const std::string & value = input()->lastValueTyped(); m_endpoint.send( value ); }; -} \ No newline at end of file +} // namespace csp::adapters::websocket \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientOutputAdapter.h b/cpp/csp/adapters/websocket/ClientOutputAdapter.h index 905822e2f..dbaec63bc 100644 --- a/cpp/csp/adapters/websocket/ClientOutputAdapter.h +++ b/cpp/csp/adapters/websocket/ClientOutputAdapter.h @@ -1,34 +1,30 @@ #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_OUTPUTADAPTER_H #define _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_OUTPUTADAPTER_H +#include #include #include #include -#include namespace csp::adapters::websocket { class ClientAdapterManager; -class ClientOutputAdapter final: public OutputAdapter +class ClientOutputAdapter final : public OutputAdapter { public: - ClientOutputAdapter( - Engine * engine, - WebsocketEndpoint& endpoint - ); + ClientOutputAdapter( Engine * engine, WebsocketEndpoint & endpoint ); void executeImpl() override; const char * name() const override { return "WebsocketClientOutputAdapter"; } private: - WebsocketEndpoint& m_endpoint; + WebsocketEndpoint & m_endpoint; }; -} - +} // namespace csp::adapters::websocket #endif \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp b/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp index f503b8e98..1154c1575 100644 --- a/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp +++ b/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp @@ -1,52 +1,49 @@ #include -namespace csp::adapters::websocket { +namespace csp::adapters::websocket +{ using namespace csp; -WebsocketEndpoint::WebsocketEndpoint( - Dictionary properties -) : m_properties(properties) -{ }; -void WebsocketEndpoint::setOnOpen(void_cb on_open) -{ m_on_open = std::move(on_open); } -void WebsocketEndpoint::setOnFail(string_cb on_fail) -{ m_on_fail = std::move(on_fail); } -void WebsocketEndpoint::setOnMessage(char_cb on_message) -{ m_on_message = std::move(on_message); } -void WebsocketEndpoint::setOnClose(void_cb on_close) -{ m_on_close = std::move(on_close); } -void WebsocketEndpoint::setOnSendFail(string_cb on_send_fail) -{ m_on_send_fail = std::move(on_send_fail); } +WebsocketEndpoint::WebsocketEndpoint( Dictionary properties ) + : m_properties( properties ) {}; +void WebsocketEndpoint::setOnOpen( void_cb on_open ) +{ + m_on_open = std::move( on_open ); +} +void WebsocketEndpoint::setOnFail( string_cb on_fail ) +{ + m_on_fail = std::move( on_fail ); +} +void WebsocketEndpoint::setOnMessage( char_cb on_message ) +{ + m_on_message = std::move( on_message ); +} +void WebsocketEndpoint::setOnClose( void_cb on_close ) +{ + m_on_close = std::move( on_close ); +} +void WebsocketEndpoint::setOnSendFail( string_cb on_send_fail ) +{ + m_on_send_fail = std::move( on_send_fail ); +} void WebsocketEndpoint::run() { m_ioc.restart(); - if(m_properties.get("use_ssl")) { - ssl::context ctx{ssl::context::sslv23}; - ctx.set_verify_mode(ssl::context::verify_peer ); + if( m_properties.get( "use_ssl" ) ) + { + ssl::context ctx{ ssl::context::sslv23 }; + ctx.set_verify_mode( ssl::context::verify_peer ); ctx.set_default_verify_paths(); - m_session = new WebsocketSessionTLS( - m_ioc, - ctx, - &m_properties, - m_on_open, - m_on_fail, - m_on_message, - m_on_close, - m_on_send_fail - ); - } else { - m_session = new WebsocketSessionNoTLS( - m_ioc, - &m_properties, - m_on_open, - m_on_fail, - m_on_message, - m_on_close, - m_on_send_fail - ); + m_session = new WebsocketSessionTLS( m_ioc, ctx, &m_properties, m_on_open, m_on_fail, m_on_message, m_on_close, + m_on_send_fail ); + } + else + { + m_session = new WebsocketSessionNoTLS( m_ioc, &m_properties, m_on_open, m_on_fail, m_on_message, m_on_close, + m_on_send_fail ); } m_session->run(); @@ -54,18 +51,21 @@ void WebsocketEndpoint::run() } void WebsocketEndpoint::stop() -{ +{ m_ioc.stop(); - if(m_session) m_session->stop(); + if( m_session ) + m_session->stop(); } - -csp::Dictionary& WebsocketEndpoint::getProperties() { +csp::Dictionary & WebsocketEndpoint::getProperties() +{ return m_properties; } -void WebsocketEndpoint::send(const std::string& s) -{ if(m_session) m_session->send(s); } - +void WebsocketEndpoint::send( const std::string & s ) +{ + if( m_session ) + m_session->send( s ); +} -} \ No newline at end of file +} // namespace csp::adapters::websocket \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/WebsocketEndpoint.h b/cpp/csp/adapters/websocket/WebsocketEndpoint.h index cfca08742..701ed6e2e 100644 --- a/cpp/csp/adapters/websocket/WebsocketEndpoint.h +++ b/cpp/csp/adapters/websocket/WebsocketEndpoint.h @@ -1,370 +1,337 @@ #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_ENDPOINT_H #define _IN_CSP_ADAPTERS_WEBSOCKETS_ENDPOINT_H +#include #include #include #include -#include -#include #include +#include #include #include #include #include -namespace csp::adapters::websocket { +namespace csp::adapters::websocket +{ using namespace csp; -namespace beast = boost::beast; // from -namespace http = beast::http; // from -namespace net = boost::asio; // from -namespace ssl = boost::asio::ssl; // from -namespace websocket = beast::websocket; // from -using tcp = boost::asio::ip::tcp; // from +namespace beast = boost::beast; // from +namespace http = beast::http; // from +namespace net = boost::asio; // from +namespace ssl = boost::asio::ssl; // from +namespace websocket = beast::websocket; // from +using tcp = boost::asio::ip::tcp; // from -using string_cb = std::function; -using char_cb = std::function; -using void_cb = std::function; +using string_cb = std::function; +using char_cb = std::function; +using void_cb = std::function; -class BaseWebsocketSession { +class BaseWebsocketSession +{ public: - virtual void stop() { }; - virtual void send( const std::string& ) { }; - virtual void do_read() { }; - virtual void do_write(const std::string& ) { }; - virtual void run() { }; + virtual void stop() {}; + virtual void send( const std::string & ) {}; + virtual void do_read() {}; + virtual void do_write( const std::string & ) {}; + virtual void run() {}; }; template -class WebsocketSession : public BaseWebsocketSession { +class WebsocketSession : public BaseWebsocketSession +{ public: - WebsocketSession( - net::io_context& ioc, - Dictionary* properties, - void_cb& on_open, - string_cb& on_fail, - char_cb& on_message, - void_cb& on_close, - string_cb& on_send_fail - ) : m_resolver( net::make_strand( ioc ) ), - m_properties( properties ), - m_on_open( on_open ), - m_on_fail( on_fail ), - m_on_message( on_message ), - m_on_close( on_close ), - m_on_send_fail( on_send_fail ) - { }; - - Derived& derived(){ return static_cast(*this); } - - void handle_message( beast::error_code& ec, std::size_t& bytes_transfered ) { - if( ec ) { + WebsocketSession( net::io_context & ioc, Dictionary * properties, void_cb & on_open, string_cb & on_fail, + char_cb & on_message, void_cb & on_close, string_cb & on_send_fail ) + : m_resolver( net::make_strand( ioc ) ) + , m_properties( properties ) + , m_on_open( on_open ) + , m_on_fail( on_fail ) + , m_on_message( on_message ) + , m_on_close( on_close ) + , m_on_send_fail( on_send_fail ) {}; + + Derived & derived() { return static_cast( *this ); } + + void handle_message( beast::error_code & ec, std::size_t & bytes_transfered ) + { + if( ec ) + { m_on_close(); return; } - m_on_message(beast::buffers_front(m_buffer.data()).data(), m_buffer.size()); - m_buffer.consume(m_buffer.size()); + m_on_message( beast::buffers_front( m_buffer.data() ).data(), m_buffer.size() ); + m_buffer.consume( m_buffer.size() ); do_read(); } - void set_headers(websocket::request_type& req) { - const csp::Dictionary &headers = *m_properties->get("headers"); + void set_headers( websocket::request_type & req ) + { + const csp::Dictionary & headers = *m_properties->get( "headers" ); for( auto it = headers.begin(); it != headers.end(); ++it ) { - const std::string key = it.key(); + const std::string key = it.key(); const std::string value = headers.get( key ); - req.set(key, value); + req.set( key, value ); } } - void do_read() override { - derived().ws().async_read( - m_buffer, - [ this ]( beast::error_code ec, std::size_t bytes_transfered ) - { handle_message( ec, bytes_transfered ); } - ); + void do_read() override + { + derived().ws().async_read( m_buffer, [this]( beast::error_code ec, std::size_t bytes_transfered ) + { handle_message( ec, bytes_transfered ); } ); } - void stop() override + void stop() override { - derived().ws().async_close( websocket::close_code::normal, [ this ]( beast::error_code ec ) { - if(ec) CSP_THROW(RuntimeException, ec.message()); - m_on_close(); - }); + derived().ws().async_close( websocket::close_code::normal, + [this]( beast::error_code ec ) + { + if( ec ) + CSP_THROW( RuntimeException, ec.message() ); + m_on_close(); + } ); } - void send( const std::string& s ) override + void send( const std::string & s ) override { - net::post( - derived().ws().get_executor(), - [this, s]() - { - m_queue.push_back(s); - if (m_queue.size() > 1) return; - do_write(m_queue.front()); - } - ); + net::post( derived().ws().get_executor(), + [this, s]() + { + m_queue.push_back( s ); + if( m_queue.size() > 1 ) + return; + do_write( m_queue.front() ); + } ); } - void do_write(const std::string& s) override + void do_write( const std::string & s ) override { - derived().ws().async_write( - net::buffer(s), - [this](beast::error_code ec, std::size_t bytes_transfered) - { - // add logging here? - m_queue.erase(m_queue.begin()); - boost::ignore_unused(bytes_transfered); - if(ec) m_on_send_fail(ec.message()); - if(m_queue.size() >0) do_write(m_queue.front()); - } - ); + derived().ws().async_write( net::buffer( s ), + [this]( beast::error_code ec, std::size_t bytes_transfered ) + { + // add logging here? + m_queue.erase( m_queue.begin() ); + boost::ignore_unused( bytes_transfered ); + if( ec ) + m_on_send_fail( ec.message() ); + if( m_queue.size() > 0 ) + do_write( m_queue.front() ); + } ); } - public: tcp::resolver m_resolver; - Dictionary* m_properties; - void_cb m_on_open; - string_cb m_on_fail; - char_cb m_on_message; - void_cb m_on_close; - string_cb m_on_send_fail; - - beast::flat_buffer m_buffer; + Dictionary * m_properties; + void_cb m_on_open; + string_cb m_on_fail; + char_cb m_on_message; + void_cb m_on_close; + string_cb m_on_send_fail; + + beast::flat_buffer m_buffer; std::vector m_queue; }; -class WebsocketSessionNoTLS final: public WebsocketSession { +class WebsocketSessionNoTLS final : public WebsocketSession +{ public: - WebsocketSessionNoTLS( - net::io_context& ioc, - Dictionary* properties, - void_cb& on_open, - string_cb& on_fail, - char_cb& on_message, - void_cb& on_close, - string_cb& on_send_fail - ) : WebsocketSession( - ioc, - properties, - on_open, - on_fail, - on_message, - on_close, - on_send_fail - ), - m_ws(net::make_strand(ioc)) - { } - - void run() override { + WebsocketSessionNoTLS( net::io_context & ioc, Dictionary * properties, void_cb & on_open, string_cb & on_fail, + char_cb & on_message, void_cb & on_close, string_cb & on_send_fail ) + : WebsocketSession( ioc, properties, on_open, on_fail, on_message, on_close, on_send_fail ) + , m_ws( net::make_strand( ioc ) ) + { + } + + void run() override + { m_resolver.async_resolve( - m_properties->get("host").c_str(), - m_properties->get("port").c_str(), - [this]( beast::error_code ec, tcp::resolver::results_type results ) { - if(ec) { - m_on_fail(ec.message()); + m_properties->get( "host" ).c_str(), m_properties->get( "port" ).c_str(), + [this]( beast::error_code ec, tcp::resolver::results_type results ) + { + if( ec ) + { + m_on_fail( ec.message() ); return; } // Set the timeout for the operation - beast::get_lowest_layer(m_ws).expires_after(std::chrono::seconds(5)); + beast::get_lowest_layer( m_ws ).expires_after( std::chrono::seconds( 5 ) ); // Make the connection on the IP address we get from a lookup - beast::get_lowest_layer(m_ws).async_connect( + beast::get_lowest_layer( m_ws ).async_connect( results, [this]( beast::error_code ec, tcp::resolver::results_type::endpoint_type ep ) { // Turn off the timeout on the tcp_stream, because // the websocket stream has its own timeout system. - if(ec) { - m_on_fail(ec.message()); + if( ec ) + { + m_on_fail( ec.message() ); return; } - beast::get_lowest_layer(m_ws).expires_never(); + beast::get_lowest_layer( m_ws ).expires_never(); - m_ws.set_option( - websocket::stream_base::timeout::suggested( - beast::role_type::client)); + m_ws.set_option( websocket::stream_base::timeout::suggested( beast::role_type::client ) ); - m_ws.set_option(websocket::stream_base::decorator( - [this](websocket::request_type& req) + m_ws.set_option( websocket::stream_base::decorator( + [this]( websocket::request_type & req ) { - set_headers(req); - req.set(http::field::user_agent, "CSP WebsocketEndpoint"); - } - )); + set_headers( req ); + req.set( http::field::user_agent, "CSP WebsocketEndpoint" ); + } ) ); - std::string host_ = m_properties->get("host") + ':' + std::to_string(ep.port()); + std::string host_ + = m_properties->get( "host" ) + ':' + std::to_string( ep.port() ); m_ws.async_handshake( - host_, - m_properties->get("route"), - [this]( beast::error_code ec ) { - if(ec) { - m_on_fail(ec.message()); + host_, m_properties->get( "route" ), + [this]( beast::error_code ec ) + { + if( ec ) + { + m_on_fail( ec.message() ); return; } m_on_open(); - m_ws.async_read( - m_buffer, - [ this ]( beast::error_code ec, std::size_t bytes_transfered ) - { handle_message( ec, bytes_transfered ); } - ); - } - ); - } - ); - } - ); + m_ws.async_read( m_buffer, [this]( beast::error_code ec, std::size_t bytes_transfered ) + { handle_message( ec, bytes_transfered ); } ); + } ); + } ); + } ); } - websocket::stream& ws() - { return m_ws; } + websocket::stream & ws() { return m_ws; } + private: websocket::stream m_ws; }; -class WebsocketSessionTLS final: public WebsocketSession { +class WebsocketSessionTLS final : public WebsocketSession +{ public: - WebsocketSessionTLS( - net::io_context& ioc, - ssl::context& ctx, - Dictionary* properties, - void_cb& on_open, - string_cb& on_fail, - char_cb& on_message, - void_cb& on_close, - string_cb& on_send_fail - ) : WebsocketSession( - ioc, - properties, - on_open, - on_fail, - on_message, - on_close, - on_send_fail - ), - m_ws(net::make_strand(ioc), ctx) - { } - - void run() override { + WebsocketSessionTLS( net::io_context & ioc, ssl::context & ctx, Dictionary * properties, void_cb & on_open, + string_cb & on_fail, char_cb & on_message, void_cb & on_close, string_cb & on_send_fail ) + : WebsocketSession( ioc, properties, on_open, on_fail, on_message, on_close, on_send_fail ) + , m_ws( net::make_strand( ioc ), ctx ) + { + } + + void run() override + { m_resolver.async_resolve( - m_properties->get("host").c_str(), - m_properties->get("port").c_str(), - [this]( beast::error_code ec, tcp::resolver::results_type results ) { - if(ec) { - m_on_fail(ec.message()); + m_properties->get( "host" ).c_str(), m_properties->get( "port" ).c_str(), + [this]( beast::error_code ec, tcp::resolver::results_type results ) + { + if( ec ) + { + m_on_fail( ec.message() ); return; } // Set the timeout for the operation - beast::get_lowest_layer(m_ws).expires_after(std::chrono::seconds(5)); + beast::get_lowest_layer( m_ws ).expires_after( std::chrono::seconds( 5 ) ); // Make the connection on the IP address we get from a lookup - beast::get_lowest_layer(m_ws).async_connect( + beast::get_lowest_layer( m_ws ).async_connect( results, [this]( beast::error_code ec, tcp::resolver::results_type::endpoint_type ep ) { - if(ec) { - m_on_fail(ec.message()); + if( ec ) + { + m_on_fail( ec.message() ); return; } - if(! SSL_set_tlsext_host_name( - m_ws.next_layer().native_handle(), - m_properties->get("host").c_str())) + if( !SSL_set_tlsext_host_name( m_ws.next_layer().native_handle(), + m_properties->get( "host" ).c_str() ) ) { - ec = beast::error_code(static_cast(::ERR_get_error()), - net::error::get_ssl_category()); - m_on_fail(ec.message()); + ec = beast::error_code( static_cast( ::ERR_get_error() ), + net::error::get_ssl_category() ); + m_on_fail( ec.message() ); return; } - m_complete_host = m_properties->get("host") + ':' + std::to_string(ep.port()); + m_complete_host = m_properties->get( "host" ) + ':' + std::to_string( ep.port() ); // ssl handler m_ws.next_layer().async_handshake( ssl::stream_base::client, - [this]( beast::error_code ec ) { - if(ec) { - m_on_fail(ec.message()); + [this]( beast::error_code ec ) + { + if( ec ) + { + m_on_fail( ec.message() ); return; } - beast::get_lowest_layer(m_ws).expires_never(); + beast::get_lowest_layer( m_ws ).expires_never(); // Set suggested timeout settings for the websocket - m_ws.set_option(websocket::stream_base::timeout::suggested(beast::role_type::client)); + m_ws.set_option( + websocket::stream_base::timeout::suggested( beast::role_type::client ) ); // Set a decorator to change the User-Agent of the handshake - m_ws.set_option(websocket::stream_base::decorator( - [this](websocket::request_type& req) + m_ws.set_option( websocket::stream_base::decorator( + [this]( websocket::request_type & req ) { - set_headers(req); - req.set(http::field::user_agent, "CSP WebsocketAdapter"); - })); - + set_headers( req ); + req.set( http::field::user_agent, "CSP WebsocketAdapter" ); + } ) ); + m_ws.async_handshake( - m_complete_host, - m_properties->get("route"), - [this]( beast::error_code ec ) { - if(ec) { - m_on_fail(ec.message()); + m_complete_host, m_properties->get( "route" ), + [this]( beast::error_code ec ) + { + if( ec ) + { + m_on_fail( ec.message() ); return; } m_on_open(); - m_ws.async_read( - m_buffer, - [ this ]( beast::error_code ec, std::size_t bytes_transfered ) - { handle_message( ec, bytes_transfered ); } - ); - } - ); - - } - ); - } - ); - } - ); + m_ws.async_read( m_buffer, + [this]( beast::error_code ec, std::size_t bytes_transfered ) + { handle_message( ec, bytes_transfered ); } ); + } ); + } ); + } ); + } ); } - websocket::stream>& ws() - { return m_ws; } + websocket::stream> & ws() { return m_ws; } private: websocket::stream> m_ws; - std::string m_complete_host; + std::string m_complete_host; }; -class WebsocketEndpoint { +class WebsocketEndpoint +{ public: WebsocketEndpoint( Dictionary properties ); - virtual ~WebsocketEndpoint() { }; - - void setOnOpen(void_cb on_open); - void setOnFail(string_cb on_fail); - void setOnMessage(char_cb on_message); - void setOnClose(void_cb on_close); - void setOnSendFail(string_cb on_send_fail); - Dictionary& getProperties(); - void run(); - void stop(); - void send(const std::string& s); + virtual ~WebsocketEndpoint() {}; + + void setOnOpen( void_cb on_open ); + void setOnFail( string_cb on_fail ); + void setOnMessage( char_cb on_message ); + void setOnClose( void_cb on_close ); + void setOnSendFail( string_cb on_send_fail ); + Dictionary & getProperties(); + void run(); + void stop(); + void send( const std::string & s ); private: - Dictionary m_properties; - BaseWebsocketSession* m_session; - net::io_context m_ioc; - void_cb m_on_open; - string_cb m_on_fail; - char_cb m_on_message; - void_cb m_on_close; - string_cb m_on_send_fail; + Dictionary m_properties; + BaseWebsocketSession * m_session; + net::io_context m_ioc; + void_cb m_on_open; + string_cb m_on_fail; + char_cb m_on_message; + void_cb m_on_close; + string_cb m_on_send_fail; }; - -} +} // namespace csp::adapters::websocket #endif \ No newline at end of file diff --git a/cpp/csp/core/BasicAllocator.h b/cpp/csp/core/BasicAllocator.h index da3aac792..636464c85 100644 --- a/cpp/csp/core/BasicAllocator.h +++ b/cpp/csp/core/BasicAllocator.h @@ -2,9 +2,9 @@ #define _IN_CSP_CORE_BASIC_ALLOCATOR_H #include +#include #include #include -#include #include #ifdef __linux__ @@ -18,8 +18,8 @@ namespace csp class BasicAllocator { public: - //elemsize is size of a single alloc, blockSize is number of elements to - //allocate per block + // elemsize is size of a single alloc, blockSize is number of elements to + // allocate per block BasicAllocator(); BasicAllocator( size_t elemSize, size_t blockSize, bool useHugePage, bool grow ); ~BasicAllocator(); @@ -48,17 +48,14 @@ class BasicAllocator void * m_freeptr; }; -inline BasicAllocator::BasicAllocator() -{} +inline BasicAllocator::BasicAllocator() {} -inline BasicAllocator::BasicAllocator( size_t elemSize, size_t blockSize, - bool useHugePage, bool grow ) +inline BasicAllocator::BasicAllocator( size_t elemSize, size_t blockSize, bool useHugePage, bool grow ) { init( elemSize, blockSize, useHugePage, grow ); } -inline void BasicAllocator::init( size_t elemSize, size_t blockSize, - bool useHugePage, bool grow ) +inline void BasicAllocator::init( size_t elemSize, size_t blockSize, bool useHugePage, bool grow ) { m_grow = grow; m_useHugePage = useHugePage; @@ -78,12 +75,12 @@ inline BasicAllocator::~BasicAllocator() else #endif ::free( entry.buffer ); - } + } } inline void BasicAllocator::allocBlock() { - //keep doubleing size in new arenas + // keep doubleing size in new arenas size_t allocSize; size_t rawsize = allocSize = m_arenas.size() ? m_arenas.back().size * 2 : m_blockSize * m_elemSize; void * buffer; @@ -92,15 +89,16 @@ inline void BasicAllocator::allocBlock() // NOTE: Only available on linux if( m_useHugePage ) { - //round to up to 2mb blocks for hugepage - constexpr size_t _2mb = 2ul * 1024 * 1024; - size_t hugesz = ( ( rawsize - 1 + _2mb ) / _2mb ) * _2mb; + // round to up to 2mb blocks for hugepage + constexpr size_t _2mb = 2ul * 1024 * 1024; + size_t hugesz = ( ( rawsize - 1 + _2mb ) / _2mb ) * _2mb; buffer = mmap( NULL, hugesz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB, -1, 0 ); - if( buffer == ( void * ) -1 ) + if( buffer == (void *)-1 ) { buffer = malloc( rawsize ); - m_arenas.push_back( ArenaInfo{ buffer, rawsize, false } );; + m_arenas.push_back( ArenaInfo{ buffer, rawsize, false } ); + ; } else { @@ -118,16 +116,16 @@ inline void BasicAllocator::allocBlock() m_arenas.push_back( ArenaInfo{ buffer, rawsize, false } ); #endif - //link blocks + // link blocks size_t numBlocks = allocSize / m_elemSize; - void * ptr = buffer; + void * ptr = buffer; for( size_t i = 0; i < numBlocks - 1; ++i ) { - void * next = ( ( uint8_t * ) ptr ) + m_elemSize; - *( uintptr_t ** )ptr = ( uintptr_t * ) next; - ptr = next; + void * next = ( (uint8_t *)ptr ) + m_elemSize; + *(uintptr_t **)ptr = (uintptr_t *)next; + ptr = next; } - *( uintptr_t ** )ptr = ( uintptr_t * ) m_freeptr; + *(uintptr_t **)ptr = (uintptr_t *)m_freeptr; m_freeptr = buffer; } @@ -137,7 +135,7 @@ inline void * BasicAllocator::allocate() if( m_freeptr != nullptr ) { void * retval = m_freeptr; - m_freeptr = *( uintptr_t ** ) m_freeptr; + m_freeptr = *(uintptr_t **)m_freeptr; return retval; } @@ -152,24 +150,27 @@ inline void * BasicAllocator::allocate() inline void BasicAllocator::free( void * ptr ) { - void * prevhead = m_freeptr; - m_freeptr = ptr; - *( uintptr_t ** )m_freeptr = ( uintptr_t * ) prevhead; + void * prevhead = m_freeptr; + m_freeptr = ptr; + *(uintptr_t **)m_freeptr = (uintptr_t *)prevhead; } -template< typename T > +template class TypedBasicAllocator : public BasicAllocator { public: TypedBasicAllocator() {} TypedBasicAllocator( size_t blockSize, bool useHugePage, bool grow ) { init( blockSize, useHugePage, grow ); } - void init( size_t blockSize, bool useHugePage, bool grow ) { BasicAllocator::init( sizeof( T ), blockSize, useHugePage, grow ); } + void init( size_t blockSize, bool useHugePage, bool grow ) + { + BasicAllocator::init( sizeof( T ), blockSize, useHugePage, grow ); + } - //note this does not construct + // note this does not construct T * allocate() { return static_cast( BasicAllocator::allocate() ); } }; -}; +}; // namespace csp #endif diff --git a/cpp/csp/core/DynamicBitSet.h b/cpp/csp/core/DynamicBitSet.h index 5fb120f06..6d5504cd2 100644 --- a/cpp/csp/core/DynamicBitSet.h +++ b/cpp/csp/core/DynamicBitSet.h @@ -13,65 +13,70 @@ class DynamicBitSet { public: - using node_type = NodeT; - using index_type = IndexT; + using node_type = NodeT; + using index_type = IndexT; static constexpr index_type npos = -1; - DynamicBitSet() : m_nodes( nullptr ), m_size( 0 ), m_numNodes( 0 ) {} + DynamicBitSet() + : m_nodes( nullptr ) + , m_size( 0 ) + , m_numNodes( 0 ) + { + } - DynamicBitSet( index_type size ) : m_size( size ) + DynamicBitSet( index_type size ) + : m_size( size ) { assert( m_size > 0 ); m_numNodes = ( m_size - 1 ) / _bits + 1; - m_nodes = new node_type[ m_numNodes ]; + m_nodes = new node_type[m_numNodes]; memset( m_nodes, 0, m_numNodes * sizeof( node_type ) ); } - DynamicBitSet( DynamicBitSet && other ) : m_size( other.m_size ), m_numNodes( other.m_numNodes ) + DynamicBitSet( DynamicBitSet && other ) + : m_size( other.m_size ) + , m_numNodes( other.m_numNodes ) { - m_nodes = other.m_nodes; + m_nodes = other.m_nodes; other.m_nodes = nullptr; } DynamicBitSet & operator=( DynamicBitSet && other ) { - m_size = other.m_size; - m_numNodes = other.m_numNodes; - m_nodes = other.m_nodes; + m_size = other.m_size; + m_numNodes = other.m_numNodes; + m_nodes = other.m_nodes; other.m_nodes = nullptr; return *this; } // Do not copy - DynamicBitSet( const DynamicBitSet & other ) = delete; + DynamicBitSet( const DynamicBitSet & other ) = delete; DynamicBitSet & operator=( const DynamicBitSet & other ) = delete; - ~DynamicBitSet() - { - delete[] m_nodes; - } + ~DynamicBitSet() { delete[] m_nodes; } void set( index_type value ) { - auto [ node, bit ] = locateValue( value ); - m_nodes[ node ] |= mask( bit ); + auto [node, bit] = locateValue( value ); + m_nodes[node] |= mask( bit ); } void reset( index_type value ) { - auto [ node, bit ] = locateValue( value ); - m_nodes[ node ] &= ~mask( bit ); + auto [node, bit] = locateValue( value ); + m_nodes[node] &= ~mask( bit ); } index_type find_first() const { for( index_type i = 0; i < m_numNodes; ++i ) { - if( m_nodes[ i ] ) - return ( i << _logBits ) + ffs( m_nodes[ i ] ) - 1; + if( m_nodes[i] ) + return ( i << _logBits ) + ffs( m_nodes[i] ) - 1; } return npos; } @@ -80,18 +85,18 @@ class DynamicBitSet { for( index_type i = m_numNodes - 1; i >= 0; --i ) { - if( m_nodes[ i ] ) - return ( i << _logBits ) + ( _bits - clz( m_nodes[ i ] ) ) - 1; + if( m_nodes[i] ) + return ( i << _logBits ) + ( _bits - clz( m_nodes[i] ) ) - 1; } return npos; } index_type find_next( index_type value ) const { - auto [ node, bit ] = locateValue( value ); + auto [node, bit] = locateValue( value ); if( likely( bit != _bits - 1 ) ) { - node_type shifted = m_nodes[ node ] >> ( bit + 1 ); + node_type shifted = m_nodes[node] >> ( bit + 1 ); if( shifted ) return value + ffs( shifted ); } @@ -99,23 +104,22 @@ class DynamicBitSet do { node++; - } - while( node < m_numNodes && !m_nodes[ node ] ); - + } while( node < m_numNodes && !m_nodes[node] ); + if( unlikely( node == m_numNodes ) ) return npos; else - bit = ffs( m_nodes[ node ] ) - 1; + bit = ffs( m_nodes[node] ) - 1; return indexValue( node, bit ); } index_type find_prev( index_type value ) const { - auto [ node, bit ] = locateValue( value ); + auto [node, bit] = locateValue( value ); if( likely( bit != 0 ) ) { - auto shifted = m_nodes[ node ] << ( _bits - bit ); + auto shifted = m_nodes[node] << ( _bits - bit ); if( shifted ) return value - ( clz( shifted ) + 1 ); } @@ -123,13 +127,12 @@ class DynamicBitSet do { node--; - } - while( node >= 0 && !( m_nodes[ node ] ) ); - + } while( node >= 0 && !( m_nodes[node] ) ); + if( unlikely( node < 0 ) ) return npos; else - bit = _bits - clz( m_nodes[ node ] ) - 1; + bit = _bits - clz( m_nodes[node] ) - 1; return indexValue( node, bit ); } @@ -140,17 +143,17 @@ class DynamicBitSet if( newNodes > m_numNodes ) { node_type * old = m_nodes; - m_nodes = new node_type[ newNodes ]; + m_nodes = new node_type[newNodes]; memcpy( m_nodes, old, m_numNodes * sizeof( node_type ) ); memset( m_nodes + m_numNodes, 0, ( newNodes - m_numNodes ) * sizeof( node_type ) ); m_numNodes = newNodes; - m_size = size; + m_size = size; delete[] old; } } - size_t size() const { return ( size_t )m_size; } + size_t size() const { return (size_t)m_size; } private: using nbit_type = uint8_t; @@ -158,39 +161,39 @@ class DynamicBitSet #ifndef WIN32 #define CLZ_CONSTEXPR constexpr #else -#define CLZ_CONSTEXPR +#define CLZ_CONSTEXPR #endif - template::value, bool> = true> - static constexpr nbit_type nbits() { return sizeof( value_type ) * 8; } + template::value, bool> = true> + static constexpr nbit_type nbits() + { + return sizeof( value_type ) * 8; + } template::value, bool> = true> - static CLZ_CONSTEXPR nbit_type log2( U n ) { return nbits() - clz(static_cast( n )) - 1; } //upcast to 32 bit to avoid truncation for log2 - static CLZ_CONSTEXPR nbit_type log2(uint64_t n) { return nbits() - clz(n) - 1; } + static CLZ_CONSTEXPR nbit_type log2( U n ) + { + return nbits() - clz( static_cast( n ) ) - 1; + } // upcast to 32 bit to avoid truncation for log2 + static CLZ_CONSTEXPR nbit_type log2( uint64_t n ) { return nbits() - clz( n ) - 1; } - static constexpr node_type mask( nbit_type bitIndex ) { return ( node_type )1 << bitIndex; } + static constexpr node_type mask( nbit_type bitIndex ) { return (node_type)1 << bitIndex; } std::pair locateValue( index_type value ) const { return std::pair( value >> _logBits, value & ( _bits - 1 ) ); } - index_type indexValue( index_type node, nbit_type bit ) const - { - return ( index_type )bit + ( node << _logBits ); - } + index_type indexValue( index_type node, nbit_type bit ) const { return (index_type)bit + ( node << _logBits ); } - static constexpr nbit_type _bits = nbits(); + static constexpr nbit_type _bits = nbits(); static inline CLZ_CONSTEXPR nbit_type _logBits = log2( _bits ); node_type * m_nodes; index_type m_size; index_type m_numNodes; - }; -} - +} // namespace csp #endif diff --git a/cpp/csp/core/Enum.h b/cpp/csp/core/Enum.h index aac11fd5d..1ec6418a9 100644 --- a/cpp/csp/core/Enum.h +++ b/cpp/csp/core/Enum.h @@ -1,21 +1,23 @@ #ifndef _IN_CSP_CORE_ENUM_H #define _IN_CSP_CORE_ENUM_H -#include +#include #include +#include #include #include +#include #include #include -#include -#include -namespace csp { +namespace csp +{ /* Example usage: -Define your enum traits struct which must define a basic enum type named _enum with UNKNOWN = 0 and NUM_TYPES members ( where NUM_TYPES is number of enum values ) -Along with an instance of said enum as a protected member m_value. You can also add any methods you like on your traits: +Define your enum traits struct which must define a basic enum type named _enum with UNKNOWN = 0 and NUM_TYPES members ( +where NUM_TYPES is number of enum values ) Along with an instance of said enum as a protected member m_value. You can +also add any methods you like on your traits: struct MyEnumTraits { @@ -44,76 +46,95 @@ using MyEnum = Enum //In a cpp file, define your string mappings like so INIT_CSP_ENUM( MyEnum, - "UNKNOWN", - "A", - "B" + "UNKNOWN", + "A", + "B" ); */ template -struct EnumFriendHelper { +struct EnumFriendHelper +{ typedef T type; }; template -struct EnumUTypeHelper { +struct EnumUTypeHelper +{ using type = typename std::underlying_type::type; }; template -struct EnumUTypeHelper { +struct EnumUTypeHelper +{ using type = EnumV; }; template -auto UnknownOnInvalidValue(int) -> decltype(T::UNKNOWN_ON_INVALID_VALUE) { return T::UNKNOWN_ON_INVALID_VALUE; } +auto UnknownOnInvalidValue( int ) -> decltype( T::UNKNOWN_ON_INVALID_VALUE ) +{ + return T::UNKNOWN_ON_INVALID_VALUE; +} template -bool UnknownOnInvalidValue(long) { return false; } +bool UnknownOnInvalidValue( long ) +{ + return false; +} START_PACKED template struct Enum : public EnumTraits { - using EnumV = typename EnumTraits::_enum; + using EnumV = typename EnumTraits::_enum; using Mapping = std::vector; using UType = typename EnumUTypeHelper::value>::type; - template::value> > - constexpr Enum( EnumV v ) { this->m_value = v; } + template::value>> + constexpr Enum( EnumV v ) + { + this->m_value = v; + } constexpr Enum() { this->m_value = EnumTraits::UNKNOWN; } - constexpr Enum( const Enum &rhs ) { this->m_value = rhs.m_value; } + constexpr Enum( const Enum & rhs ) { this->m_value = rhs.m_value; } - Enum( const char *s ) { this->m_value = reverseMap().fromString( s ); } - Enum( const std::string &s ) : Enum( s.c_str() ) {} + Enum( const char * s ) { this->m_value = reverseMap().fromString( s ); } + Enum( const std::string & s ) + : Enum( s.c_str() ) + { + } Enum( UType v ); - static const std::string &asString(EnumV v) { return mapping()[ v ]; } + static const std::string & asString( EnumV v ) { return mapping()[v]; } - const std::string &asString() const { return asString( this -> m_value ); } - const char *asCString() const { return asString( this -> m_value ).c_str(); } + const std::string & asString() const { return asString( this->m_value ); } + const char * asCString() const { return asString( this->m_value ).c_str(); } - //this pulls in all comparison operators for free since w will auto-convert to the raw enum - constexpr operator EnumV() const { return this -> m_value; } + // this pulls in all comparison operators for free since w will auto-convert to the raw enum + constexpr operator EnumV() const { return this->m_value; } - constexpr UType value() const { return this -> m_value; } + constexpr UType value() const { return this->m_value; } - //common convenience methods - bool isKnown() const { return this -> m_value != EnumTraits::UNKNOWN; } - bool isUnknown() const { return this -> m_value == EnumTraits::UNKNOWN; } + // common convenience methods + bool isKnown() const { return this->m_value != EnumTraits::UNKNOWN; } + bool isUnknown() const { return this->m_value == EnumTraits::UNKNOWN; } - struct iterator + struct iterator { - iterator( int v ) : m_v( v ) {} + iterator( int v ) + : m_v( v ) + { + } - Enum operator*() { return Enum( ( EnumV ) m_v ); } - bool operator==(const iterator &rhs) const { return m_v == rhs.m_v; } - bool operator!=(const iterator &rhs) const { return !(*this == rhs); } + Enum operator*() { return Enum( (EnumV)m_v ); } + bool operator==( const iterator & rhs ) const { return m_v == rhs.m_v; } + bool operator!=( const iterator & rhs ) const { return !( *this == rhs ); } - iterator &operator++() { + iterator & operator++() + { ++m_v; return *this; } @@ -122,103 +143,109 @@ struct Enum : public EnumTraits int m_v; }; - static constexpr size_t numTypes() { return ( size_t ) EnumTraits::NUM_TYPES; } + static constexpr size_t numTypes() { return (size_t)EnumTraits::NUM_TYPES; } - static iterator begin() { return iterator(0); } - static iterator end() { return iterator(numTypes()); } + static iterator begin() { return iterator( 0 ); } + static iterator end() { return iterator( numTypes() ); } protected: - using Aliases = std::unordered_multimap; + using Aliases = std::unordered_multimap; - struct ReverseMap : public std::unordered_map + struct ReverseMap + : public std::unordered_map { - using BaseT = std::unordered_map; + using BaseT = std::unordered_map; - ReverseMap( const Mapping &mapping ) + ReverseMap( const Mapping & mapping ) { int v = 0; - for( auto &s : mapping ) + for( auto & s : mapping ) { - assert( this -> find( s.c_str() ) == this -> end() ); - (*this)[strdup(s.c_str())] = (EnumV) v++; + assert( this->find( s.c_str() ) == this->end() ); + ( *this )[strdup( s.c_str() )] = (EnumV)v++; } } - ~ReverseMap() - { - clear(); - } + ~ReverseMap() { clear(); } - void clear() + void clear() { - for( auto &entry : *this ) + for( auto & entry : *this ) free( const_cast( entry.first ) ); BaseT::clear(); } - EnumV fromString( const char *s ) const + EnumV fromString( const char * s ) const { - auto it = this -> find( s ); - if( it == this -> end() ) + auto it = this->find( s ); + if( it == this->end() ) { if( UnknownOnInvalidValue( 0 ) ) return EnumTraits::UNKNOWN; - CSP_THROW( ValueError, "Unrecognized enum value: " << s << " for enum " << typeid( EnumTraits ).name() ); + CSP_THROW( ValueError, + "Unrecognized enum value: " << s << " for enum " << typeid( EnumTraits ).name() ); } return it->second; } }; - //This is defined by INIT macro + // This is defined by INIT macro static const Mapping & mapping(); - static const ReverseMap & reverseMap() - { + static const ReverseMap & reverseMap() + { static ReverseMap s_reverseMap( mapping() ); - return s_reverseMap; + return s_reverseMap; } } END_PACKED; template -Enum::Enum( UType v ) +Enum::Enum( UType v ) { if( v < 0 || v >= numTypes() ) { - if( UnknownOnInvalidValue(0) ) - this -> m_value = EnumTraits::UNKNOWN; + if( UnknownOnInvalidValue( 0 ) ) + this->m_value = EnumTraits::UNKNOWN; else - CSP_THROW(ValueError, "enum value: " << v << " out of range for enum " << typeid(EnumTraits).name()); - } else - this -> m_value = (EnumV) v; + CSP_THROW( ValueError, "enum value: " << v << " out of range for enum " << typeid( EnumTraits ).name() ); + } + else + this->m_value = (EnumV)v; } template -std::ostream &operator<<(std::ostream &o, Enum e) { +std::ostream & operator<<( std::ostream & o, Enum e ) +{ o << e.asString(); return o; } -}; +}; // namespace csp -//Make all Enum types hashable -namespace std { +// Make all Enum types hashable +namespace std +{ template -struct hash> { - size_t operator()(csp::Enum e) const { - return std::hash::EnumV>()(e); +struct hash> +{ + size_t operator()( csp::Enum e ) const + { + return std::hash::EnumV>()( e ); } }; -} +} // namespace std -#define INIT_CSP_ENUM(ENUM, ...) \ - template<> const ENUM::Mapping & ENUM::mapping() { \ - static ENUM::Mapping s_mapping( { __VA_ARGS__ } ); \ - return s_mapping; \ +#define INIT_CSP_ENUM( ENUM, ... ) \ + template<> \ + const ENUM::Mapping & ENUM::mapping() \ + { \ + static ENUM::Mapping s_mapping( { __VA_ARGS__ } ); \ + return s_mapping; \ } #endif diff --git a/cpp/csp/core/EnumBitSet.h b/cpp/csp/core/EnumBitSet.h index 660681029..46a24665d 100644 --- a/cpp/csp/core/EnumBitSet.h +++ b/cpp/csp/core/EnumBitSet.h @@ -1,91 +1,139 @@ #ifndef _IN_CSP_CORE_ENUMBITSET_H #define _IN_CSP_CORE_ENUMBITSET_H -#include -#include #include #include +#include +#include namespace csp { -//Utility class to hold enums as a bitmask ( where enum values are incremental from 0 ) -//enum must have a NUM_TYPES entry for number of entries -template< typename EnumT > +// Utility class to hold enums as a bitmask ( where enum values are incremental from 0 ) +// enum must have a NUM_TYPES entry for number of entries +template class EnumBitSet { using value_type = uint64_t; public: - constexpr EnumBitSet() : m_bitset( 0 ) {} - constexpr EnumBitSet( EnumT v ) : m_bitset( enumBit( v ) ) {} - constexpr EnumBitSet( const EnumBitSet & rhs ) : m_bitset( rhs.m_bitset ) {} + constexpr EnumBitSet() + : m_bitset( 0 ) + { + } + constexpr EnumBitSet( EnumT v ) + : m_bitset( enumBit( v ) ) + { + } + constexpr EnumBitSet( const EnumBitSet & rhs ) + : m_bitset( rhs.m_bitset ) + { + } constexpr EnumBitSet( std::initializer_list enums ); - void add( EnumT e ) { m_bitset |= enumBit( e ); } - void clear( EnumT e ) { m_bitset &= ~enumBit( e ); } - bool isSet( EnumT e ) const { return m_bitset & enumBit( e ); } - void reset() { m_bitset = 0; } + void add( EnumT e ) { m_bitset |= enumBit( e ); } + void clear( EnumT e ) { m_bitset &= ~enumBit( e ); } + bool isSet( EnumT e ) const { return m_bitset & enumBit( e ); } + void reset() { m_bitset = 0; } - bool empty() { return !m_bitset; } + bool empty() { return !m_bitset; } explicit operator bool() const { return m_bitset; } - size_t size() const { return __builtin_popcountl( m_bitset ); } + size_t size() const { return __builtin_popcountl( m_bitset ); } - constexpr EnumBitSet operator ~() const { return EnumBitSet( ~m_bitset & ( ( ( (value_type) 1u ) << int( EnumT::NUM_TYPES ) ) - 1 ) ); } - constexpr EnumBitSet operator |( typename EnumT::EnumV v ) const { return EnumBitSet( m_bitset | enumBit( v ) ); } - constexpr EnumBitSet operator |( EnumT e ) const { return EnumBitSet( m_bitset | enumBit( e ) ); } - constexpr EnumBitSet operator |( const EnumBitSet & rhs ) const { return EnumBitSet( m_bitset | rhs.m_bitset ); } - constexpr EnumBitSet operator &( typename EnumT::EnumV v ) const { return EnumBitSet( m_bitset & enumBit( v ) ); } - constexpr EnumBitSet operator &( EnumT e ) const { return EnumBitSet( m_bitset & enumBit( e ) ); } - constexpr EnumBitSet operator &( const EnumBitSet & rhs ) const { return EnumBitSet( m_bitset & rhs.m_bitset ); } - - EnumBitSet & operator |=( typename EnumT::EnumV v ) { m_bitset |= enumBit( v ); return *this; } - EnumBitSet & operator |=( EnumT e ) { m_bitset |= enumBit( e ); return *this; } - EnumBitSet & operator |=( const EnumBitSet & rhs ) { m_bitset |= rhs.m_bitset; return *this; } - EnumBitSet & operator &=( const EnumBitSet & rhs ) { m_bitset &= rhs.m_bitset; return *this; } + constexpr EnumBitSet operator~() const + { + return EnumBitSet( ~m_bitset & ( ( ( (value_type)1u ) << int( EnumT::NUM_TYPES ) ) - 1 ) ); + } + constexpr EnumBitSet operator|( typename EnumT::EnumV v ) const { return EnumBitSet( m_bitset | enumBit( v ) ); } + constexpr EnumBitSet operator|( EnumT e ) const { return EnumBitSet( m_bitset | enumBit( e ) ); } + constexpr EnumBitSet operator|( const EnumBitSet & rhs ) const { return EnumBitSet( m_bitset | rhs.m_bitset ); } + constexpr EnumBitSet operator&( typename EnumT::EnumV v ) const { return EnumBitSet( m_bitset & enumBit( v ) ); } + constexpr EnumBitSet operator&( EnumT e ) const { return EnumBitSet( m_bitset & enumBit( e ) ); } + constexpr EnumBitSet operator&( const EnumBitSet & rhs ) const { return EnumBitSet( m_bitset & rhs.m_bitset ); } + + EnumBitSet & operator|=( typename EnumT::EnumV v ) + { + m_bitset |= enumBit( v ); + return *this; + } + EnumBitSet & operator|=( EnumT e ) + { + m_bitset |= enumBit( e ); + return *this; + } + EnumBitSet & operator|=( const EnumBitSet & rhs ) + { + m_bitset |= rhs.m_bitset; + return *this; + } + EnumBitSet & operator&=( const EnumBitSet & rhs ) + { + m_bitset &= rhs.m_bitset; + return *this; + } - bool operator==( const EnumBitSet & rhs ) const { return m_bitset == rhs.m_bitset; } - bool operator!=( const EnumBitSet & rhs ) const { return m_bitset != rhs.m_bitset; } + bool operator==( const EnumBitSet & rhs ) const { return m_bitset == rhs.m_bitset; } + bool operator!=( const EnumBitSet & rhs ) const { return m_bitset != rhs.m_bitset; } struct iterator { - iterator( value_type bitset ) : m_bitset( bitset ) {} - - EnumT operator*() const { return ( EnumT ) __builtin_ctzl( m_bitset ); } //returns lowest order bit - iterator operator++() { m_bitset &= ( m_bitset - 1 ); return *this; } //clear out least significant bit - iterator operator++( int ) { iterator it = *this; m_bitset &= ( m_bitset - 1 ); return it; } + iterator( value_type bitset ) + : m_bitset( bitset ) + { + } + + EnumT operator*() const { return (EnumT)__builtin_ctzl( m_bitset ); } // returns lowest order bit + iterator operator++() + { + m_bitset &= ( m_bitset - 1 ); + return *this; + } // clear out least significant bit + iterator operator++( int ) + { + iterator it = *this; + m_bitset &= ( m_bitset - 1 ); + return it; + } bool operator==( iterator rhs ) { return m_bitset == rhs.m_bitset; } - bool operator!=( iterator rhs ) { return !(*this == rhs ); } + bool operator!=( iterator rhs ) { return !( *this == rhs ); } private: value_type m_bitset; }; iterator begin() const { return iterator( m_bitset ); } - iterator end() const { return iterator( 0 ); } + iterator end() const { return iterator( 0 ); } value_type rawBitSet() const { return m_bitset; } void setRawBitSet( value_type bitset ) { m_bitset = bitset; } private: - constexpr EnumBitSet( value_type v ) : m_bitset( v ) {} - static constexpr value_type enumBit( typename EnumT::EnumV e ) { return static_cast( 1 ) << ( unsigned char ) e; } + constexpr EnumBitSet( value_type v ) + : m_bitset( v ) + { + } + static constexpr value_type enumBit( typename EnumT::EnumV e ) + { + return static_cast( 1 ) << (unsigned char)e; + } - static_assert( ( 1ul << ( int( EnumT::NUM_TYPES ) - 1 ) ) < std::numeric_limits::max(), "Too many enums for EnumBitSet to hold" ); + static_assert( ( 1ul << ( int( EnumT::NUM_TYPES ) - 1 ) ) < std::numeric_limits::max(), + "Too many enums for EnumBitSet to hold" ); value_type m_bitset; }; -template< typename EnumT > -inline constexpr EnumBitSet::EnumBitSet( std::initializer_list enums ) : m_bitset( 0 ) +template +inline constexpr EnumBitSet::EnumBitSet( std::initializer_list enums ) + : m_bitset( 0 ) { for( auto e : enums ) m_bitset |= enumBit( e ); } -} +} // namespace csp #endif diff --git a/cpp/csp/core/Exception.cpp b/cpp/csp/core/Exception.cpp index e4480754b..9bfe16205 100644 --- a/cpp/csp/core/Exception.cpp +++ b/cpp/csp/core/Exception.cpp @@ -4,22 +4,22 @@ #include #include -#include #include -#include +#include #include +#include #ifndef WIN32 #include #endif -//From https://stackoverflow.com/questions/2443135/how-do-i-find-where-an-exception-was-thrown-in-c/2443366 +// From https://stackoverflow.com/questions/2443135/how-do-i-find-where-an-exception-was-thrown-in-c/2443366 static void csp_terminate( void ); static bool set_sigabrt_handler(); // invoke set_terminate as part of global constant initialization // This for uncaught exceptions -static const bool SET_TERMINATE = std::set_terminate( csp_terminate ); +static const bool SET_TERMINATE = std::set_terminate( csp_terminate ); static const bool SET_SIGABRT_HANDLER = set_sigabrt_handler(); static void printBacktrace( char ** messages, int size, std::ostream & dest ) @@ -33,32 +33,31 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) for( int i = 0; i < size; ++i ) { char *begin_name = 0, *begin_offset = 0; - char tmp[1024]; - strncpy( tmp, messages[i], sizeof(tmp) ); - tmp[ sizeof( tmp ) - 1 ] = 0; + char tmp[1024]; + strncpy( tmp, messages[i], sizeof( tmp ) ); + tmp[sizeof( tmp ) - 1] = 0; // find parentheses and +address offset surrounding the mangled name: // ./module(function+0x15c) [0x8048a6d] - for (char *p = tmp; *p; ++p) + for( char * p = tmp; *p; ++p ) { - if (*p == '(') + if( *p == '(' ) begin_name = p; - else if (*p == '+') + else if( *p == '+' ) { begin_offset = p; break; } } - if (begin_name && begin_offset - && begin_name < begin_offset) + if( begin_name && begin_offset && begin_name < begin_offset ) { begin_name++; *begin_offset = '\0'; #ifndef WIN32 - int status; - char* demangled = abi::__cxa_demangle(begin_name, NULL, NULL, &status); + int status; + char * demangled = abi::__cxa_demangle( begin_name, NULL, NULL, &status ); dest << "[bt]: (" << i << ") " << ( status == 0 ? demangled : messages[i] ) << std::endl; free( demangled ); #else @@ -67,7 +66,7 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) } else { - dest << "[bt]: (" << i << ") " << messages[i] << std::endl; + dest << "[bt]: (" << i << ") " << messages[i] << std::endl; } } @@ -76,11 +75,11 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) void printBacktrace() { -//TODO get stack traces on windows +// TODO get stack traces on windows #ifndef WIN32 - void *array[50]; - int size = backtrace( array, 50 ); - auto messages = backtrace_symbols( array, size ); + void * array[50]; + int size = backtrace( array, 50 ); + auto messages = backtrace_symbols( array, size ); printBacktrace( messages, size, std::cerr ); free( messages ); #endif @@ -90,28 +89,25 @@ void csp_terminate() { static int tried_throw = 0; - try + try { // try once to re-throw currently active exception - if( !tried_throw++ ) + if( !tried_throw++ ) throw; } catch( const csp::Exception & ex ) { - std::cerr << __FUNCTION__ << " caught unhandled csp::Exception. what(): " - << ex.what() << std::endl; + std::cerr << __FUNCTION__ << " caught unhandled csp::Exception. what(): " << ex.what() << std::endl; if( ex.btsize() > 0 ) printBacktrace( ex.btmessages(), ex.btsize(), std::cerr ); } catch( const std::exception & e ) { - std::cerr << __FUNCTION__ << " caught unhandled std::exception. what(): " - << e.what() << std::endl; + std::cerr << __FUNCTION__ << " caught unhandled std::exception. what(): " << e.what() << std::endl; } - catch( ... ) + catch( ... ) { - std::cerr << __FUNCTION__ << " caught unknown/unhandled exception." - << std::endl; + std::cerr << __FUNCTION__ << " caught unknown/unhandled exception." << std::endl; } printBacktrace(); @@ -122,19 +118,18 @@ void csp_terminate() abort(); } -//This is for coredumps +// This is for coredumps #ifndef WIN32 -void sigabrt_handler(int sig_num, siginfo_t* info, void* ctx) +void sigabrt_handler( int sig_num, siginfo_t * info, void * ctx ) { - std::cerr << "signal " << sig_num - << " (" << strsignal(sig_num) << "), address is " - << info -> si_addr << " from " << std::endl; + std::cerr << "signal " << sig_num << " (" << strsignal( sig_num ) << "), address is " << info->si_addr << " from " + << std::endl; printBacktrace(); - signal(SIGABRT, SIG_DFL); - signal(SIGSEGV, SIG_DFL); - signal(SIGBUS, SIG_DFL); + signal( SIGABRT, SIG_DFL ); + signal( SIGSEGV, SIG_DFL ); + signal( SIGBUS, SIG_DFL ); abort(); } @@ -144,28 +139,28 @@ bool set_sigabrt_handler() static struct sigaction sigact; sigact.sa_sigaction = sigabrt_handler; - sigact.sa_flags = SA_RESTART | SA_SIGINFO; + sigact.sa_flags = SA_RESTART | SA_SIGINFO; - sigaction(SIGABRT, &sigact, NULL); - sigaction(SIGSEGV, &sigact, NULL); - sigaction(SIGBUS, &sigact, NULL); + sigaction( SIGABRT, &sigact, NULL ); + sigaction( SIGSEGV, &sigact, NULL ); + sigaction( SIGBUS, &sigact, NULL ); return true; } #else -void sigabrt_handler(int sig_num) +void sigabrt_handler( int sig_num ) { std::cerr << "signal " << sig_num << " from " << std::endl; printBacktrace(); - signal(SIGABRT, SIG_DFL); - signal(SIGSEGV, SIG_DFL); + signal( SIGABRT, SIG_DFL ); + signal( SIGSEGV, SIG_DFL ); abort(); } bool set_sigabrt_handler() { - signal(SIGABRT, sigabrt_handler); - signal(SIGSEGV, sigabrt_handler); + signal( SIGABRT, sigabrt_handler ); + signal( SIGSEGV, sigabrt_handler ); return true; } #endif @@ -173,50 +168,51 @@ bool set_sigabrt_handler() void csp::Exception::setbt() { #ifndef WIN32 - void *array[50]; - m_backtracesize = backtrace( array, 50 ); - char **messages = backtrace_symbols( array, m_backtracesize ); + void * array[50]; + m_backtracesize = backtrace( array, 50 ); + char ** messages = backtrace_symbols( array, m_backtracesize ); m_backtracemessages = messages; #endif } -static char ** dupe_backtraces( char** bt, int n ) +static char ** dupe_backtraces( char ** bt, int n ) { - if( bt == nullptr ) return nullptr; + if( bt == nullptr ) + return nullptr; size_t len = n * sizeof( char * ); for( int i = 0; i < n; ++i ) - len += strlen( bt[ i ] ) + 1; - char ** newbt = (char **) malloc( len ); + len += strlen( bt[i] ) + 1; + char ** newbt = (char **)malloc( len ); memcpy( newbt, bt, len ); for( int i = 0; i < n; ++i ) - newbt[i] = (char *)( newbt + ( ( (char **) bt[ i ] ) - bt ) ); + newbt[i] = (char *)( newbt + ( ( (char **)bt[i] ) - bt ) ); return newbt; } -csp::Exception::Exception( const csp::Exception &orig ) : - m_full( orig.m_full ), - m_exType( orig.m_exType ), - m_description( orig.m_description ), - m_file( orig.m_file ), - m_function( orig.m_function ), - m_line( orig.m_line ), - m_backtracesize( orig.m_backtracesize ), - m_backtracemessages( dupe_backtraces( orig.m_backtracemessages, orig.m_backtracesize ) ) +csp::Exception::Exception( const csp::Exception & orig ) + : m_full( orig.m_full ) + , m_exType( orig.m_exType ) + , m_description( orig.m_description ) + , m_file( orig.m_file ) + , m_function( orig.m_function ) + , m_line( orig.m_line ) + , m_backtracesize( orig.m_backtracesize ) + , m_backtracemessages( dupe_backtraces( orig.m_backtracemessages, orig.m_backtracesize ) ) { } -csp::Exception::Exception( csp::Exception && donor ) : - m_full( std::move( donor.m_full ) ), - m_exType( std::move( donor.m_exType ) ), - m_description( std::move( donor.m_description ) ), - m_file( std::move( donor.m_file ) ), - m_function( std::move( donor.m_function ) ), - m_line( donor.m_line ), - m_backtracesize( donor.m_backtracesize ), - m_backtracemessages( donor.m_backtracemessages ) +csp::Exception::Exception( csp::Exception && donor ) + : m_full( std::move( donor.m_full ) ) + , m_exType( std::move( donor.m_exType ) ) + , m_description( std::move( donor.m_description ) ) + , m_file( std::move( donor.m_file ) ) + , m_function( std::move( donor.m_function ) ) + , m_line( donor.m_line ) + , m_backtracesize( donor.m_backtracesize ) + , m_backtracemessages( donor.m_backtracemessages ) { donor.m_backtracemessages = nullptr; } diff --git a/cpp/csp/core/Exception.h b/cpp/csp/core/Exception.h index 074143de7..2518fe53d 100644 --- a/cpp/csp/core/Exception.h +++ b/cpp/csp/core/Exception.h @@ -1,11 +1,11 @@ #ifndef _IN_CSP_CORE_EXCEPTION_H #define _IN_CSP_CORE_EXCEPTION_H +#include +#include #include #include #include -#include -#include namespace csp { @@ -13,19 +13,26 @@ namespace csp class Exception : public std::exception { public: - Exception( const char * exType, const std::string & description, const char * file, const char * func, int line ) : - m_exType( exType ), m_description( description ), m_file( file ), m_function( func ), m_line( line ), m_backtracemessages( nullptr ) - { + Exception( const char * exType, const std::string & description, const char * file, const char * func, int line ) + : m_exType( exType ) + , m_description( description ) + , m_file( file ) + , m_function( func ) + , m_line( line ) + , m_backtracemessages( nullptr ) + { setbt(); } - Exception(const char* exType, const std::string& description) : Exception(exType, description, "", "", -1) - {} + Exception( const char * exType, const std::string & description ) + : Exception( exType, description, "", "", -1 ) + { + } ~Exception() { free( m_backtracemessages ); } Exception( const Exception & ); - Exception( Exception&& ); + Exception( Exception && ); - const char * what() const noexcept override { return full( false ).c_str(); } + const char * what() const noexcept override { return full( false ).c_str(); } const std::string & full( bool includeBacktrace ) const noexcept { m_full.clear(); @@ -36,13 +43,14 @@ class Exception : public std::exception m_full += '\n' + backtraceString(); return m_full; } - const std::string & description() const noexcept { return m_description; } + const std::string & description() const noexcept { return m_description; } char ** btmessages() const { return m_backtracemessages; } - int btsize() const { return m_backtracesize; } + int btsize() const { return m_backtracesize; } std::string backtraceString() const; - void writeBacktrace( std::ostream & ) const; // streams are generally not-copyable (e.g. std::cerr) so we need this overload + void writeBacktrace( + std::ostream & ) const; // streams are generally not-copyable (e.g. std::cerr) so we need this overload void writeBacktrace( std::ostream && dest ) const { writeBacktrace( dest ); } private: @@ -58,49 +66,79 @@ class Exception : public std::exception char ** m_backtracemessages; }; -#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) -#define CSP_DECLARE_EXCEPTION( DerivedException, BaseException ) class DerivedException : public BaseException { public: DerivedException( const char * exType, const std::string &r, const char * file, const char * func, int line ) : BaseException( exType, r, file, func, line ) {} }; - -CSP_DECLARE_EXCEPTION( AssertionError, Exception ) -CSP_DECLARE_EXCEPTION( RuntimeException, Exception ) -CSP_DECLARE_EXCEPTION( InvalidArgument, RuntimeException ) -CSP_DECLARE_EXCEPTION( NotImplemented, RuntimeException ) -CSP_DECLARE_EXCEPTION( ValueError, RuntimeException ) -CSP_DECLARE_EXCEPTION( KeyError, RuntimeException ) -CSP_DECLARE_EXCEPTION( TypeError, RuntimeException ) -CSP_DECLARE_EXCEPTION( RangeError, RuntimeException ) -CSP_DECLARE_EXCEPTION( OverflowError, RuntimeException ) -CSP_DECLARE_EXCEPTION( DivideByZero, RuntimeException ) -CSP_DECLARE_EXCEPTION( RecursionError, RuntimeException ) -CSP_DECLARE_EXCEPTION( IOError, RuntimeException ) -CSP_DECLARE_EXCEPTION( OSError, RuntimeException ) -CSP_DECLARE_EXCEPTION( OutOfMemoryError, RuntimeException ) -CSP_DECLARE_EXCEPTION( FileNotFoundError, IOError ) +#define __FILENAME__ ( strrchr( __FILE__, '/' ) ? strrchr( __FILE__, '/' ) + 1 : __FILE__ ) +#define CSP_DECLARE_EXCEPTION( DerivedException, BaseException ) \ + class DerivedException : public BaseException \ + { \ + public: \ + DerivedException( const char * exType, const std::string & r, const char * file, const char * func, int line ) \ + : BaseException( exType, r, file, func, line ) \ + { \ + } \ + }; + +CSP_DECLARE_EXCEPTION( AssertionError, Exception ) +CSP_DECLARE_EXCEPTION( RuntimeException, Exception ) +CSP_DECLARE_EXCEPTION( InvalidArgument, RuntimeException ) +CSP_DECLARE_EXCEPTION( NotImplemented, RuntimeException ) +CSP_DECLARE_EXCEPTION( ValueError, RuntimeException ) +CSP_DECLARE_EXCEPTION( KeyError, RuntimeException ) +CSP_DECLARE_EXCEPTION( TypeError, RuntimeException ) +CSP_DECLARE_EXCEPTION( RangeError, RuntimeException ) +CSP_DECLARE_EXCEPTION( OverflowError, RuntimeException ) +CSP_DECLARE_EXCEPTION( DivideByZero, RuntimeException ) +CSP_DECLARE_EXCEPTION( RecursionError, RuntimeException ) +CSP_DECLARE_EXCEPTION( IOError, RuntimeException ) +CSP_DECLARE_EXCEPTION( OSError, RuntimeException ) +CSP_DECLARE_EXCEPTION( OutOfMemoryError, RuntimeException ) +CSP_DECLARE_EXCEPTION( FileNotFoundError, IOError ) template -[[noreturn]] NO_INLINE void throw_exc(T&& e); +[[noreturn]] NO_INLINE void throw_exc( T && e ); template -[[noreturn]] inline void throw_exc(T&& e) {throw e;} - -#define CSP_THROW( EX_TYPE, MSG ) do { std::stringstream desc; desc << MSG ; csp::throw_exc(EX_TYPE( #EX_TYPE, desc.str(), __FILENAME__ , __FUNCTION__ , __LINE__ )); } while( 0 ) -#define CSP_THROW_EX( EX_TYPE, MSG, ... ) do { std::stringstream desc; desc << MSG ; csp::throw_exc(EX_TYPE( #EX_TYPE, desc.str(), __FILENAME__ , __FUNCTION__ , __LINE__ , __VA_ARGS__ )); } while( 0 ) - -#define CSP_TRUE_OR_THROW( EXPR, EXCEPTION_TYPE, MESSAGE ) do {if( unlikely(!((EXPR))) ) { CSP_THROW( EXCEPTION_TYPE, MESSAGE ); }} while(false) -#define CSP_TRUE_OR_THROW_RUNTIME(EXPR, MESSAGE) CSP_TRUE_OR_THROW(EXPR, csp::RuntimeException, MESSAGE) -#define CSP_NOT_IMPLEMENTED CSP_THROW(csp::NotImplemented, ""); - -// This is kind of the same as CSP_ASSERT, the difference is that whatever will be validated with this check, will be checked in both debug and release builds. -// Generally CSP_TRUE_OR_THROW_RUNTIME is better since usually the error message should be customized. -#define CSP_ENSURE_TRUE(EXPR) CSP_TRUE_OR_THROW_RUNTIME(EXPR, #EXPR) +[[noreturn]] inline void throw_exc( T && e ) +{ + throw e; +} -#ifdef NDEBUG -#define CSP_ASSERT( EXPR ) ( (void ) (0)) +#define CSP_THROW( EX_TYPE, MSG ) \ + do \ + { \ + std::stringstream desc; \ + desc << MSG; \ + csp::throw_exc( EX_TYPE( #EX_TYPE, desc.str(), __FILENAME__, __FUNCTION__, __LINE__ ) ); \ + } while( 0 ) +#define CSP_THROW_EX( EX_TYPE, MSG, ... ) \ + do \ + { \ + std::stringstream desc; \ + desc << MSG; \ + csp::throw_exc( EX_TYPE( #EX_TYPE, desc.str(), __FILENAME__, __FUNCTION__, __LINE__, __VA_ARGS__ ) ); \ + } while( 0 ) + +#define CSP_TRUE_OR_THROW( EXPR, EXCEPTION_TYPE, MESSAGE ) \ + do \ + { \ + if( unlikely( !( ( EXPR ) ) ) ) \ + { \ + CSP_THROW( EXCEPTION_TYPE, MESSAGE ); \ + } \ + } while( false ) +#define CSP_TRUE_OR_THROW_RUNTIME( EXPR, MESSAGE ) CSP_TRUE_OR_THROW( EXPR, csp::RuntimeException, MESSAGE ) +#define CSP_NOT_IMPLEMENTED CSP_THROW( csp::NotImplemented, "" ); + +// This is kind of the same as CSP_ASSERT, the difference is that whatever will be validated with this check, will be +// checked in both debug and release builds. Generally CSP_TRUE_OR_THROW_RUNTIME is better since usually the error +// message should be customized. +#define CSP_ENSURE_TRUE( EXPR ) CSP_TRUE_OR_THROW_RUNTIME( EXPR, #EXPR ) + +#ifdef NDEBUG +#define CSP_ASSERT( EXPR ) ( (void)( 0 ) ) #else -#define CSP_ASSERT( EXPR ) CSP_TRUE_OR_THROW(EXPR, csp::AssertionError, #EXPR) +#define CSP_ASSERT( EXPR ) CSP_TRUE_OR_THROW( EXPR, csp::AssertionError, #EXPR ) #endif - -} +} // namespace csp #endif diff --git a/cpp/csp/core/FileUtils.h b/cpp/csp/core/FileUtils.h index a4fa9e6ca..0a972d211 100644 --- a/cpp/csp/core/FileUtils.h +++ b/cpp/csp/core/FileUtils.h @@ -8,25 +8,26 @@ namespace csp::utils inline std::string dirname( const std::string & path ) { - return std::filesystem::path(path).parent_path().string(); + return std::filesystem::path( path ).parent_path().string(); } -//files and directories are treated equally +// files and directories are treated equally inline bool fileExists( const std::string & fileOrDir ) { - return std::filesystem::exists(fileOrDir); + return std::filesystem::exists( fileOrDir ); } -inline void mkdir( const std::string & path, - std::filesystem::perms perms = std::filesystem::perms::owner_all | std::filesystem::perms::group_all | std::filesystem::perms::others_read | std::filesystem::perms::others_exec ) +inline void mkdir( const std::string & path, + std::filesystem::perms perms = std::filesystem::perms::owner_all | std::filesystem::perms::group_all + | std::filesystem::perms::others_read | std::filesystem::perms::others_exec ) { - if (!fileExists(path)) + if( !fileExists( path ) ) { std::error_code err; - if (!std::filesystem::create_directories(path, err)) - CSP_THROW(IOError, "Failed to create path " << path << ": " << err.message()); - std::filesystem::permissions(path, perms); + if( !std::filesystem::create_directories( path, err ) ) + CSP_THROW( IOError, "Failed to create path " << path << ": " << err.message() ); + std::filesystem::permissions( path, perms ); } } -} +} // namespace csp::utils diff --git a/cpp/csp/core/Generator.h b/cpp/csp/core/Generator.h index 8f33eaddf..61a2c4ae9 100644 --- a/cpp/csp/core/Generator.h +++ b/cpp/csp/core/Generator.h @@ -5,7 +5,7 @@ namespace csp { -template< typename V, typename ...Args > +template class Generator { public: @@ -17,11 +17,11 @@ class Generator virtual void init( Args... ) = 0; // Called to get the next value. Return true if value retrieved, false otherwise - virtual bool next( V &value ) = 0; + virtual bool next( V & value ) = 0; }; -template< typename V, typename ...Args > +template using GeneratorPtr = std::shared_ptr>; -} +} // namespace csp #endif diff --git a/cpp/csp/core/Hash.h b/cpp/csp/core/Hash.h index 4352dcab5..a76205924 100644 --- a/cpp/csp/core/Hash.h +++ b/cpp/csp/core/Hash.h @@ -9,44 +9,41 @@ namespace csp::hash { -//C-string hash helpers +// C-string hash helpers struct CStrHash { std::size_t operator()( const char * s ) const noexcept { - //Unabashedly stolen from Python 2.7 string hash - const unsigned char * p = (const unsigned char *) s; - std::size_t x = *p << 7; + // Unabashedly stolen from Python 2.7 string hash + const unsigned char * p = (const unsigned char *)s; + std::size_t x = *p << 7; while( *p ) - x = (1000003*x) ^ *p++; - + x = ( 1000003 * x ) ^ *p++; + return x; } }; struct CStrEq { - bool operator()( const char * lhs, const char * rhs ) const noexcept - { - return strcmp( lhs, rhs ) == 0; - } + bool operator()( const char * lhs, const char * rhs ) const noexcept { return strcmp( lhs, rhs ) == 0; } }; inline size_t hash_bytes( const void * data, size_t len ) { - return std::hash{}( std::string_view( ( const char * ) data, len ) ); + return std::hash{}( std::string_view( (const char *)data, len ) ); } -//Convenient hash of pair so it can be used as a key in unordered_map -struct hash_pair -{ - template +// Convenient hash of pair so it can be used as a key in unordered_map +struct hash_pair +{ + template size_t operator()( const std::pair & p ) const - { - return std::hash{}(p.first) ^ std::hash{}(p.second); - } + { + return std::hash{}( p.first ) ^ std::hash{}( p.second ); + } }; -} +} // namespace csp::hash #endif diff --git a/cpp/csp/core/Likely.h b/cpp/csp/core/Likely.h index 74829c5d0..a61b0f102 100644 --- a/cpp/csp/core/Likely.h +++ b/cpp/csp/core/Likely.h @@ -3,11 +3,11 @@ // We should move to [[likely]] [[unlikely]] attributes once we enable c++20 #ifndef WIN32 -#define likely(x) __builtin_expect ( (x), 1 ) -#define unlikely(x) __builtin_expect ( (x), 0 ) +#define likely( x ) __builtin_expect( ( x ), 1 ) +#define unlikely( x ) __builtin_expect( ( x ), 0 ) #else -#define likely(x) x -#define unlikely(x) x +#define likely( x ) x +#define unlikely( x ) x #endif #endif diff --git a/cpp/csp/core/Platform.h b/cpp/csp/core/Platform.h index 37474faf6..8f69d44ed 100644 --- a/cpp/csp/core/Platform.h +++ b/cpp/csp/core/Platform.h @@ -1,15 +1,15 @@ #ifndef _IN_CSP_CORE_PLATFORM_H #define _IN_CSP_CORE_PLATFORM_H -#include #include #include +#include -//TODO move Likely.h defines into Platform.h +// TODO move Likely.h defines into Platform.h #ifdef WIN32 #define NOMINMAX -#include #include #include +#include #undef ERROR #undef GetMessage @@ -17,26 +17,26 @@ #define DLL_LOCAL #ifdef CSPTYPESIMPL_EXPORTS -#define CSPTYPESIMPL_EXPORT __declspec(dllexport) +#define CSPTYPESIMPL_EXPORT __declspec( dllexport ) #else -#define CSPTYPESIMPL_EXPORT __declspec(dllimport) +#define CSPTYPESIMPL_EXPORT __declspec( dllimport ) #endif #ifdef CSPIMPL_EXPORTS -#define CSPIMPL_EXPORT __declspec(dllexport) +#define CSPIMPL_EXPORT __declspec( dllexport ) #else -#define CSPIMPL_EXPORT __declspec(dllimport) +#define CSPIMPL_EXPORT __declspec( dllimport ) #endif -#define START_PACKED __pragma( pack(push, 1) ) -#define END_PACKED __pragma( pack(pop)) +#define START_PACKED __pragma( pack( push, 1 ) ) +#define END_PACKED __pragma( pack( pop ) ) -#define NO_INLINE __declspec(noinline) +#define NO_INLINE __declspec( noinline ) inline tm * localtime_r( const time_t * timep, tm * result ) { - tm * rv = localtime(timep); - if (rv) + tm * rv = localtime( timep ); + if( rv ) *result = *rv; return result; @@ -44,47 +44,53 @@ inline tm * localtime_r( const time_t * timep, tm * result ) #define timegm _mkgmtime -inline int nanosleep(const timespec* req, timespec* rem) +inline int nanosleep( const timespec * req, timespec * rem ) { - assert(rem == nullptr); + assert( rem == nullptr ); int64_t millis = req->tv_sec * 1000 + req->tv_nsec * 1000000; - Sleep(millis); + Sleep( millis ); return 0; } -inline uint8_t clz(uint64_t n) +inline uint8_t clz( uint64_t n ) { unsigned long index = 0; - if (_BitScanReverse64(&index, n)) - return 64 - index - 1; + if( _BitScanReverse64( &index, n ) ) + return 64 - index - 1; return 0; } -inline uint8_t clz(uint32_t n) +inline uint8_t clz( uint32_t n ) { unsigned long index = 0; - if (_BitScanReverse(&index, n)) - return 32 - index - 1; + if( _BitScanReverse( &index, n ) ) + return 32 - index - 1; return 0; } -inline uint8_t clz(uint16_t n) { return clz(static_cast(n)) - 16; } -inline uint8_t clz(uint8_t n) { return clz(static_cast(n)) - 24; } +inline uint8_t clz( uint16_t n ) +{ + return clz( static_cast( n ) ) - 16; +} +inline uint8_t clz( uint8_t n ) +{ + return clz( static_cast( n ) ) - 24; +} template::value, bool> = true> -inline uint8_t ffs(U n) -{ +inline uint8_t ffs( U n ) +{ unsigned long index = 0; - if (_BitScanForward(&index, n)) - return index + 1; + if( _BitScanForward( &index, n ) ) + return index + 1; return 0; } -inline uint8_t ffs(uint64_t n) +inline uint8_t ffs( uint64_t n ) { unsigned long index = 0; - if (_BitScanForward64(&index, n)) - return index + 1; + if( _BitScanForward64( &index, n ) ) + return index + 1; return 0; } @@ -93,25 +99,43 @@ inline uint8_t ffs(uint64_t n) #define CSPIMPL_EXPORT #define CSPTYPESIMPL_EXPORT -#define DLL_LOCAL __attribute__ ((visibility ("hidden"))) +#define DLL_LOCAL __attribute__( ( visibility( "hidden" ) ) ) #define START_PACKED -#define END_PACKED __attribute__((packed)) +#define END_PACKED __attribute__( ( packed ) ) -#define NO_INLINE __attribute__ ((noinline)) +#define NO_INLINE __attribute__( ( noinline ) ) -inline constexpr uint8_t clz(uint32_t n) { return __builtin_clz(n); } -inline constexpr uint8_t clz(uint64_t n) { return __builtin_clzl(n); } +inline constexpr uint8_t clz( uint32_t n ) +{ + return __builtin_clz( n ); +} +inline constexpr uint8_t clz( uint64_t n ) +{ + return __builtin_clzl( n ); +} // clz (count leading zeros) returns number of leading zeros before MSB (i.e. clz(00110..) = 2 ) // __builtin_clz auto-promotes to 32-bits: need to subtract off extra leading zeros -inline constexpr uint8_t clz(uint16_t n) { return clz(static_cast(n)) - 16; } -inline constexpr uint8_t clz(uint8_t n) { return clz(static_cast(n)) - 24; } +inline constexpr uint8_t clz( uint16_t n ) +{ + return clz( static_cast( n ) ) - 16; +} +inline constexpr uint8_t clz( uint8_t n ) +{ + return clz( static_cast( n ) ) - 24; +} // ffs (find first set) returns offset of first set bit (i.e. ffs(..0110) = 2 ), with ffs(0) = 0 template::value, bool> = true> -inline constexpr uint8_t ffs( U n ) { return __builtin_ffs(n); } -inline constexpr uint8_t ffs( uint64_t n ) { return __builtin_ffsl(n); } +inline constexpr uint8_t ffs( U n ) +{ + return __builtin_ffs( n ); +} +inline constexpr uint8_t ffs( uint64_t n ) +{ + return __builtin_ffsl( n ); +} #endif diff --git a/cpp/csp/core/QueueWaiter.h b/cpp/csp/core/QueueWaiter.h index 307930cd5..3ce516bbe 100644 --- a/cpp/csp/core/QueueWaiter.h +++ b/cpp/csp/core/QueueWaiter.h @@ -1,10 +1,10 @@ #ifndef _IN_CSP_CORE_QUEUEBLOCKINGWAIT_H #define _IN_CSP_CORE_QUEUEBLOCKINGWAIT_H -#include #include -#include #include +#include +#include namespace csp { @@ -14,8 +14,10 @@ class TimeDelta; class QueueWaiter { public: - QueueWaiter() : m_eventsPending( false ) - {} + QueueWaiter() + : m_eventsPending( false ) + { + } void notify() { @@ -23,15 +25,15 @@ class QueueWaiter if( !m_eventsPending ) m_condition.notify_one(); m_eventsPending = true; - } bool wait( TimeDelta maxWaitTime ) { std::unique_lock lock( m_lock ); - bool rv = false; + bool rv = false; if( !m_eventsPending && maxWaitTime.asNanoseconds() > 0 ) - rv = m_condition.wait_for( lock, std::chrono::nanoseconds( maxWaitTime.asNanoseconds() ), [this]() { return m_eventsPending; } ); + rv = m_condition.wait_for( lock, std::chrono::nanoseconds( maxWaitTime.asNanoseconds() ), + [this]() { return m_eventsPending; } ); if( rv ) m_eventsPending = false; @@ -44,6 +46,6 @@ class QueueWaiter bool m_eventsPending; }; -} +} // namespace csp #endif diff --git a/cpp/csp/core/SRMWLockFreeQueue.h b/cpp/csp/core/SRMWLockFreeQueue.h index 902c8a364..e2407dd5e 100644 --- a/cpp/csp/core/SRMWLockFreeQueue.h +++ b/cpp/csp/core/SRMWLockFreeQueue.h @@ -1,43 +1,44 @@ #ifndef _IN_CSP_CORE_SRMWLOCKFREEQUEUE_H #define _IN_CSP_CORE_SRMWLOCKFREEQUEUE_H +#include #include #include #include -#include namespace csp { /* - The SRMWLockFreeQueue is safe for multipe writers / single reader access. Likely not the most efficient impl, but certainly - one of the simplest! template type is required to have an intrinsic next pointer + The SRMWLockFreeQueue is safe for multipe writers / single reader access. Likely not the most efficient impl, but + certainly one of the simplest! template type is required to have an intrinsic next pointer */ -template< typename T > -class alignas(CACHELINE_SIZE) SRMWLockFreeQueue +template +class alignas( CACHELINE_SIZE ) SRMWLockFreeQueue { public: class Batch { public: - Batch() : m_head( nullptr ), m_tail( nullptr ) {} + Batch() + : m_head( nullptr ) + , m_tail( nullptr ) + { + } void append( T * event ) { - //note we link the batch events in reverse order so that we can do a quick CAS switcheroo + // note we link the batch events in reverse order so that we can do a quick CAS switcheroo if( !m_head ) m_head = m_tail = event; else { - event -> next = m_head; - m_head = event; + event->next = m_head; + m_head = event; } } - void clear() - { - m_head = m_tail = nullptr; - } + void clear() { m_head = m_tail = nullptr; } bool empty() const { return m_head == nullptr; } @@ -51,82 +52,88 @@ class alignas(CACHELINE_SIZE) SRMWLockFreeQueue friend class SRMWLockFreeQueue; }; - SRMWLockFreeQueue( bool blocking = false ) : m_head( nullptr ), - m_wait( blocking ? new QueueWaiter : nullptr ), m_curItems( nullptr ) {} + SRMWLockFreeQueue( bool blocking = false ) + : m_head( nullptr ) + , m_wait( blocking ? new QueueWaiter : nullptr ) + , m_curItems( nullptr ) + { + } ~SRMWLockFreeQueue() { delete( m_wait ); } bool empty() const { return m_head == nullptr && m_curItems == nullptr; } void push( T * ); - //atomic push, batch will be cleared after this call - void push( Batch & batch ); + // atomic push, batch will be cleared after this call + void push( Batch & batch ); bool wait( TimeDelta maxWait ); - //pop calls can return NULL if empty - //pop a single item + // pop calls can return NULL if empty + // pop a single item T * pop( TimeDelta maxWait = TimeDelta() ); T * peek(); - //pop all pending items, need to iterate over T -> next + // pop all pending items, need to iterate over T -> next T * popAll( TimeDelta maxWait = TimeDelta() ); private: - std::atomic m_head; - QueueWaiter * m_wait; - alignas(CACHELINE_SIZE) T * m_curItems; + std::atomic m_head; + QueueWaiter * m_wait; + alignas( CACHELINE_SIZE ) T * m_curItems; }; -template< typename T > +template inline void SRMWLockFreeQueue::push( T * item ) { - //ABA problem? Not sure, even if head pointer did go through an ABA cycle - //it seems like the queue will still stay stable - item -> next = m_head.load( std::memory_order_relaxed ); - while( !m_head.compare_exchange_weak( item -> next, item, std::memory_order_release ) ) - {} + // ABA problem? Not sure, even if head pointer did go through an ABA cycle + // it seems like the queue will still stay stable + item->next = m_head.load( std::memory_order_relaxed ); + while( !m_head.compare_exchange_weak( item->next, item, std::memory_order_release ) ) + { + } if( unlikely( m_wait != nullptr ) ) - m_wait -> notify(); + m_wait->notify(); } -template< typename T > +template inline void SRMWLockFreeQueue::push( Batch & batch ) { - //Batch already gaurantees events are in the correct reverse ordering - //so we just set m_head to batch head and link batch tail to previous head - batch.m_tail -> next = m_head.load( std::memory_order_relaxed ); - while( !m_head.compare_exchange_weak( batch.m_tail -> next, batch.m_head, std::memory_order_release ) ) - {} + // Batch already gaurantees events are in the correct reverse ordering + // so we just set m_head to batch head and link batch tail to previous head + batch.m_tail->next = m_head.load( std::memory_order_relaxed ); + while( !m_head.compare_exchange_weak( batch.m_tail->next, batch.m_head, std::memory_order_release ) ) + { + } batch.clear(); if( unlikely( m_wait != nullptr ) ) - m_wait -> notify(); + m_wait->notify(); } -template< typename T > +template inline T * SRMWLockFreeQueue::pop( TimeDelta maxWait ) { - //current impl has push creating a linked list in reverse order - //we want to maintian a single popEvent interface for future impls, so - //we popAll into m_curItems and work off of that as long its available + // current impl has push creating a linked list in reverse order + // we want to maintian a single popEvent interface for future impls, so + // we popAll into m_curItems and work off of that as long its available - //single reader thread here + // single reader thread here if( !m_curItems ) m_curItems = popAll( maxWait ); if( m_curItems ) { - T * ret = m_curItems; - m_curItems = m_curItems -> next; + T * ret = m_curItems; + m_curItems = m_curItems->next; return ret; } return nullptr; } -template< typename T > +template inline T * SRMWLockFreeQueue::peek() { if( !m_curItems ) @@ -134,34 +141,34 @@ inline T * SRMWLockFreeQueue::peek() return m_curItems; } -template< typename T > +template inline bool SRMWLockFreeQueue::wait( TimeDelta maxWait ) { if( m_wait == nullptr || maxWait.asNanoseconds() <= 0 ) return false; - return m_wait -> wait( maxWait ); + return m_wait->wait( maxWait ); } -template< typename T > +template inline T * SRMWLockFreeQueue::popAll( TimeDelta maxWait ) { if( unlikely( m_wait != nullptr && maxWait.asNanoseconds() > 0 && m_head == nullptr ) ) - m_wait -> wait( maxWait ); + m_wait->wait( maxWait ); T * head = m_head.exchange( nullptr ); T * prev = nullptr; while( head ) { - T * next = head -> next; - head -> next = prev; - prev = head; - head = next; + T * next = head->next; + head->next = prev; + prev = head; + head = next; } - return prev; + return prev; } -} +} // namespace csp #endif diff --git a/cpp/csp/core/System.h b/cpp/csp/core/System.h index 90f56f5aa..4cefaa7d7 100644 --- a/cpp/csp/core/System.h +++ b/cpp/csp/core/System.h @@ -1,11 +1,11 @@ #ifndef _IN_CSP_CORE_SYSTEM_H #define _IN_CSP_CORE_SYSTEM_H -//Common low level system methods / defines -#include +// Common low level system methods / defines +#include #include #include -#include +#include #ifndef WIN32 #include @@ -16,14 +16,14 @@ namespace csp static constexpr size_t CACHELINE_SIZE = 64; -//useful for logging type information +// useful for logging type information template std::string cpp_type_name() { - int status = 0; - std::string result = typeid(*(T*)nullptr).name(); + int status = 0; + std::string result = typeid( *(T *)nullptr ).name(); #ifndef WIN32 - char * demangled = abi::__cxa_demangle(result.c_str(), NULL, NULL, &status); + char * demangled = abi::__cxa_demangle( result.c_str(), NULL, NULL, &status ); if( demangled ) { result = demangled; @@ -33,6 +33,6 @@ std::string cpp_type_name() return result; } -} +} // namespace csp #endif diff --git a/cpp/csp/core/TaggedPointerUnion.h b/cpp/csp/core/TaggedPointerUnion.h index 43a41a85f..fcc13a465 100644 --- a/cpp/csp/core/TaggedPointerUnion.h +++ b/cpp/csp/core/TaggedPointerUnion.h @@ -6,82 +6,86 @@ namespace csp { -template struct TypeList {}; +template +struct TypeList +{ +}; template -struct IndexOf { }; +struct IndexOf +{ +}; -template +template struct IndexOf> { static constexpr std::size_t value = 0; }; -template -struct IndexOf > +template +struct IndexOf> { static constexpr std::size_t value = IndexOf>::value + 1; }; - template class TaggedPointerUnion { public: - static inline constexpr size_t NUM_TAGS = sizeof...(Ts); - //we can be more efficient if needed for more types..., we can store an integeger value rather - //than a bit per type. but current use cases we only need 2 types so not bothering + static inline constexpr size_t NUM_TAGS = sizeof...( Ts ); + // we can be more efficient if needed for more types..., we can store an integeger value rather + // than a bit per type. but current use cases we only need 2 types so not bothering static inline constexpr size_t TAG_BITS = NUM_TAGS; static inline constexpr size_t TAG_MASK = ( 1UL << TAG_BITS ) - 1; - TaggedPointerUnion() - { - reset(); - } + TaggedPointerUnion() { reset(); } void * raw() const { return m_ptr; } - - void * unmasked() const { return ( void * ) ( ( ( uint64_t ) m_ptr ) & ~TAG_MASK ); } + + void * unmasked() const { return (void *)( ( (uint64_t)m_ptr ) & ~TAG_MASK ); } void reset() { m_ptr = nullptr; } operator bool() const { return m_ptr != nullptr; } - template + template void set( T * p ) { m_ptr = p; setMask(); } - //note that this will NOT do error checking, on caller to check isSet as needed - template + // note that this will NOT do error checking, on caller to check isSet as needed + template T * get() const { - return ( T * ) unmasked(); + return (T *)unmasked(); } template bool isSet() const { constexpr size_t bitmask = 1 << typeBit(); - return ( ( ( uint64_t ) m_ptr ) & bitmask ) == bitmask; + return ( ( (uint64_t)m_ptr ) & bitmask ) == bitmask; } - template< typename T > - static constexpr size_t typeBit() { return IndexOf>::value; } + template + static constexpr size_t typeBit() + { + return IndexOf>::value; + } private: - template< typename T > + template void setMask() - { + { constexpr size_t bitmask = 1 << typeBit(); - m_ptr = ( void * ) ( ( ( uint64_t ) m_ptr ) | bitmask ); + m_ptr = (void *)( ( (uint64_t)m_ptr ) | bitmask ); } void * m_ptr; }; -} +} // namespace csp #endif diff --git a/cpp/csp/core/Time.cpp b/cpp/csp/core/Time.cpp index cb7055cad..348647aaf 100644 --- a/cpp/csp/core/Time.cpp +++ b/cpp/csp/core/Time.cpp @@ -4,31 +4,28 @@ namespace csp { // XXXXXX -//This Code was taken from glibc time/offtime.c, the reason being that gmtime_r actually calls the global tzlock +// This Code was taken from glibc time/offtime.c, the reason being that gmtime_r actually calls the global tzlock //(even though technically we dont need tz here ). This makes it lockless -static const unsigned short int __mon_yday[2][13] = - { +static const unsigned short int __mon_yday[2][13] = { /* Normal years. */ { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }, /* Leap years. */ - { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } - }; - + { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } }; tm DateTime::asTM() const { - tm TM{0}; - time_t days, rem, y; - const unsigned short int *ip; + tm TM{ 0 }; + time_t days, rem, y; + const unsigned short int * ip; days = m_ticks / NANOS_PER_DAY; - rem = m_ticks % NANOS_PER_DAY; - while (rem < 0) + rem = m_ticks % NANOS_PER_DAY; + while( rem < 0 ) { rem += NANOS_PER_DAY; --days; } - while (rem >= NANOS_PER_DAY) + while( rem >= NANOS_PER_DAY ) { rem -= NANOS_PER_DAY; ++days; @@ -38,40 +35,37 @@ tm DateTime::asTM() const TM.tm_min = rem / ( 60 * NANOS_PER_SECOND ); TM.tm_sec = ( rem % ( 60 * NANOS_PER_SECOND ) ) / NANOS_PER_SECOND; /* January 1, 1970 was a Thursday. */ - TM.tm_wday = (4 + days) % 7; - if (TM.tm_wday < 0) + TM.tm_wday = ( 4 + days ) % 7; + if( TM.tm_wday < 0 ) TM.tm_wday += 7; y = 1970; -#define DIV(a, b) ((a) / (b) - ((a) % (b) < 0)) -#define LEAPS_THRU_END_OF(y) (DIV (y, 4) - DIV (y, 100) + DIV (y, 400)) -#define __isleap(year) \ - ((year) % 4 == 0 && ((year) % 100 != 0 || (year) % 400 == 0)) +#define DIV( a, b ) ( ( a ) / ( b ) - ( ( a ) % ( b ) < 0 ) ) +#define LEAPS_THRU_END_OF( y ) ( DIV( y, 4 ) - DIV( y, 100 ) + DIV( y, 400 ) ) +#define __isleap( year ) ( ( year ) % 4 == 0 && ( ( year ) % 100 != 0 || ( year ) % 400 == 0 ) ) - while (days < 0 || days >= (__isleap(y) ? 366 : 365)) + while( days < 0 || days >= ( __isleap( y ) ? 366 : 365 ) ) { /* Guess a corrected year, assuming 365 days per year. */ - time_t yg = y + days / 365 - (days % 365 < 0); + time_t yg = y + days / 365 - ( days % 365 < 0 ); /* Adjust DAYS and Y to match the guessed year. */ - days -= ((yg - y) * 365 - + LEAPS_THRU_END_OF (yg - 1) - - LEAPS_THRU_END_OF (y - 1)); + days -= ( ( yg - y ) * 365 + LEAPS_THRU_END_OF( yg - 1 ) - LEAPS_THRU_END_OF( y - 1 ) ); y = yg; } TM.tm_year = y - 1900; - if (TM.tm_year != y - 1900) + if( TM.tm_year != y - 1900 ) CSP_THROW( RuntimeException, "Failed to convert DateTime to struct tm: year overflow" ); TM.tm_yday = days; - ip = __mon_yday[__isleap(y)]; - for (y = 11; days < (long int) ip[y]; --y) + ip = __mon_yday[__isleap( y )]; + for( y = 11; days < (long int)ip[y]; --y ) continue; days -= ip[y]; - TM.tm_mon = y; + TM.tm_mon = y; TM.tm_mday = days + 1; return TM; } -//XXXX +// XXXX -}; +}; // namespace csp diff --git a/cpp/csp/core/Time.h b/cpp/csp/core/Time.h index 07a2247ac..358b7b10a 100644 --- a/cpp/csp/core/Time.h +++ b/cpp/csp/core/Time.h @@ -4,14 +4,14 @@ #include #include #include +#include #include #include #include #include #include -#include -#include #include +#include namespace csp { @@ -24,81 +24,118 @@ const int64_t NANOS_PER_DAY = NANOS_PER_SECOND * SECONDS_PER_DAY; class TimeDelta { public: - constexpr TimeDelta() : TimeDelta( TimeDelta::NONE() ) {} - constexpr TimeDelta( int64_t seconds, int64_t nanoseconds ) : TimeDelta( seconds * NANOS_PER_SECOND + nanoseconds ) - {} + constexpr TimeDelta() + : TimeDelta( TimeDelta::NONE() ) + { + } + constexpr TimeDelta( int64_t seconds, int64_t nanoseconds ) + : TimeDelta( seconds * NANOS_PER_SECOND + nanoseconds ) + { + } - int64_t asNanoseconds() const { return m_ticks; } + int64_t asNanoseconds() const { return m_ticks; } int64_t asMicroseconds() const { return m_ticks / NANOS_PER_MICROSECOND; } int64_t asMilliseconds() const { return m_ticks / NANOS_PER_MILLISECOND; } - int64_t asSeconds() const { return m_ticks / NANOS_PER_SECOND; } - - int32_t days() const { return asSeconds() / SECONDS_PER_DAY; } - int32_t hours() const { return ( asSeconds() % SECONDS_PER_DAY ) / 3600; } - int32_t minutes() const { return ( asSeconds() % 3600 ) / 60; } - int32_t seconds() const { return asSeconds() % 60; } - - int32_t nanoseconds() const { return asNanoseconds() % NANOS_PER_SECOND; } - - TimeDelta abs() const { return TimeDelta( std::abs( m_ticks ) ); } - int sign() const { - if(m_ticks < 0) { + int64_t asSeconds() const { return m_ticks / NANOS_PER_SECOND; } + + int32_t days() const { return asSeconds() / SECONDS_PER_DAY; } + int32_t hours() const { return ( asSeconds() % SECONDS_PER_DAY ) / 3600; } + int32_t minutes() const { return ( asSeconds() % 3600 ) / 60; } + int32_t seconds() const { return asSeconds() % 60; } + + int32_t nanoseconds() const { return asNanoseconds() % NANOS_PER_SECOND; } + + TimeDelta abs() const { return TimeDelta( std::abs( m_ticks ) ); } + int sign() const + { + if( m_ticks < 0 ) + { return -1; - } else if (m_ticks > 0) { + } + else if( m_ticks > 0 ) + { return 1; } return 0; } - bool isNone() const { return (*this) == TimeDelta::NONE(); } + bool isNone() const { return ( *this ) == TimeDelta::NONE(); } std::string asString() const; - - //from XX units - static constexpr TimeDelta fromNanoseconds( int64_t nanos ) { return TimeDelta( nanos ); } - static constexpr TimeDelta fromMicroseconds( int64_t micros ) { return fromNanoseconds( micros * NANOS_PER_MICROSECOND ); } - static constexpr TimeDelta fromMilliseconds( int64_t millis ) { return fromNanoseconds( millis * NANOS_PER_MILLISECOND ); } - static constexpr TimeDelta fromSeconds( int64_t seconds ) { return fromNanoseconds( seconds * NANOS_PER_SECOND ); } - static constexpr TimeDelta fromMinutes( int64_t minutes ) { return fromSeconds( minutes * 60 ); } - static constexpr TimeDelta fromHours( int64_t hours ) { return fromSeconds( hours * 3600 ); } - static constexpr TimeDelta fromDays( int64_t days ) { return fromSeconds( days * SECONDS_PER_DAY ); } - - //HH:MM:SS.nn format, see Time::fromString + + // from XX units + static constexpr TimeDelta fromNanoseconds( int64_t nanos ) { return TimeDelta( nanos ); } + static constexpr TimeDelta fromMicroseconds( int64_t micros ) + { + return fromNanoseconds( micros * NANOS_PER_MICROSECOND ); + } + static constexpr TimeDelta fromMilliseconds( int64_t millis ) + { + return fromNanoseconds( millis * NANOS_PER_MILLISECOND ); + } + static constexpr TimeDelta fromSeconds( int64_t seconds ) { return fromNanoseconds( seconds * NANOS_PER_SECOND ); } + static constexpr TimeDelta fromMinutes( int64_t minutes ) { return fromSeconds( minutes * 60 ); } + static constexpr TimeDelta fromHours( int64_t hours ) { return fromSeconds( hours * 3600 ); } + static constexpr TimeDelta fromDays( int64_t days ) { return fromSeconds( days * SECONDS_PER_DAY ); } + + // HH:MM:SS.nn format, see Time::fromString static TimeDelta fromString( const std::string & str ); bool operator==( const TimeDelta & rhs ) const { return m_ticks == rhs.m_ticks; } - bool operator!=( const TimeDelta & rhs ) const { return !( (*this) == rhs ); } + bool operator!=( const TimeDelta & rhs ) const { return !( ( *this ) == rhs ); } - bool operator< ( const TimeDelta & rhs ) const { return m_ticks < rhs.m_ticks; } - bool operator<=( const TimeDelta & rhs ) const { return m_ticks <= rhs.m_ticks; } - bool operator> ( const TimeDelta & rhs ) const { return m_ticks > rhs.m_ticks; } - bool operator>=( const TimeDelta & rhs ) const { return m_ticks >= rhs.m_ticks; } + bool operator<( const TimeDelta & rhs ) const { return m_ticks < rhs.m_ticks; } + bool operator<=( const TimeDelta & rhs ) const { return m_ticks <= rhs.m_ticks; } + bool operator>( const TimeDelta & rhs ) const { return m_ticks > rhs.m_ticks; } + bool operator>=( const TimeDelta & rhs ) const { return m_ticks >= rhs.m_ticks; } - TimeDelta operator +( const TimeDelta & rhs ) const { return TimeDelta( m_ticks + rhs.m_ticks ); } - TimeDelta operator -( const TimeDelta & rhs ) const { return TimeDelta( m_ticks - rhs.m_ticks ); } - TimeDelta & operator+=( const TimeDelta & rhs ) { m_ticks += rhs.m_ticks; return *this; } - TimeDelta & operator-=( const TimeDelta & rhs ) { m_ticks -= rhs.m_ticks; return *this; } + TimeDelta operator+( const TimeDelta & rhs ) const { return TimeDelta( m_ticks + rhs.m_ticks ); } + TimeDelta operator-( const TimeDelta & rhs ) const { return TimeDelta( m_ticks - rhs.m_ticks ); } + TimeDelta & operator+=( const TimeDelta & rhs ) + { + m_ticks += rhs.m_ticks; + return *this; + } + TimeDelta & operator-=( const TimeDelta & rhs ) + { + m_ticks -= rhs.m_ticks; + return *this; + } - TimeDelta operator*( int mult ) const { return TimeDelta( m_ticks * mult ); } - TimeDelta & operator*=( int mult ) { m_ticks *= mult; return *this; } + TimeDelta operator*( int mult ) const { return TimeDelta( m_ticks * mult ); } + TimeDelta & operator*=( int mult ) + { + m_ticks *= mult; + return *this; + } - TimeDelta operator/( int div ) const { return TimeDelta( m_ticks / div ); } - TimeDelta & operator/=( int div ) { m_ticks /= div; return *this; } + TimeDelta operator/( int div ) const { return TimeDelta( m_ticks / div ); } + TimeDelta & operator/=( int div ) + { + m_ticks /= div; + return *this; + } int64_t operator/( TimeDelta div ) const { return m_ticks / div.m_ticks; } - TimeDelta operator-() const { return TimeDelta( -m_ticks ); } + TimeDelta operator-() const { return TimeDelta( -m_ticks ); } - static constexpr TimeDelta ZERO() { return TimeDelta( 0 ); } - static constexpr TimeDelta NONE() { return TimeDelta( std::numeric_limits::min() ); } - static constexpr TimeDelta MIN_VALUE() { return TimeDelta( std::numeric_limits::min() + 1 ); } //min reserved for NONE + static constexpr TimeDelta ZERO() { return TimeDelta( 0 ); } + static constexpr TimeDelta NONE() { return TimeDelta( std::numeric_limits::min() ); } + static constexpr TimeDelta MIN_VALUE() + { + return TimeDelta( std::numeric_limits::min() + 1 ); + } // min reserved for NONE static constexpr TimeDelta MAX_VALUE() { return TimeDelta( std::numeric_limits::max() ); } private: - //the fact that we store this as nanos is an implementation detail - constexpr TimeDelta( int64_t raw_nanos ) : m_ticks( raw_nanos ) {} + // the fact that we store this as nanos is an implementation detail + constexpr TimeDelta( int64_t raw_nanos ) + : m_ticks( raw_nanos ) + { + } int64_t m_ticks; -}; +}; inline std::string TimeDelta::asString() const { @@ -119,19 +156,19 @@ inline std::string TimeDelta::asString() const inline TimeDelta TimeDelta::fromString( const std::string & str ) { - //can do S, as seconds - //M:S for minutes:seconds - //H:M:S for hours:minutes:seconds - //all version can take .nnn for fractions of a second - int h = 0; - int m = 0; - int s = 0; - int v1 = 0; - int v2 = 0; - int v3 = 0; - int ns = 0; + // can do S, as seconds + // M:S for minutes:seconds + // H:M:S for hours:minutes:seconds + // all version can take .nnn for fractions of a second + int h = 0; + int m = 0; + int s = 0; + int v1 = 0; + int v2 = 0; + int v3 = 0; + int ns = 0; const char * c_str = str.c_str(); - int n = sscanf( c_str, "%d:%d:%d", &v1, &v2, &v3 ); + int n = sscanf( c_str, "%d:%d:%d", &v1, &v2, &v3 ); if( n == 0 ) CSP_THROW( ValueError, "Failed to convert " << str << " to TimeDelta" ); @@ -148,18 +185,18 @@ inline TimeDelta TimeDelta::fromString( const std::string & str ) m = v2; h = v1; } - - char * frac = strrchr( ( char * ) c_str, '.' ); + + char * frac = strrchr( (char *)c_str, '.' ); if( frac ) { - ns = atoi( frac + 1 ); + ns = atoi( frac + 1 ); int len = strlen( frac + 1 ); ns *= pow( 10, 9 - len ); } return TimeDelta( NANOS_PER_SECOND * ( h * 3600 + m * 60 + s ) + ns ); } -inline std::ostream & operator <<( std::ostream &os, const TimeDelta & d ) +inline std::ostream & operator<<( std::ostream & os, const TimeDelta & d ) { os << d.asString(); return os; @@ -168,38 +205,49 @@ inline std::ostream & operator <<( std::ostream &os, const TimeDelta & d ) class Date { public: - Date() : Date( NONE() ) {} + Date() + : Date( NONE() ) + { + } Date( int16_t year, int8_t month, int8_t day ); - int16_t year() const { return repr().year; } + int16_t year() const { return repr().year; } int8_t month() const { return repr().month; } - int8_t day() const { return repr().day; } + int8_t day() const { return repr().day; } - bool isNone() const { return (*this) == Date::NONE(); } + bool isNone() const { return ( *this ) == Date::NONE(); } - size_t strftime( char *result, size_t max_size, const char *fmt ) const; + size_t strftime( char * result, size_t max_size, const char * fmt ) const; bool operator==( const Date & rhs ) const { return m_data.value == rhs.m_data.value; } - bool operator!=( const Date & rhs ) const { return !( (*this) == rhs ); } - - bool operator< ( const Date & rhs ) const { return m_data.value < rhs.m_data.value; } - bool operator<=( const Date & rhs ) const { return m_data.value <= rhs.m_data.value; } - bool operator> ( const Date & rhs ) const { return m_data.value > rhs.m_data.value; } - bool operator>=( const Date & rhs ) const { return m_data.value >= rhs.m_data.value; } - - Date operator-( const TimeDelta & delta ) const; - Date operator+( const TimeDelta & delta ) const; + bool operator!=( const Date & rhs ) const { return !( ( *this ) == rhs ); } + + bool operator<( const Date & rhs ) const { return m_data.value < rhs.m_data.value; } + bool operator<=( const Date & rhs ) const { return m_data.value <= rhs.m_data.value; } + bool operator>( const Date & rhs ) const { return m_data.value > rhs.m_data.value; } + bool operator>=( const Date & rhs ) const { return m_data.value >= rhs.m_data.value; } + + Date operator-( const TimeDelta & delta ) const; + Date operator+( const TimeDelta & delta ) const; TimeDelta operator-( const Date & rhs ) const; - Date & operator-=( const TimeDelta & delta ) { *this = *this - delta; return *this; } - Date & operator+=( const TimeDelta & delta ) { *this = *this + delta; return *this; } + Date & operator-=( const TimeDelta & delta ) + { + *this = *this - delta; + return *this; + } + Date & operator+=( const TimeDelta & delta ) + { + *this = *this + delta; + return *this; + } - //day of week, 0 = Sunday, 6 = Saturday - int weekday() const { return asTM().tm_wday; } - bool isWeekday() const { return !isWeekend(); } + // day of week, 0 = Sunday, 6 = Saturday + int weekday() const { return asTM().tm_wday; } + bool isWeekday() const { return !isWeekend(); } bool isWeekend() const; - std::string asString() const { return asYYYYMMDD(); } + std::string asString() const { return asYYYYMMDD(); } std::string asYYYYMMDD() const; size_t hash() const { return std::hash()( m_data.value ); } @@ -210,12 +258,15 @@ class Date static Date NONE() { return Date( -1, -1, -1 ); } private: - Date( const tm & TM ) : Date( TM.tm_year + 1900, TM.tm_mon + 1, TM.tm_mday ) {} + Date( const tm & TM ) + : Date( TM.tm_year + 1900, TM.tm_mon + 1, TM.tm_mday ) + { + } tm asTM( bool do_mktime = true ) const; - //ordering here is important! year is placed at the high order bits - //for correct comparisons against m_value + // ordering here is important! year is placed at the high order bits + // for correct comparisons against m_value struct _repr { int8_t day; @@ -232,9 +283,9 @@ class Date }; const _repr & repr() const { return m_data.repr; } - _repr & repr() { return m_data.repr; } + _repr & repr() { return m_data.repr; } - data m_data; //first 16 bits = year, next 8 = month, next 8 = day + data m_data; // first 16 bits = year, next 8 = month, next 8 = day }; inline Date::Date( int16_t year, int8_t month, int8_t day ) @@ -246,10 +297,10 @@ inline Date::Date( int16_t year, int8_t month, int8_t day ) inline tm Date::asTM( bool do_mktime ) const { - tm TM{0}; + tm TM{ 0 }; TM.tm_year = year() - 1900; TM.tm_mon = month() - 1; - TM.tm_mday = day(); + TM.tm_mday = day(); TM.tm_isdst = -1; if( do_mktime ) mktime( &TM ); @@ -278,7 +329,7 @@ inline bool Date::isWeekend() const return d == 0 || d == 6; } -inline size_t Date::strftime( char *result, size_t max_size, const char *fmt ) const +inline size_t Date::strftime( char * result, size_t max_size, const char * fmt ) const { tm time = asTM(); return ::strftime( result, max_size, fmt, &time ); @@ -297,20 +348,20 @@ inline Date Date::fromYYYYMMDD( const std::string & date ) inline Date Date::today() { - tm TM; + tm TM; time_t t = time( NULL ); localtime_r( &t, &TM ); return Date( TM ); } -inline std::string Date::asYYYYMMDD() const +inline std::string Date::asYYYYMMDD() const { char buf[32]; sprintf( buf, "%04d%02d%02d", year(), month(), day() ); return buf; } -inline std::ostream & operator <<( std::ostream &os, const Date & d ) +inline std::ostream & operator<<( std::ostream & os, const Date & d ) { os << d.asString(); return os; @@ -319,55 +370,58 @@ inline std::ostream & operator <<( std::ostream &os, const Date & d ) class Time { public: - Time() : Time( -1 ) {} //NONE + Time() + : Time( -1 ) + { + } // NONE Time( int hour, int minute, int second, int32_t nanosecond = 0 ); - int hour() const { return asSeconds() / 3600; } - int minute() const { return ( asSeconds() % 3600 ) / 60; } - int second() const { return asSeconds() % 60; } + int hour() const { return asSeconds() / 3600; } + int minute() const { return ( asSeconds() % 3600 ) / 60; } + int second() const { return asSeconds() % 60; } int32_t nanosecond() const { return m_ticks % NANOS_PER_SECOND; } - bool isNone() const { return (*this) == Time::NONE(); } + bool isNone() const { return ( *this ) == Time::NONE(); } - //from XX since midnight - static Time fromNanoseconds( int64_t nanos ) { return Time( nanos ); } + // from XX since midnight + static Time fromNanoseconds( int64_t nanos ) { return Time( nanos ); } static Time fromMicroseconds( int64_t micros ) { return fromNanoseconds( micros * NANOS_PER_MICROSECOND ); } static Time fromMilliseconds( int64_t millis ) { return fromNanoseconds( millis * NANOS_PER_MILLISECOND ); } - static Time fromSeconds( int64_t seconds ) { return fromNanoseconds( seconds * NANOS_PER_SECOND ); } + static Time fromSeconds( int64_t seconds ) { return fromNanoseconds( seconds * NANOS_PER_SECOND ); } - //as XX units since midnight - int64_t asNanoseconds() const { return m_ticks; } + // as XX units since midnight + int64_t asNanoseconds() const { return m_ticks; } int64_t asMicroseconds() const { return m_ticks / NANOS_PER_MICROSECOND; } int64_t asMilliseconds() const { return m_ticks / NANOS_PER_MILLISECOND; } - int64_t asSeconds() const { return m_ticks / NANOS_PER_SECOND; } + int64_t asSeconds() const { return m_ticks / NANOS_PER_SECOND; } bool operator==( const Time & rhs ) const { return m_ticks == rhs.m_ticks; } - bool operator!=( const Time & rhs ) const { return !( (*this) == rhs ); } - - bool operator< ( const Time & rhs ) const { return m_ticks < rhs.m_ticks; } - bool operator<=( const Time & rhs ) const { return m_ticks <= rhs.m_ticks; } - bool operator> ( const Time & rhs ) const { return m_ticks > rhs.m_ticks; } - bool operator>=( const Time & rhs ) const { return m_ticks >= rhs.m_ticks; } + bool operator!=( const Time & rhs ) const { return !( ( *this ) == rhs ); } - Time operator +( const TimeDelta & delta ) const { return Time( m_ticks + delta.asNanoseconds() ); } - Time operator -( const TimeDelta & delta ) const { return Time( m_ticks - delta.asNanoseconds() ); } - TimeDelta operator -( const Time & rhs ) const { return TimeDelta::fromNanoseconds( m_ticks - rhs.m_ticks ); } + bool operator<( const Time & rhs ) const { return m_ticks < rhs.m_ticks; } + bool operator<=( const Time & rhs ) const { return m_ticks <= rhs.m_ticks; } + bool operator>( const Time & rhs ) const { return m_ticks > rhs.m_ticks; } + bool operator>=( const Time & rhs ) const { return m_ticks >= rhs.m_ticks; } - Time& operator +=( const TimeDelta & delta ); - Time& operator -=( const TimeDelta & delta ); + Time operator+( const TimeDelta & delta ) const { return Time( m_ticks + delta.asNanoseconds() ); } + Time operator-( const TimeDelta & delta ) const { return Time( m_ticks - delta.asNanoseconds() ); } + TimeDelta operator-( const Time & rhs ) const { return TimeDelta::fromNanoseconds( m_ticks - rhs.m_ticks ); } + + Time & operator+=( const TimeDelta & delta ); + Time & operator-=( const TimeDelta & delta ); std::string asString() const; static Time fromString( const std::string & ); static Time NONE() { return Time( -1 ); } - static Time MIN_VALUE() { return Time( 0, 0, 0 ); } + static Time MIN_VALUE() { return Time( 0, 0, 0 ); } private: Time( int64_t raw ); void checkRange( int64_t t ); - //stored as nanos since midnight + // stored as nanos since midnight int64_t m_ticks; }; @@ -397,20 +451,20 @@ inline Time::Time( int hour, int minute, int second, int32_t nanosecond ) m_ticks = ( int64_t( hour ) * 3600 + int64_t( minute ) * 60 + int64_t( second ) ) * NANOS_PER_SECOND + nanosecond; } -inline Time& Time::operator +=( const TimeDelta & delta ) +inline Time & Time::operator+=( const TimeDelta & delta ) { - int64_t newval = m_ticks + delta.asNanoseconds(); + int64_t newval = m_ticks + delta.asNanoseconds(); checkRange( newval ); m_ticks = newval; - return *this; + return *this; } -inline Time& Time::operator -=( const TimeDelta & delta ) +inline Time & Time::operator-=( const TimeDelta & delta ) { - int64_t newval = m_ticks - delta.asNanoseconds(); + int64_t newval = m_ticks - delta.asNanoseconds(); checkRange( newval ); m_ticks = newval; - return *this; + return *this; } inline std::string Time::asString() const @@ -422,121 +476,145 @@ inline std::string Time::asString() const inline Time Time::fromString( const std::string & str ) { - int h = 0; - int m = 0; - int s = 0; + int h = 0; + int m = 0; + int s = 0; char f[16] = ""; - int ns = 0; - int n = sscanf( str.c_str(), "%d:%d:%d.%s", &h, &m, &s, f ); + int ns = 0; + int n = sscanf( str.c_str(), "%d:%d:%d.%s", &h, &m, &s, f ); if( n == 0 ) CSP_THROW( ValueError, "Failed to convert " << str << " to Time" ); - char *end; - ns = strtol( f, &end, 10 ); + char * end; + ns = strtol( f, &end, 10 ); int len = end - f; ns *= pow( 10, 9 - len ); return Time( h, m, s, ns ); } -inline std::ostream & operator <<( std::ostream &os, const Time & t ) +inline std::ostream & operator<<( std::ostream & os, const Time & t ) { os << t.asString(); return os; } -// Time is internally stored as an int64_t nanoseconds since 1970. +// Time is internally stored as an int64_t nanoseconds since 1970. // All DateTime objects are stored as UTC and should be treated as such class DateTime { public: - DateTime() : DateTime( DateTime::NONE() ) {} - DateTime( int year, int month, int day, - int hour = 0, int minute = 0, int second = 0, int nanosecond = 0 ); + DateTime() + : DateTime( DateTime::NONE() ) + { + } + DateTime( int year, int month, int day, int hour = 0, int minute = 0, int second = 0, int nanosecond = 0 ); DateTime( Date date, Time time ); - //Note this returns a shared thread-local buffer, invalidated on next call on same thread + // Note this returns a shared thread-local buffer, invalidated on next call on same thread const char * asCString() const; const char * asCString( char * buf, size_t buflen ) const; - std::string asString() const { return asCString(); } + std::string asString() const { return asCString(); } - //Helper creation methods + // Helper creation methods static DateTime now(); static DateTime fromString( const std::string & dtstr ); - //from XX units since epoch - static DateTime fromNanoseconds( int64_t nanos ) { return DateTime( nanos ); } - static DateTime fromMicroseconds( int64_t micros ) { return fromNanoseconds( micros * NANOS_PER_MICROSECOND ); } - static DateTime fromMilliseconds( int64_t millis ) { return fromNanoseconds( millis * NANOS_PER_MILLISECOND ); } - static DateTime fromSeconds( int64_t seconds ) { return fromNanoseconds( seconds * NANOS_PER_SECOND ); } + // from XX units since epoch + static DateTime fromNanoseconds( int64_t nanos ) { return DateTime( nanos ); } + static DateTime fromMicroseconds( int64_t micros ) { return fromNanoseconds( micros * NANOS_PER_MICROSECOND ); } + static DateTime fromMilliseconds( int64_t millis ) { return fromNanoseconds( millis * NANOS_PER_MILLISECOND ); } + static DateTime fromSeconds( int64_t seconds ) { return fromNanoseconds( seconds * NANOS_PER_SECOND ); } - //as XX units of time since epoch - int64_t asNanoseconds() const { return m_ticks; } + // as XX units of time since epoch + int64_t asNanoseconds() const { return m_ticks; } int64_t asMicroseconds() const { return m_ticks / NANOS_PER_MICROSECOND; } int64_t asMilliseconds() const { return m_ticks / NANOS_PER_MILLISECOND; } - int64_t asSeconds() const { return m_ticks / NANOS_PER_SECOND; } + int64_t asSeconds() const { return m_ticks / NANOS_PER_SECOND; } - bool isNone() const { return (*this) == DateTime::NONE(); } - bool isMin() const { return (*this) == DateTime::MIN_VALUE(); } - bool isMax() const { return (*this) == DateTime::MAX_VALUE(); } + bool isNone() const { return ( *this ) == DateTime::NONE(); } + bool isMin() const { return ( *this ) == DateTime::MIN_VALUE(); } + bool isMax() const { return ( *this ) == DateTime::MAX_VALUE(); } - //returns time / date component of the datetime - Time time() const { return isNone() ? Time::NONE() : Time::fromNanoseconds( m_ticks % ( SECONDS_PER_DAY * NANOS_PER_SECOND ) ); } + // returns time / date component of the datetime + Time time() const + { + return isNone() ? Time::NONE() : Time::fromNanoseconds( m_ticks % ( SECONDS_PER_DAY * NANOS_PER_SECOND ) ); + } Date date() const; - //return a datetime with same date but with given time - DateTime withTime( Time t ) const { return DateTime( ( m_ticks / ( SECONDS_PER_DAY * NANOS_PER_SECOND ) ) * SECONDS_PER_DAY * NANOS_PER_SECOND + t.asNanoseconds() ); } + // return a datetime with same date but with given time + DateTime withTime( Time t ) const + { + return DateTime( ( m_ticks / ( SECONDS_PER_DAY * NANOS_PER_SECOND ) ) * SECONDS_PER_DAY * NANOS_PER_SECOND + + t.asNanoseconds() ); + } - //round to given timedelta ( commonly used in bucketing ) - DateTime roundDown( TimeDelta td ) const { return DateTime::fromNanoseconds( ( m_ticks / td.asNanoseconds() ) * td.asNanoseconds() ); } + // round to given timedelta ( commonly used in bucketing ) + DateTime roundDown( TimeDelta td ) const + { + return DateTime::fromNanoseconds( ( m_ticks / td.asNanoseconds() ) * td.asNanoseconds() ); + } bool operator==( const DateTime & rhs ) const { return m_ticks == rhs.m_ticks; } - bool operator!=( const DateTime & rhs ) const { return !( (*this) == rhs ); } - - bool operator< ( const DateTime & rhs ) const { return m_ticks < rhs.m_ticks; } - bool operator<=( const DateTime & rhs ) const { return m_ticks <= rhs.m_ticks; } - bool operator> ( const DateTime & rhs ) const { return m_ticks > rhs.m_ticks; } - bool operator>=( const DateTime & rhs ) const { return m_ticks >= rhs.m_ticks; } - - DateTime operator +( const TimeDelta & delta ) const { return DateTime( m_ticks + delta.asNanoseconds() ); } - DateTime operator -( const TimeDelta & delta ) const { return DateTime( m_ticks - delta.asNanoseconds() ); } - TimeDelta operator -( const DateTime & rhs ) const { return TimeDelta::fromNanoseconds( m_ticks - rhs.m_ticks ); } - - DateTime& operator +=( const TimeDelta & delta ) { m_ticks += delta.asNanoseconds(); return *this; } - DateTime& operator -=( const TimeDelta & delta ) { m_ticks -= delta.asNanoseconds(); return *this; } - - static constexpr DateTime NONE() { return DateTime(std::numeric_limits::min()); } - static constexpr DateTime MIN_VALUE() { return DateTime( std::numeric_limits::min() + 1 ); } //min reserved for NONE + bool operator!=( const DateTime & rhs ) const { return !( ( *this ) == rhs ); } + + bool operator<( const DateTime & rhs ) const { return m_ticks < rhs.m_ticks; } + bool operator<=( const DateTime & rhs ) const { return m_ticks <= rhs.m_ticks; } + bool operator>( const DateTime & rhs ) const { return m_ticks > rhs.m_ticks; } + bool operator>=( const DateTime & rhs ) const { return m_ticks >= rhs.m_ticks; } + + DateTime operator+( const TimeDelta & delta ) const { return DateTime( m_ticks + delta.asNanoseconds() ); } + DateTime operator-( const TimeDelta & delta ) const { return DateTime( m_ticks - delta.asNanoseconds() ); } + TimeDelta operator-( const DateTime & rhs ) const { return TimeDelta::fromNanoseconds( m_ticks - rhs.m_ticks ); } + + DateTime & operator+=( const TimeDelta & delta ) + { + m_ticks += delta.asNanoseconds(); + return *this; + } + DateTime & operator-=( const TimeDelta & delta ) + { + m_ticks -= delta.asNanoseconds(); + return *this; + } + + static constexpr DateTime NONE() { return DateTime( std::numeric_limits::min() ); } + static constexpr DateTime MIN_VALUE() + { + return DateTime( std::numeric_limits::min() + 1 ); + } // min reserved for NONE static constexpr DateTime MAX_VALUE() { return DateTime( std::numeric_limits::max() ); } protected: - //the fact that we store this as nanos is an implementation detail - constexpr DateTime( int64_t raw_nanos ) : m_ticks( raw_nanos ) {} + // the fact that we store this as nanos is an implementation detail + constexpr DateTime( int64_t raw_nanos ) + : m_ticks( raw_nanos ) + { + } tm asTM() const; int64_t m_ticks; }; -inline DateTime::DateTime( int year, int month, int day, - int hour, int minute, int second, int nanosecond ) +inline DateTime::DateTime( int year, int month, int day, int hour, int minute, int second, int nanosecond ) { tm TM; memset( &TM, 0, sizeof( TM ) ); - TM.tm_year = year - 1900; - TM.tm_mon = month - 1; - TM.tm_mday = day; - TM.tm_hour = hour; - TM.tm_min = minute; - TM.tm_sec = second; + TM.tm_year = year - 1900; + TM.tm_mon = month - 1; + TM.tm_mday = day; + TM.tm_hour = hour; + TM.tm_min = minute; + TM.tm_sec = second; TM.tm_isdst = -1; m_ticks = timegm( &TM ); m_ticks = m_ticks * NANOS_PER_SECOND + nanosecond; - } -inline DateTime::DateTime( Date date, Time time ) : - DateTime( date.year(), date.month(), date.day(), time.hour(), time.minute(), time.second(), time.nanosecond() ) +inline DateTime::DateTime( Date date, Time time ) + : DateTime( date.year(), date.month(), date.day(), time.hour(), time.minute(), time.second(), time.nanosecond() ) { } @@ -544,7 +622,7 @@ inline DateTime DateTime::now() { timespec ts; #ifdef WIN32 - timespec_get(&ts, TIME_UTC); + timespec_get( &ts, TIME_UTC ); #else clock_gettime( CLOCK_REALTIME, &ts ); #endif @@ -556,7 +634,6 @@ inline DateTime DateTime::fromString( const std::string & str ) return { Date::fromYYYYMMDD( str ), Time::fromString( str.c_str() + 9 ) }; } - inline const char * DateTime::asCString() const { static thread_local char s_buf[128]; @@ -565,15 +642,15 @@ inline const char * DateTime::asCString() const inline const char * DateTime::asCString( char * buf, size_t buflen ) const { - if( (*this) == DateTime::NONE() ) + if( ( *this ) == DateTime::NONE() ) return strncpy( buf, "none", buflen ); - if( (*this) == DateTime::MIN_VALUE() ) + if( ( *this ) == DateTime::MIN_VALUE() ) return strncpy( buf, "min", buflen ); - if( (*this) == DateTime::MAX_VALUE() ) + if( ( *this ) == DateTime::MAX_VALUE() ) return strncpy( buf, "max", buflen ); - + tm TM = asTM(); size_t len; @@ -584,52 +661,53 @@ inline const char * DateTime::asCString( char * buf, size_t buflen ) const if( nanos < 0 ) nanos += NANOS_PER_SECOND; - snprintf( buf + len, buflen - len, ".%09ld", (long int) nanos ); + snprintf( buf + len, buflen - len, ".%09ld", (long int)nanos ); return buf; } -inline std::ostream & operator <<( std::ostream &os, const DateTime & dt ) +inline std::ostream & operator<<( std::ostream & os, const DateTime & dt ) { os << dt.asString(); return os; } -//Helper class to extract day/month/year/etc info from raw timestamp -//ie DateTimeEx dte( existingDt ) -//dte.day, etc etc +// Helper class to extract day/month/year/etc info from raw timestamp +// ie DateTimeEx dte( existingDt ) +// dte.day, etc etc class DateTimeEx : public DateTime { public: DateTimeEx( const DateTime & dt ); - int day() const { return m_tm.tm_mday; } + int day() const { return m_tm.tm_mday; } int month() const { return m_tm.tm_mon + 1; } - int year() const { return m_tm.tm_year + 1900; } - - int hour() const { return m_tm.tm_hour; } + int year() const { return m_tm.tm_year + 1900; } + + int hour() const { return m_tm.tm_hour; } int minute() const { return m_tm.tm_min; } int second() const { return m_tm.tm_sec; } - //the fractional second access are non-cumulative, meaning microseconds() includes milliseconds(). - //ie if we have micros 222333, milliseconds() wil return 222 and microseconds will return 222333 + // the fractional second access are non-cumulative, meaning microseconds() includes milliseconds(). + // ie if we have micros 222333, milliseconds() wil return 222 and microseconds will return 222333 int milliseconds() const { return nanoseconds() / NANOS_PER_MILLISECOND; } int microseconds() const { return nanoseconds() / NANOS_PER_MICROSECOND; } - int nanoseconds() const + int nanoseconds() const { auto nanos = m_ticks % NANOS_PER_SECOND; if( unlikely( nanos < 0 ) ) - nanos += NANOS_PER_SECOND; + nanos += NANOS_PER_SECOND; return nanos; } - - //day of week, 0 = Sunday, 6 = Saturday + + // day of week, 0 = Sunday, 6 = Saturday int weekday() const { return m_tm.tm_wday; } private: tm m_tm; }; -inline DateTimeEx::DateTimeEx( const DateTime & dt ) : DateTime( dt ) +inline DateTimeEx::DateTimeEx( const DateTime & dt ) + : DateTime( dt ) { m_tm = asTM(); } @@ -653,41 +731,42 @@ inline TimeDelta Date::operator-( const Date & rhs ) const return DateTime( *this, Time( 0, 0, 0 ) ) - DateTime( rhs, Time( 0, 0, 0 ) ); } -}; +}; // namespace csp -//hash definition for unordered_set / unordered_map keys +// hash definition for unordered_set / unordered_map keys namespace std { - template<> struct hash +template<> +struct hash +{ + size_t operator()( const csp::DateTime & dt ) const { - size_t operator()( const csp::DateTime & dt ) const - { - return std::hash< int64_t >()( static_cast( dt.asNanoseconds() ) ); - } - }; + return std::hash()( static_cast( dt.asNanoseconds() ) ); + } +}; - template<> struct hash - { - size_t operator()( const csp::Date & dt ) const - { - return dt.hash(); - } - }; +template<> +struct hash +{ + size_t operator()( const csp::Date & dt ) const { return dt.hash(); } +}; - template<> struct hash +template<> +struct hash +{ + size_t operator()( const csp::Time & t ) const { - size_t operator()( const csp::Time & t ) const - { - return std::hash< int64_t >()( static_cast( t.asNanoseconds() ) ); - } - }; + return std::hash()( static_cast( t.asNanoseconds() ) ); + } +}; - template<> struct hash +template<> +struct hash +{ + size_t operator()( const csp::TimeDelta & td ) const { - size_t operator()( const csp::TimeDelta & td ) const - { - return std::hash< int64_t >()( static_cast( td.asNanoseconds() ) ); - } - }; -} + return std::hash()( static_cast( td.asNanoseconds() ) ); + } +}; +} // namespace std #endif diff --git a/cpp/csp/cppnodes/baselibimpl.cpp b/cpp/csp/cppnodes/baselibimpl.cpp index 52a5537d9..7b90525a1 100644 --- a/cpp/csp/cppnodes/baselibimpl.cpp +++ b/cpp/csp/cppnodes/baselibimpl.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include namespace csp::cppnodes @@ -12,8 +12,7 @@ def sample(trigger: ts['Y'], x: ts['T']): */ DECLARE_CPPNODE( sample ) { - INIT_CPPNODE( sample ) - {} + INIT_CPPNODE( sample ) {} TS_INPUT( Generic, trigger ); TS_INPUT( Generic, x ); @@ -42,16 +41,14 @@ def firstN(x: ts['T'], N: int): DECLARE_CPPNODE( firstN ) { - INIT_CPPNODE( firstN ) - { - } + INIT_CPPNODE( firstN ) {} - TS_INPUT( Generic, x ); + TS_INPUT( Generic, x ); SCALAR_INPUT( int64_t, N ); TS_OUTPUT( Generic ); - STATE_VAR( int, s_count{0} ); + STATE_VAR( int, s_count{ 0 } ); START() { @@ -77,8 +74,7 @@ def count(x: ts['T']): */ DECLARE_CPPNODE( count ) { - INIT_CPPNODE( count ) - {} + INIT_CPPNODE( count ) {} TS_INPUT( Generic, x ); TS_OUTPUT( int64_t ); @@ -99,8 +95,7 @@ def _delay_by_timedelta(x: ts['T'], delay: timedelta): */ DECLARE_CPPNODE( _delay_by_timedelta ) { - INIT_CPPNODE( _delay_by_timedelta ) - {} + INIT_CPPNODE( _delay_by_timedelta ) {} TS_INPUT( Generic, x ); SCALAR_INPUT( TimeDelta, delay ); @@ -132,8 +127,7 @@ DECLARE_CPPNODE( _delay_by_ticks ) TS_OUTPUT( Generic ); - INIT_CPPNODE( _delay_by_ticks ) - {} + INIT_CPPNODE( _delay_by_ticks ) {} START() { @@ -147,11 +141,12 @@ DECLARE_CPPNODE( _delay_by_ticks ) { if( csp.ticked( x ) && csp.count( x ) > delay ) { - switchCspType( x.type(), [this]( auto tag ) - { - using ElemT = typename decltype(tag)::type; - RETURN( x.valueAtIndex( delay ) ); - } ); + switchCspType( x.type(), + [this]( auto tag ) + { + using ElemT = typename decltype( tag )::type; + RETURN( x.valueAtIndex( delay ) ); + } ); } } }; @@ -166,8 +161,7 @@ def merge(x: ts['T'], y : ts[ 'T' ] ): */ DECLARE_CPPNODE( merge ) { - INIT_CPPNODE( merge ) - {} + INIT_CPPNODE( merge ) {} TS_INPUT( Generic, x ); TS_INPUT( Generic, y ); @@ -185,7 +179,6 @@ DECLARE_CPPNODE( merge ) EXPORT_CPPNODE( merge ); - /* @csp.node(cppimpl=_cspbaselibimpl.split) def split(flag: ts[bool], x: ts['T']): @@ -194,13 +187,12 @@ def split(flag: ts[bool], x: ts['T']): */ DECLARE_CPPNODE( split ) { - INIT_CPPNODE( split ) - {} + INIT_CPPNODE( split ) {} - TS_INPUT( bool, flag ); + TS_INPUT( bool, flag ); TS_INPUT( Generic, x ); TS_NAMED_OUTPUT_RENAMED( Generic, false, false_ ); - TS_NAMED_OUTPUT_RENAMED( Generic, true, true_ ); + TS_NAMED_OUTPUT_RENAMED( Generic, true, true_ ); START() { @@ -228,8 +220,7 @@ def cast_int_to_float(x: csp.ts[int]): */ DECLARE_CPPNODE( cast_int_to_float ) { - INIT_CPPNODE( cast_int_to_float ) - {} + INIT_CPPNODE( cast_int_to_float ) {} TS_INPUT( int64_t, x ); TS_OUTPUT( double ); @@ -250,10 +241,9 @@ def filter(flag: ts[bool], x: ts['T']): */ DECLARE_CPPNODE( filter ) { - INIT_CPPNODE( filter ) - {} + INIT_CPPNODE( filter ) {} - TS_INPUT( bool, flag ); + TS_INPUT( bool, flag ); TS_INPUT( Generic, x ); TS_OUTPUT( Generic ); @@ -278,8 +268,7 @@ def _drop_dups_float(x: ts[float], eps: float): */ DECLARE_CPPNODE( _drop_dups_float ) { - INIT_CPPNODE( _drop_dups_float ) - {} + INIT_CPPNODE( _drop_dups_float ) {} TS_INPUT( double, x ); @@ -287,17 +276,17 @@ DECLARE_CPPNODE( _drop_dups_float ) TS_OUTPUT( double ); - STATE_VAR( bool, s_first{true} ); + STATE_VAR( bool, s_first{ true } ); STATE_VAR( double, s_prev{} ); INVOKE() { if( csp.ticked( x ) ) { - if( s_first || ( isnan( x ) != isnan( s_prev ) ) || ( !isnan( x ) && fabs( x - s_prev ) >= eps )) + if( s_first || ( isnan( x ) != isnan( s_prev ) ) || ( !isnan( x ) && fabs( x - s_prev ) >= eps ) ) { s_first = false; - s_prev = x; + s_prev = x; RETURN( x ); } } @@ -336,9 +325,9 @@ def unroll(x: ts[['T']]): */ DECLARE_CPPNODE( unroll ) { - TS_INPUT( Generic, x ); - ALARM( Generic, alarm ); - STATE_VAR( uint32_t, s_pending{0} ); + TS_INPUT( Generic, x ); + ALARM( Generic, alarm ); + STATE_VAR( uint32_t, s_pending{ 0 } ); TS_OUTPUT( Generic ); @@ -346,54 +335,55 @@ DECLARE_CPPNODE( unroll ) INIT_CPPNODE( unroll ) { - //we need to access type information using the input / outout defs because the actual - //ts() instances arent created at this point + // we need to access type information using the input / outout defs because the actual + // ts() instances arent created at this point auto & x_def = tsinputDef( "x" ); - if( x_def.type -> type() != CspType::Type::ARRAY ) - CSP_THROW( TypeError, "unroll expected ts array type, got " << x_def.type -> type() ); + if( x_def.type->type() != CspType::Type::ARRAY ) + CSP_THROW( TypeError, "unroll expected ts array type, got " << x_def.type->type() ); auto * aType = static_cast( x_def.type.get() ); - elemType = aType -> elemType(); + elemType = aType->elemType(); - //we cant easily support unrolling list of typed lists ( ts[ [[int]] ] ). Since we dont recurse type info more than one level - //the input elemType would be DIALECT_GENERIC, but the output type would be the correct ARRAY:Type type. Briding the two here is - //complex and not (currently) wirht the effort, so fallback to python by throwing NotImplemented + // we cant easily support unrolling list of typed lists ( ts[ [[int]] ] ). Since we dont recurse type info more + // than one level the input elemType would be DIALECT_GENERIC, but the output type would be the correct + // ARRAY:Type type. Briding the two here is complex and not (currently) wirht the effort, so fallback to python + // by throwing NotImplemented auto & out_def = tsoutputDef( "" ); - if( out_def.type -> type() == CspType::Type::ARRAY ) + if( out_def.type->type() == CspType::Type::ARRAY ) CSP_THROW( NotImplemented, "unroll cppimpl doesnt currently support unrolloing lists of typed lists" ); } INVOKE() { - //single switch up front, no need to do it multiple times - switchCspType( elemType, [this]( auto tag ) - { - using ElemT = typename decltype(tag)::type; - using ArrayT = typename CspType::Type::toCArrayType::type; - - if( csp.ticked( x ) ) - { - auto & v = x.lastValue(); - size_t sz = v.size(); - if( likely( sz > 0 ) ) - { - size_t idx = 0; - if( !s_pending ) - CSP_OUTPUT( static_cast( v[idx++] ) ); - - s_pending += sz - idx; - for( ; idx < sz; ++idx ) - csp.schedule_alarm( alarm, TimeDelta::ZERO(), static_cast( v[idx] ) ); - } - } + // single switch up front, no need to do it multiple times + switchCspType( elemType, + [this]( auto tag ) + { + using ElemT = typename decltype( tag )::type; + using ArrayT = typename CspType::Type::toCArrayType::type; - if( csp.ticked( alarm ) ) - { - --s_pending; - RETURN( alarm.lastValue() ); - } + if( csp.ticked( x ) ) + { + auto & v = x.lastValue(); + size_t sz = v.size(); + if( likely( sz > 0 ) ) + { + size_t idx = 0; + if( !s_pending ) + CSP_OUTPUT( static_cast( v[idx++] ) ); + + s_pending += sz - idx; + for( ; idx < sz; ++idx ) + csp.schedule_alarm( alarm, TimeDelta::ZERO(), static_cast( v[idx] ) ); + } + } - } ); + if( csp.ticked( alarm ) ) + { + --s_pending; + RETURN( alarm.lastValue() ); + } + } ); } }; @@ -414,45 +404,47 @@ DECLARE_CPPNODE( collect ) INIT_CPPNODE( collect ) { - //we cant process 'T' of type typed list ( it [int] ) because the input type would be ARRAY:INT64 - //but the output type would become ARRAY:DIALECT_GENERIC, which we cant create. just fallback to python + // we cant process 'T' of type typed list ( it [int] ) because the input type would be ARRAY:INT64 + // but the output type would become ARRAY:DIALECT_GENERIC, which we cant create. just fallback to python auto & x_def = tsinputDef( "x" ); - if( x_def.type -> type() == CspType::Type::ARRAY ) + if( x_def.type->type() == CspType::Type::ARRAY ) CSP_THROW( NotImplemented, "cppimpl of collect cannot handle typed lists inputs" ); auto & out_def = tsoutputDef( "" ); - if( out_def.type -> type() != CspType::Type::ARRAY ) - CSP_THROW( TypeError, "cppimpl for collect expected output type to be list, got " << out_def.type -> type() ); + if( out_def.type->type() != CspType::Type::ARRAY ) + CSP_THROW( TypeError, "cppimpl for collect expected output type to be list, got " << out_def.type->type() ); auto * aType = static_cast( out_def.type.get() ); - elemType = aType -> elemType(); + elemType = aType->elemType(); - if( elemType -> type() != x_def.type -> type() ) - CSP_THROW( TypeError, "cppimpl for collect has unexpected type mistmatch, input type is " << x_def.type -> type () << - " but output array type is " << elemType -> type() ); + if( elemType->type() != x_def.type->type() ) + CSP_THROW( TypeError, + "cppimpl for collect has unexpected type mistmatch, input type is " + << x_def.type->type() << " but output array type is " << elemType->type() ); } START() { - //to avoid the need to check in every invoke + // to avoid the need to check in every invoke if( x.size() == 0 ) csp.make_passive( x ); } INVOKE() { - //single switch up front, no need to do it multiple times - //we expect all elements to be of the same type - switchCspType( elemType, [this]( auto tag ) + // single switch up front, no need to do it multiple times + // we expect all elements to be of the same type + switchCspType( elemType, + [this]( auto tag ) { - using TagType = decltype( tag ); - using ElemT = typename TagType::type; - using ArrayT = typename CspType::Type::toCArrayType::type; - - ArrayT & out = unnamed_output().reserveSpace(); - out.clear(); - for( auto it = x.tickedinputs(); it; ++it ) - out.emplace_back( it -> lastValueTyped() ); + using TagType = decltype( tag ); + using ElemT = typename TagType::type; + using ArrayT = typename CspType::Type::toCArrayType::type; + + ArrayT & out = unnamed_output().reserveSpace(); + out.clear(); + for( auto it = x.tickedinputs(); it; ++it ) + out.emplace_back( it->lastValueTyped() ); } ); } }; @@ -475,7 +467,7 @@ DECLARE_CPPNODE( demultiplex ) INIT_CPPNODE( demultiplex ) { auto & key_def = tsinputDef( "key" ); - if( key_def.type -> type() != CspType::Type::STRING ) + if( key_def.type->type() != CspType::Type::STRING ) CSP_THROW( NotImplemented, "cppimpl for demultiplex not supported on non-string key types" ); } @@ -488,8 +480,8 @@ DECLARE_CPPNODE( demultiplex ) { if( csp.valid( key ) ) { - auto &key_str = key.lastValue(); - auto elemId = unnamed_output().elemId( key_str ); + auto & key_str = key.lastValue(); + auto elemId = unnamed_output().elemId( key_str ); if( elemId != InputId::ELEM_ID_NONE ) { unnamed_output()[elemId].output( x ); @@ -517,12 +509,12 @@ DECLARE_CPPNODE( multiplex ) TS_OUTPUT( Generic ); - STATE_VAR( bool, s_key_valid{false} ); + STATE_VAR( bool, s_key_valid{ false } ); INIT_CPPNODE( multiplex ) { auto & key_def = tsinputDef( "key" ); - if( key_def.type -> type() != CspType::Type::STRING ) + if( key_def.type->type() != CspType::Type::STRING ) CSP_THROW( NotImplemented, "cppimpl for multiplex not supported on non-string key types" ); } @@ -530,7 +522,7 @@ DECLARE_CPPNODE( multiplex ) { if( csp.ticked( key ) ) { - auto &key_str = key.lastValue(); + auto & key_str = key.lastValue(); csp.make_passive( x ); int64_t elemId = x.elemId( key_str ); @@ -549,10 +541,9 @@ DECLARE_CPPNODE( multiplex ) if( s_key_valid ) { - auto &key_str = key.lastValue(); - int64_t elemId = x.elemId( key_str ); - if( csp.ticked( x[elemId] ) || - ( tick_on_index && csp.ticked(key) && csp.valid(x[elemId]) ) ) + auto & key_str = key.lastValue(); + int64_t elemId = x.elemId( key_str ); + if( csp.ticked( x[elemId] ) || ( tick_on_index && csp.ticked( key ) && csp.valid( x[elemId] ) ) ) { CSP_OUTPUT( x[elemId] ); } @@ -576,7 +567,7 @@ DECLARE_CPPNODE( times ) TS_INPUT( Generic, x ); TS_OUTPUT( DateTime ); - INIT_CPPNODE( times ) { } + INIT_CPPNODE( times ) {} INVOKE() { @@ -600,7 +591,7 @@ DECLARE_CPPNODE( times_ns ) TS_INPUT( Generic, x ); TS_OUTPUT( int64_t ); - INIT_CPPNODE( times_ns ) { } + INIT_CPPNODE( times_ns ) {} INVOKE() { @@ -617,10 +608,9 @@ def struct_field(x: ts['T'], field: str, fieldType: 'Y'): */ DECLARE_CPPNODE( struct_field ) { - INIT_CPPNODE( struct_field ) - {} + INIT_CPPNODE( struct_field ) {} - TS_INPUT( StructPtr, x ); + TS_INPUT( StructPtr, x ); SCALAR_INPUT( std::string, field ); TS_OUTPUT( Generic ); @@ -628,19 +618,20 @@ DECLARE_CPPNODE( struct_field ) START() { auto * structType = static_cast( x.type() ); - m_fieldAccess = structType -> meta() -> field( field ); + m_fieldAccess = structType->meta()->field( field ); if( !m_fieldAccess ) - CSP_THROW( TypeError, "Struct " << structType -> meta() -> name() << " has no field named " << field.value() ); + CSP_THROW( TypeError, "Struct " << structType->meta()->name() << " has no field named " << field.value() ); } INVOKE() { - if( m_fieldAccess -> isSet( x.lastValue().get() ) ) + if( m_fieldAccess->isSet( x.lastValue().get() ) ) { - switchCspType( m_fieldAccess -> type(), [this]( auto tag ) + switchCspType( m_fieldAccess->type(), + [this]( auto tag ) { - using T = typename decltype(tag)::type; - RETURN( m_fieldAccess -> value( x.lastValue().get() ) ); + using T = typename decltype( tag )::type; + RETURN( m_fieldAccess->value( x.lastValue().get() ) ); } ); } } @@ -651,8 +642,8 @@ DECLARE_CPPNODE( struct_field ) EXPORT_CPPNODE( struct_field ); -//fromts and collectts are unfortunately identical except for tickeditems() vs validitems() -//but i dont think its enough to warrant any refactoring at the moment +// fromts and collectts are unfortunately identical except for tickeditems() vs validitems() +// but i dont think its enough to warrant any refactoring at the moment /* @csp.node def struct_fromts(cls: 'T', inputs: {str: ts[object]}): @@ -661,28 +652,29 @@ def struct_fromts(cls: 'T', inputs: {str: ts[object]}): DECLARE_CPPNODE( struct_fromts ) { TS_DICTBASKET_INPUT( Generic, inputs ); - TS_INPUT( Generic, trigger ); - SCALAR_INPUT( StructMetaPtr, cls ); - SCALAR_INPUT( bool, use_trigger ); + TS_INPUT( Generic, trigger ); + SCALAR_INPUT( StructMetaPtr, cls ); + SCALAR_INPUT( bool, use_trigger ); TS_OUTPUT( StructPtr ); - INIT_CPPNODE( struct_fromts ) - { - } + INIT_CPPNODE( struct_fromts ) {} START() { for( size_t elemId = 0; elemId < inputs.shape().size(); ++elemId ) { - auto & key = inputs.shape()[ elemId ]; - auto & structField = cls.value() -> field( key ); + auto & key = inputs.shape()[elemId]; + auto & structField = cls.value()->field( key ); if( !structField ) - CSP_THROW( ValueError, cls.value() -> name() << ".fromts() received unknown struct field \"" << key << "\"" ); + CSP_THROW( ValueError, + cls.value()->name() << ".fromts() received unknown struct field \"" << key << "\"" ); - if( structField -> type() -> type() != inputs[ elemId ].type() -> type() ) - CSP_THROW( TypeError, cls.value() -> name() << ".fromts() field \"" << key << "\" expected ts type " - << structField -> type() -> type() << " but got " << inputs[ elemId ].type() -> type() ); + if( structField->type()->type() != inputs[elemId].type()->type() ) + CSP_THROW( TypeError, + cls.value()->name() + << ".fromts() field \"" << key << "\" expected ts type " << structField->type()->type() + << " but got " << inputs[elemId].type()->type() ); m_structFields.push_back( structField.get() ); } @@ -693,16 +685,16 @@ DECLARE_CPPNODE( struct_fromts ) INVOKE() { - auto out = cls.value() -> create(); + auto out = cls.value()->create(); for( auto it = inputs.validinputs(); it; ++it ) { auto * fieldAccess = m_structFields[it.elemId()]; - switchCspType( it -> type(), [&it,&out,fieldAccess]( auto tag ) + switchCspType( it->type(), + [&it, &out, fieldAccess]( auto tag ) { - using ElemT = typename decltype(tag)::type; - fieldAccess -> setValue( out.get(), it -> lastValueTyped() ); - } - ); + using ElemT = typename decltype( tag )::type; + fieldAccess->setValue( out.get(), it->lastValueTyped() ); + } ); } CSP_OUTPUT( std::move( out ) ); @@ -724,22 +716,23 @@ DECLARE_CPPNODE( struct_collectts ) SCALAR_INPUT( StructMetaPtr, cls ); TS_OUTPUT( StructPtr ); - INIT_CPPNODE( struct_collectts ) - { - } + INIT_CPPNODE( struct_collectts ) {} START() { for( size_t elemId = 0; elemId < inputs.shape().size(); ++elemId ) { - auto & key = inputs.shape()[ elemId ]; - auto & structField = cls.value() -> field( key ); + auto & key = inputs.shape()[elemId]; + auto & structField = cls.value()->field( key ); if( !structField ) - CSP_THROW( ValueError, cls.value() -> name() << ".collectts() received unknown struct field \"" << key << "\"" ); + CSP_THROW( ValueError, + cls.value()->name() << ".collectts() received unknown struct field \"" << key << "\"" ); - if( structField -> type() -> type() != inputs[ elemId ].type() -> type() ) - CSP_THROW( TypeError, cls.value() -> name() << ".collectts() field \"" << key << "\" expected ts type " - << structField -> type() -> type() << " but got " << inputs[ elemId ].type() -> type() ); + if( structField->type()->type() != inputs[elemId].type()->type() ) + CSP_THROW( TypeError, + cls.value()->name() + << ".collectts() field \"" << key << "\" expected ts type " + << structField->type()->type() << " but got " << inputs[elemId].type()->type() ); m_structFields.push_back( structField.get() ); } @@ -747,16 +740,16 @@ DECLARE_CPPNODE( struct_collectts ) INVOKE() { - auto out = cls.value() -> create(); + auto out = cls.value()->create(); for( auto it = inputs.tickedinputs(); it; ++it ) { auto * fieldAccess = m_structFields[it.elemId()]; - switchCspType( it -> type(), [&it,&out,fieldAccess]( auto tag ) + switchCspType( it->type(), + [&it, &out, fieldAccess]( auto tag ) { - using ElemT = typename decltype(tag)::type; - fieldAccess -> setValue( out.get(), it -> lastValueTyped() ); - } - ); + using ElemT = typename decltype( tag )::type; + fieldAccess->setValue( out.get(), it->lastValueTyped() ); + } ); } CSP_OUTPUT( std::move( out ) ); @@ -767,4 +760,4 @@ DECLARE_CPPNODE( struct_collectts ) EXPORT_CPPNODE( struct_collectts ); -} +} // namespace csp::cppnodes diff --git a/cpp/csp/cppnodes/basketlibimpl.cpp b/cpp/csp/cppnodes/basketlibimpl.cpp index 0749a3133..7670d99af 100644 --- a/cpp/csp/cppnodes/basketlibimpl.cpp +++ b/cpp/csp/cppnodes/basketlibimpl.cpp @@ -12,13 +12,13 @@ DECLARE_CPPNODE( _sync_list ) ALARM( bool, a_end ); - STATE_VAR( size_t, s_count{0} ); + STATE_VAR( size_t, s_count{ 0 } ); STATE_VAR( Scheduler::Handle, s_alarm_handle ); STATE_VAR( std::vector, s_current_ticked{} ); TS_LISTBASKET_OUTPUT( Generic ); - INIT_CPPNODE( _sync_list ) { } + INIT_CPPNODE( _sync_list ) {} START() { @@ -36,11 +36,11 @@ DECLARE_CPPNODE( _sync_list ) for( auto it = x.tickedinputs(); it; ++it ) { - if( s_current_ticked[ it.elemId() ] == false ) + if( s_current_ticked[it.elemId()] == false ) { s_count++; } - s_current_ticked[ it.elemId() ] = true; + s_current_ticked[it.elemId()] = true; } } @@ -52,9 +52,9 @@ DECLARE_CPPNODE( _sync_list ) { for( size_t elemId = 0; elemId < x.size(); elemId++ ) { - if( s_current_ticked[ elemId ] ) + if( s_current_ticked[elemId] ) { - unnamed_output()[ elemId ].output( x[ elemId ] ); + unnamed_output()[elemId].output( x[elemId] ); } } } @@ -85,7 +85,7 @@ DECLARE_CPPNODE( _sample_list ) TS_LISTBASKET_OUTPUT( Generic ); - INIT_CPPNODE( _sample_list ) { } + INIT_CPPNODE( _sample_list ) {} START() { @@ -94,12 +94,12 @@ DECLARE_CPPNODE( _sample_list ) INVOKE() { - if( csp.ticked(trigger) ) + if( csp.ticked( trigger ) ) { - for( auto it = x.validinputs(); it; ++it) + for( auto it = x.validinputs(); it; ++it ) { auto idx = it.elemId(); - unnamed_output()[ idx ].output( x[idx] ); + unnamed_output()[idx].output( x[idx] ); } } } @@ -107,4 +107,4 @@ DECLARE_CPPNODE( _sample_list ) EXPORT_CPPNODE( _sample_list ); -} \ No newline at end of file +} // namespace csp::cppnodes \ No newline at end of file diff --git a/cpp/csp/cppnodes/mathimpl.cpp b/cpp/csp/cppnodes/mathimpl.cpp index eeb865f8f..68c2b7413 100644 --- a/cpp/csp/cppnodes/mathimpl.cpp +++ b/cpp/csp/cppnodes/mathimpl.cpp @@ -1,6 +1,5 @@ -#include #include - +#include namespace csp::cppnodes { @@ -11,20 +10,25 @@ Math operations // Unary operation -template +template DECLARE_CPPNODE( _unary_op ) { TS_INPUT( ArgT, x ); TS_OUTPUT( OutT ); - //Expanded out INIT_CPPNODE without create call... - CSP csp; - const char * name() const override { return "_unary_op"; } + // Expanded out INIT_CPPNODE without create call... + CSP csp; + const char * name() const override + { + return "_unary_op"; + } public: _STATIC_CREATE_METHOD( SINGLE_ARG( _unary_op ) ); - _unary_op( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _unary_op( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } INVOKE() { @@ -32,91 +36,183 @@ DECLARE_CPPNODE( _unary_op ) } }; -template inline T _abs( T x ){ return std::abs( x ); } -template inline double _ln( T x ){ return std::log( x ); } -template inline double _log2( T x ){ return std::log2( x ); } -template inline double _log10( T x ){ return std::log10( x ); } -template inline double _exp( T x ){ return std::exp( x ); } -template inline double _exp2( T x ){ return std::exp2( x ); } -template inline double _sqrt( T x ){ return std::sqrt( x ); } -template inline double _erf( T x ){ return std::erf( x ); } -template inline double _sin( T x ){ return std::sin( x ); } -template inline double _cos( T x ){ return std::cos( x ); } -template inline double _tan( T x ){ return std::tan( x ); } -template inline double _asin( T x ){ return std::asin( x ); } -template inline double _acos( T x ){ return std::acos( x ); } -template inline double _atan( T x ){ return std::atan( x ); } -template inline double _sinh( T x ){ return std::sinh( x ); } -template inline double _cosh( T x ){ return std::cosh( x ); } -template inline double _tanh( T x ){ return std::tanh( x ); } -template inline double _asinh( T x ){ return std::asinh( x ); } -template inline double _acosh( T x ){ return std::acosh( x ); } -template inline double _atanh( T x ){ return std::atanh( x ); } - -inline bool _not_( bool x ){ return !x; } -inline int64_t _bitwise_not(int64_t x) { return ~x; } - -#define EXPORT_UNARY_OP( Name, ArgType, OutType, Func ) EXPORT_TEMPLATE_CPPNODE( Name, SINGLE_ARG( _unary_op ) ) - EXPORT_UNARY_OP( abs_f, double, double, abs ); - EXPORT_UNARY_OP( abs_i, int64_t, int64_t, abs ); - EXPORT_UNARY_OP( ln_f, double, double, ln ); - EXPORT_UNARY_OP( ln_i, int64_t, double, ln ); - EXPORT_UNARY_OP( log2_f, double, double, log2 ); - EXPORT_UNARY_OP( log2_i, int64_t, double, log2 ); - EXPORT_UNARY_OP( log10_f, double, double, log10 ); - EXPORT_UNARY_OP( log10_i, int64_t, double, log10 ); - EXPORT_UNARY_OP( exp_f, double, double, exp ); - EXPORT_UNARY_OP( exp_i, int64_t, double, exp ); - EXPORT_UNARY_OP( exp2_f, double, double, exp2 ); - EXPORT_UNARY_OP( exp2_i, int64_t, double, exp2 ); - EXPORT_UNARY_OP( sqrt_f, double, double, sqrt ); - EXPORT_UNARY_OP( sqrt_i, int64_t, double, sqrt ); - EXPORT_UNARY_OP( erf_f, double, double, erf ); - EXPORT_UNARY_OP( erf_i, int64_t, double, erf ); - EXPORT_UNARY_OP( sin_f, double, double, sin ); - EXPORT_UNARY_OP( sin_i, int64_t, double, sin ); - EXPORT_UNARY_OP( cos_f, double, double, cos ); - EXPORT_UNARY_OP( cos_i, int64_t, double, cos ); - EXPORT_UNARY_OP( tan_f, double, double, tan ); - EXPORT_UNARY_OP( tan_i, int64_t, double, tan ); - EXPORT_UNARY_OP( asin_f, double, double, asin ); - EXPORT_UNARY_OP( asin_i, int64_t, double, asin ); - EXPORT_UNARY_OP( acos_f, double, double, acos ); - EXPORT_UNARY_OP( acos_i, int64_t, double, acos ); - EXPORT_UNARY_OP( atan_f, double, double, atan ); - EXPORT_UNARY_OP( atan_i, int64_t, double, atan ); - EXPORT_UNARY_OP( sinh_f, double, double, sinh ); - EXPORT_UNARY_OP( sinh_i, int64_t, double, sinh ); - EXPORT_UNARY_OP( cosh_f, double, double, cosh ); - EXPORT_UNARY_OP( cosh_i, int64_t, double, cosh ); - EXPORT_UNARY_OP( tanh_f, double, double, tanh ); - EXPORT_UNARY_OP( tanh_i, int64_t, double, tanh ); - EXPORT_UNARY_OP( asinh_f, double, double, asinh ); - EXPORT_UNARY_OP( asinh_i, int64_t, double, asinh ); - EXPORT_UNARY_OP( acosh_f, double, double, acosh ); - EXPORT_UNARY_OP( acosh_i, int64_t, double, acosh ); - EXPORT_UNARY_OP( atanh_f, double, double, atanh ); - EXPORT_UNARY_OP( atanh_i, int64_t, double, atanh ); - EXPORT_UNARY_OP( not_, bool, bool, not_ ); - EXPORT_UNARY_OP( bitwise_not, int64_t, int64_t, bitwise_not ); +template +inline T _abs( T x ) +{ + return std::abs( x ); +} +template +inline double _ln( T x ) +{ + return std::log( x ); +} +template +inline double _log2( T x ) +{ + return std::log2( x ); +} +template +inline double _log10( T x ) +{ + return std::log10( x ); +} +template +inline double _exp( T x ) +{ + return std::exp( x ); +} +template +inline double _exp2( T x ) +{ + return std::exp2( x ); +} +template +inline double _sqrt( T x ) +{ + return std::sqrt( x ); +} +template +inline double _erf( T x ) +{ + return std::erf( x ); +} +template +inline double _sin( T x ) +{ + return std::sin( x ); +} +template +inline double _cos( T x ) +{ + return std::cos( x ); +} +template +inline double _tan( T x ) +{ + return std::tan( x ); +} +template +inline double _asin( T x ) +{ + return std::asin( x ); +} +template +inline double _acos( T x ) +{ + return std::acos( x ); +} +template +inline double _atan( T x ) +{ + return std::atan( x ); +} +template +inline double _sinh( T x ) +{ + return std::sinh( x ); +} +template +inline double _cosh( T x ) +{ + return std::cosh( x ); +} +template +inline double _tanh( T x ) +{ + return std::tanh( x ); +} +template +inline double _asinh( T x ) +{ + return std::asinh( x ); +} +template +inline double _acosh( T x ) +{ + return std::acosh( x ); +} +template +inline double _atanh( T x ) +{ + return std::atanh( x ); +} + +inline bool _not_( bool x ) +{ + return !x; +} +inline int64_t _bitwise_not( int64_t x ) +{ + return ~x; +} + +#define EXPORT_UNARY_OP( Name, ArgType, OutType, Func ) \ + EXPORT_TEMPLATE_CPPNODE( Name, SINGLE_ARG( _unary_op ) ) +EXPORT_UNARY_OP( abs_f, double, double, abs ); +EXPORT_UNARY_OP( abs_i, int64_t, int64_t, abs ); +EXPORT_UNARY_OP( ln_f, double, double, ln ); +EXPORT_UNARY_OP( ln_i, int64_t, double, ln ); +EXPORT_UNARY_OP( log2_f, double, double, log2 ); +EXPORT_UNARY_OP( log2_i, int64_t, double, log2 ); +EXPORT_UNARY_OP( log10_f, double, double, log10 ); +EXPORT_UNARY_OP( log10_i, int64_t, double, log10 ); +EXPORT_UNARY_OP( exp_f, double, double, exp ); +EXPORT_UNARY_OP( exp_i, int64_t, double, exp ); +EXPORT_UNARY_OP( exp2_f, double, double, exp2 ); +EXPORT_UNARY_OP( exp2_i, int64_t, double, exp2 ); +EXPORT_UNARY_OP( sqrt_f, double, double, sqrt ); +EXPORT_UNARY_OP( sqrt_i, int64_t, double, sqrt ); +EXPORT_UNARY_OP( erf_f, double, double, erf ); +EXPORT_UNARY_OP( erf_i, int64_t, double, erf ); +EXPORT_UNARY_OP( sin_f, double, double, sin ); +EXPORT_UNARY_OP( sin_i, int64_t, double, sin ); +EXPORT_UNARY_OP( cos_f, double, double, cos ); +EXPORT_UNARY_OP( cos_i, int64_t, double, cos ); +EXPORT_UNARY_OP( tan_f, double, double, tan ); +EXPORT_UNARY_OP( tan_i, int64_t, double, tan ); +EXPORT_UNARY_OP( asin_f, double, double, asin ); +EXPORT_UNARY_OP( asin_i, int64_t, double, asin ); +EXPORT_UNARY_OP( acos_f, double, double, acos ); +EXPORT_UNARY_OP( acos_i, int64_t, double, acos ); +EXPORT_UNARY_OP( atan_f, double, double, atan ); +EXPORT_UNARY_OP( atan_i, int64_t, double, atan ); +EXPORT_UNARY_OP( sinh_f, double, double, sinh ); +EXPORT_UNARY_OP( sinh_i, int64_t, double, sinh ); +EXPORT_UNARY_OP( cosh_f, double, double, cosh ); +EXPORT_UNARY_OP( cosh_i, int64_t, double, cosh ); +EXPORT_UNARY_OP( tanh_f, double, double, tanh ); +EXPORT_UNARY_OP( tanh_i, int64_t, double, tanh ); +EXPORT_UNARY_OP( asinh_f, double, double, asinh ); +EXPORT_UNARY_OP( asinh_i, int64_t, double, asinh ); +EXPORT_UNARY_OP( acosh_f, double, double, acosh ); +EXPORT_UNARY_OP( acosh_i, int64_t, double, acosh ); +EXPORT_UNARY_OP( atanh_f, double, double, atanh ); +EXPORT_UNARY_OP( atanh_i, int64_t, double, atanh ); +EXPORT_UNARY_OP( not_, bool, bool, not_ ); +EXPORT_UNARY_OP( bitwise_not, int64_t, int64_t, bitwise_not ); #undef EXPORT_UNARY_OP // Binary operation -template +template DECLARE_CPPNODE( _binary_op ) { TS_INPUT( ArgT, x ); TS_INPUT( ArgT, y ); TS_OUTPUT( OutT ); - //Expanded out INIT_CPPNODE without create call... - CSP csp; - const char * name() const override { return "_binary_op"; } + // Expanded out INIT_CPPNODE without create call... + CSP csp; + const char * name() const override + { + return "_binary_op"; + } public: _STATIC_CREATE_METHOD( SINGLE_ARG( _binary_op ) ); - _binary_op( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _binary_op( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } INVOKE() { @@ -126,50 +222,105 @@ DECLARE_CPPNODE( _binary_op ) }; // Math ops -template inline T _add( T x, T y ){ return x + y; } -template inline T _sub( T x, T y ){ return x - y; } -template inline T _mul( T x, T y ){ return x * y; } -template inline T _max( T x, T y ){ return std::max( x, y ); } -template inline T _min( T x, T y ){ return std::min( x, y ); } -template inline double _div( T x, T y ){ return x / ( double )y; } -inline int64_t _pow_i( int64_t x, int64_t y ){ return ( int64_t )pow( ( double )x, ( double )y ); } -inline double _pow_f( double x, double y ){ return pow( x, y ); } +template +inline T _add( T x, T y ) +{ + return x + y; +} +template +inline T _sub( T x, T y ) +{ + return x - y; +} +template +inline T _mul( T x, T y ) +{ + return x * y; +} +template +inline T _max( T x, T y ) +{ + return std::max( x, y ); +} +template +inline T _min( T x, T y ) +{ + return std::min( x, y ); +} +template +inline double _div( T x, T y ) +{ + return x / (double)y; +} +inline int64_t _pow_i( int64_t x, int64_t y ) +{ + return (int64_t)pow( (double)x, (double)y ); +} +inline double _pow_f( double x, double y ) +{ + return pow( x, y ); +} // Comparison ops -template inline bool _eq( T x, T y ){ return x == y; } -template inline bool _ne( T x, T y ){ return x != y; } -template inline bool _gt( T x, T y ){ return x > y; } -template inline bool _ge( T x, T y ){ return x >= y; } -template inline bool _lt( T x, T y ){ return x < y; } -template inline bool _le( T x, T y ){ return x <= y; } - -#define EXPORT_BINARY_OP( Name, ArgType, OutType, Func ) EXPORT_TEMPLATE_CPPNODE( Name, SINGLE_ARG( _binary_op ) ) - EXPORT_BINARY_OP( add_i, int64_t, int64_t, _add ); - EXPORT_BINARY_OP( sub_i, int64_t, int64_t, _sub ); - EXPORT_BINARY_OP( mul_i, int64_t, int64_t, _mul ); - EXPORT_BINARY_OP( div_i, int64_t, double, _div ); - EXPORT_BINARY_OP( pow_i, int64_t, int64_t, _pow_i ); - EXPORT_BINARY_OP( max_i, int64_t, int64_t, _max ); - EXPORT_BINARY_OP( min_i, int64_t, int64_t, _min ); - EXPORT_BINARY_OP( add_f, double, double, _add ); - EXPORT_BINARY_OP( sub_f, double, double, _sub ); - EXPORT_BINARY_OP( mul_f, double, double, _mul ); - EXPORT_BINARY_OP( div_f, double, double, _div ); - EXPORT_BINARY_OP( pow_f, double, double, _pow_f ); - EXPORT_BINARY_OP( max_f, double, double, _max ); - EXPORT_BINARY_OP( min_f, double, double, _min ); - EXPORT_BINARY_OP( eq_i, int64_t, bool, _eq ); - EXPORT_BINARY_OP( ne_i, int64_t, bool, _ne ); - EXPORT_BINARY_OP( gt_i, int64_t, bool, _gt ); - EXPORT_BINARY_OP( ge_i, int64_t, bool, _ge ); - EXPORT_BINARY_OP( lt_i, int64_t, bool, _lt ); - EXPORT_BINARY_OP( le_i, int64_t, bool, _le ); - EXPORT_BINARY_OP( eq_f, double, bool, _eq ); - EXPORT_BINARY_OP( ne_f, double, bool, _ne ); - EXPORT_BINARY_OP( gt_f, double, bool, _gt ); - EXPORT_BINARY_OP( ge_f, double, bool, _ge ); - EXPORT_BINARY_OP( lt_f, double, bool, _lt ); - EXPORT_BINARY_OP( le_f, double, bool, _le ); +template +inline bool _eq( T x, T y ) +{ + return x == y; +} +template +inline bool _ne( T x, T y ) +{ + return x != y; +} +template +inline bool _gt( T x, T y ) +{ + return x > y; +} +template +inline bool _ge( T x, T y ) +{ + return x >= y; +} +template +inline bool _lt( T x, T y ) +{ + return x < y; +} +template +inline bool _le( T x, T y ) +{ + return x <= y; +} + +#define EXPORT_BINARY_OP( Name, ArgType, OutType, Func ) \ + EXPORT_TEMPLATE_CPPNODE( Name, SINGLE_ARG( _binary_op ) ) +EXPORT_BINARY_OP( add_i, int64_t, int64_t, _add ); +EXPORT_BINARY_OP( sub_i, int64_t, int64_t, _sub ); +EXPORT_BINARY_OP( mul_i, int64_t, int64_t, _mul ); +EXPORT_BINARY_OP( div_i, int64_t, double, _div ); +EXPORT_BINARY_OP( pow_i, int64_t, int64_t, _pow_i ); +EXPORT_BINARY_OP( max_i, int64_t, int64_t, _max ); +EXPORT_BINARY_OP( min_i, int64_t, int64_t, _min ); +EXPORT_BINARY_OP( add_f, double, double, _add ); +EXPORT_BINARY_OP( sub_f, double, double, _sub ); +EXPORT_BINARY_OP( mul_f, double, double, _mul ); +EXPORT_BINARY_OP( div_f, double, double, _div ); +EXPORT_BINARY_OP( pow_f, double, double, _pow_f ); +EXPORT_BINARY_OP( max_f, double, double, _max ); +EXPORT_BINARY_OP( min_f, double, double, _min ); +EXPORT_BINARY_OP( eq_i, int64_t, bool, _eq ); +EXPORT_BINARY_OP( ne_i, int64_t, bool, _ne ); +EXPORT_BINARY_OP( gt_i, int64_t, bool, _gt ); +EXPORT_BINARY_OP( ge_i, int64_t, bool, _ge ); +EXPORT_BINARY_OP( lt_i, int64_t, bool, _lt ); +EXPORT_BINARY_OP( le_i, int64_t, bool, _le ); +EXPORT_BINARY_OP( eq_f, double, bool, _eq ); +EXPORT_BINARY_OP( ne_f, double, bool, _ne ); +EXPORT_BINARY_OP( gt_f, double, bool, _gt ); +EXPORT_BINARY_OP( ge_f, double, bool, _ge ); +EXPORT_BINARY_OP( lt_f, double, bool, _lt ); +EXPORT_BINARY_OP( le_f, double, bool, _le ); #undef EXPORT_BINARY_OP -} +} // namespace csp::cppnodes diff --git a/cpp/csp/cppnodes/statsimpl.cpp b/cpp/csp/cppnodes/statsimpl.cpp index ac68f3354..2ab10f6c9 100644 --- a/cpp/csp/cppnodes/statsimpl.cpp +++ b/cpp/csp/cppnodes/statsimpl.cpp @@ -9,13 +9,10 @@ class _tick_window_updates : public _generic_tick_window_updates::_generic_tick_window_updates; _STATIC_CREATE_METHOD( _tick_window_updates ); - inline double createNan() - { - return std::numeric_limits::quiet_NaN(); - } + inline double createNan() { return std::numeric_limits::quiet_NaN(); } - inline void validateShape() { } - inline void checkValid() { } + inline void validateShape() {} + inline void checkValid() {} }; EXPORT_CPPNODE( _tick_window_updates ); @@ -26,13 +23,10 @@ class _time_window_updates : public _generic_time_window_updates::_generic_time_window_updates; _STATIC_CREATE_METHOD( _time_window_updates ); - inline double createNan() - { - return std::numeric_limits::quiet_NaN(); - } + inline double createNan() { return std::numeric_limits::quiet_NaN(); } - inline void validateShape() { } - inline void checkValid() { } + inline void validateShape() {} + inline void checkValid() {} }; EXPORT_CPPNODE( _time_window_updates ); @@ -43,10 +37,7 @@ class _cross_sectional_as_list : public _generic_cross_sectional, _cross_sectional_as_list>::_generic_cross_sectional; _STATIC_CREATE_METHOD( _cross_sectional_as_list ); - inline void computeCrossSectional() - { - s_window.copy_values( &unnamed_output().reserveSpace() ); - } + inline void computeCrossSectional() { s_window.copy_values( &unnamed_output().reserveSpace() ); } }; EXPORT_CPPNODE( _cross_sectional_as_list ); @@ -65,7 +56,7 @@ DECLARE_CPPNODE( _min_hit_by_tick ) TS_OUTPUT( bool ); - INIT_CPPNODE( _min_hit_by_tick ) { } + INIT_CPPNODE( _min_hit_by_tick ) {} START() { @@ -97,7 +88,7 @@ DECLARE_CPPNODE( _in_sequence_check ) TS_INPUT( Generic, x ); TS_INPUT( Generic, y ); - INIT_CPPNODE( _in_sequence_check ) { } + INIT_CPPNODE( _in_sequence_check ) {} INVOKE() { @@ -121,7 +112,7 @@ DECLARE_CPPNODE( _sync_nan_f ) TS_NAMED_OUTPUT( double, x_sync ); TS_NAMED_OUTPUT( double, y_sync ); - INIT_CPPNODE( _sync_nan_f ) { } + INIT_CPPNODE( _sync_nan_f ) {} INVOKE() { @@ -159,13 +150,18 @@ DECLARE_CPPNODE( _compute ) STATE_VAR( DataValidator, s_computation ); TS_OUTPUT( double ); - //Expanded out INIT_CPPNODE without create call... - CSP csp; - const char * name() const override { return "_compute"; } + // Expanded out INIT_CPPNODE without create call... + CSP csp; + const char * name() const override + { + return "_compute"; + } public: - _compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } START() { @@ -182,12 +178,12 @@ DECLARE_CPPNODE( _compute ) } if( csp.ticked( additions ) ) { - for( double x: additions.lastValue() ) + for( double x : additions.lastValue() ) s_computation.add( x ); } if( csp.ticked( removals ) ) { - for( double x: removals.lastValue() ) + for( double x : removals.lastValue() ) s_computation.remove( x ); } if( csp.ticked( trigger ) ) @@ -206,7 +202,7 @@ class _computeCommonArgs : public _compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na ); + validator = DataValidator( this->min_data_points, this->ignore_na ); } }; @@ -220,7 +216,7 @@ class _computeOneArg : public _compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na, this -> arg ); + validator = DataValidator( this->min_data_points, this->ignore_na, this->arg ); } }; @@ -235,7 +231,7 @@ class _computeTwoArg : public _compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na, this -> arg1, this -> arg2 ); + validator = DataValidator( this->min_data_points, this->ignore_na, this->arg1, this->arg2 ); } }; @@ -251,32 +247,32 @@ class _computeEMA : public _compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, true, alpha, this -> ignore_na, horizon, adjust ); + validator = DataValidator( this->min_data_points, true, alpha, this->ignore_na, horizon, adjust ); } }; // Export node templates -EXPORT_TEMPLATE_CPPNODE( _count, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _sum, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _kahan_sum, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _mean, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _first, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _prod, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _last, _computeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _unique, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _min_max, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _var, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _sem, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _skew, SINGLE_ARG( _computeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _rank, SINGLE_ARG( _computeTwoArg ) ); -EXPORT_TEMPLATE_CPPNODE( _kurt, SINGLE_ARG( _computeTwoArg ) ); -EXPORT_TEMPLATE_CPPNODE( _ema_compute, _computeEMA ); -EXPORT_TEMPLATE_CPPNODE( _ema_adjusted, _computeEMA); +EXPORT_TEMPLATE_CPPNODE( _count, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _sum, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _kahan_sum, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _mean, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _first, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _prod, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _last, _computeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _unique, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _min_max, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _var, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _sem, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _skew, SINGLE_ARG( _computeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _rank, SINGLE_ARG( _computeTwoArg ) ); +EXPORT_TEMPLATE_CPPNODE( _kurt, SINGLE_ARG( _computeTwoArg ) ); +EXPORT_TEMPLATE_CPPNODE( _ema_compute, _computeEMA ); +EXPORT_TEMPLATE_CPPNODE( _ema_adjusted, _computeEMA ); EXPORT_TEMPLATE_CPPNODE( _ema_debias_alpha, _computeEMA ); - // The following nodes are written independently from _compute -// They either have an additional input (i.e. ddof for covariance) or are implemented differently (i.e. weighted mean, which needs weight inputs) +// They either have an additional input (i.e. ddof for covariance) or are implemented differently (i.e. weighted mean, +// which needs weight inputs) /* Computation node for statistics requiring an int argument @@ -301,13 +297,18 @@ DECLARE_CPPNODE( _bivariate_compute ) TS_OUTPUT( double ); - //Expanded out INIT_CPPNODE without create call... - CSP csp; - const char * name() const override { return "_bivariate_compute"; } + // Expanded out INIT_CPPNODE without create call... + CSP csp; + const char * name() const override + { + return "_bivariate_compute"; + } public: - _bivariate_compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _bivariate_compute( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } START() { @@ -333,7 +334,7 @@ DECLARE_CPPNODE( _bivariate_compute ) { const std::vector & removals_x = x_rem.lastValue(); const std::vector & removals_y = y_rem.lastValue(); - for ( size_t i = 0; i < removals_x.size(); i++ ) + for( size_t i = 0; i < removals_x.size(); i++ ) s_computation.remove( removals_x[i], removals_y[i] ); } if( csp.ticked( trigger ) ) @@ -352,7 +353,7 @@ class _bivarComputeCommonArgs : public _bivariate_compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na ); + validator = DataValidator( this->min_data_points, this->ignore_na ); } }; @@ -366,7 +367,7 @@ class _bivarComputeOneArg : public _bivariate_compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na, this -> arg ); + validator = DataValidator( this->min_data_points, this->ignore_na, this->arg ); } }; @@ -381,18 +382,17 @@ class _bivarComputeTwoArg : public _bivariate_compute void initDataValidator( DataValidator & validator ) override { - validator = DataValidator( this -> min_data_points, this -> ignore_na, this -> arg1, this -> arg2 ); + validator = DataValidator( this->min_data_points, this->ignore_na, this->arg1, this->arg2 ); } }; -EXPORT_TEMPLATE_CPPNODE( _weighted_mean, _bivarComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _corr, _bivarComputeCommonArgs ); -EXPORT_TEMPLATE_CPPNODE( _weighted_var, SINGLE_ARG( _bivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _weighted_sem, SINGLE_ARG( _bivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _covar, SINGLE_ARG( _bivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _weighted_skew, SINGLE_ARG( _bivarComputeOneArg ) ); -EXPORT_TEMPLATE_CPPNODE( _weighted_kurt, SINGLE_ARG( _bivarComputeTwoArg ) ); - +EXPORT_TEMPLATE_CPPNODE( _weighted_mean, _bivarComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _corr, _bivarComputeCommonArgs ); +EXPORT_TEMPLATE_CPPNODE( _weighted_var, SINGLE_ARG( _bivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _weighted_sem, SINGLE_ARG( _bivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _covar, SINGLE_ARG( _bivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _weighted_skew, SINGLE_ARG( _bivarComputeOneArg ) ); +EXPORT_TEMPLATE_CPPNODE( _weighted_kurt, SINGLE_ARG( _bivarComputeTwoArg ) ); // Trivariate and multivariate statistics template @@ -414,7 +414,7 @@ DECLARE_CPPNODE( _trivariate_compute ) TS_OUTPUT( double ); - INIT_CPPNODE( _trivariate_compute ) { } + INIT_CPPNODE( _trivariate_compute ) {} START() { @@ -454,9 +454,9 @@ DECLARE_CPPNODE( _trivariate_compute ) }; EXPORT_TEMPLATE_CPPNODE( _weighted_covar, _trivariate_compute ); -EXPORT_TEMPLATE_CPPNODE( _weighted_corr, _trivariate_compute ); +EXPORT_TEMPLATE_CPPNODE( _weighted_corr, _trivariate_compute ); -DECLARE_CPPNODE ( _quantile ) +DECLARE_CPPNODE( _quantile ) { TS_INPUT( std::vector, additions ); TS_INPUT( std::vector, removals ); @@ -471,7 +471,7 @@ DECLARE_CPPNODE ( _quantile ) TS_LISTBASKET_OUTPUT( double ); - INIT_CPPNODE( _quantile ) { } + INIT_CPPNODE( _quantile ) {} START() { @@ -486,24 +486,23 @@ DECLARE_CPPNODE ( _quantile ) } if( csp.ticked( additions ) ) { - for( double x: additions.lastValue() ) + for( double x : additions.lastValue() ) s_qtl.add( x ); } if( csp.ticked( removals ) ) { - for ( double x: removals.lastValue() ) + for( double x : removals.lastValue() ) s_qtl.remove( x ); } if( csp.ticked( trigger ) ) { - for ( size_t i = 0; i < quants.value().size(); i++ ) + for( size_t i = 0; i < quants.value().size(); i++ ) unnamed_output()[i].output( s_qtl.compute( i ) ); } } - }; -EXPORT_CPPNODE ( _quantile ); +EXPORT_CPPNODE( _quantile ); template DECLARE_CPPNODE( _exp_timewise ) @@ -518,7 +517,7 @@ DECLARE_CPPNODE( _exp_timewise ) STATE_VAR( DataValidator, s_computation ); TS_OUTPUT( double ); - INIT_CPPNODE( _exp_timewise ) { } + INIT_CPPNODE( _exp_timewise ) {} START() { @@ -527,7 +526,7 @@ DECLARE_CPPNODE( _exp_timewise ) INVOKE() { - if ( csp.ticked( reset ) ) + if( csp.ticked( reset ) ) { s_computation.reset(); } @@ -544,8 +543,8 @@ DECLARE_CPPNODE( _exp_timewise ) } }; -EXPORT_TEMPLATE_CPPNODE( _ema_timewise, _exp_timewise ); -EXPORT_TEMPLATE_CPPNODE( _ema_debias_halflife, _exp_timewise ); +EXPORT_TEMPLATE_CPPNODE( _ema_timewise, _exp_timewise ); +EXPORT_TEMPLATE_CPPNODE( _ema_debias_halflife, _exp_timewise ); DECLARE_CPPNODE( _arg_min_max ) { @@ -562,7 +561,7 @@ DECLARE_CPPNODE( _arg_min_max ) STATE_VAR( DataValidator, s_computation ); TS_OUTPUT( DateTime ); - INIT_CPPNODE( _arg_min_max ) { } + INIT_CPPNODE( _arg_min_max ) {} START() { @@ -583,7 +582,7 @@ DECLARE_CPPNODE( _arg_min_max ) if( csp.ticked( removals ) ) { - for( double v: removals.lastValue() ) + for( double v : removals.lastValue() ) s_computation.remove( v ); } @@ -596,4 +595,4 @@ DECLARE_CPPNODE( _arg_min_max ) EXPORT_CPPNODE( _arg_min_max ); -} +} // namespace csp::cppnodes diff --git a/cpp/csp/cppnodes/statsimpl.h b/cpp/csp/cppnodes/statsimpl.h index dc8b9bf96..88a8ce17a 100644 --- a/cpp/csp/cppnodes/statsimpl.h +++ b/cpp/csp/cppnodes/statsimpl.h @@ -4,13 +4,13 @@ #include #include +#include +#include +#include #include #include #include #include -#include -#include -#include namespace csp::cppnodes { @@ -23,595 +23,495 @@ static constexpr double EPSILON = 1e-9; class Count { - public: - Count() - { - reset(); - } +public: + Count() { reset(); } - void add( double x ) - { - m_count++; - } + void add( double x ) { m_count++; } - void remove( double x ) - { - m_count--; - } + void remove( double x ) { m_count--; } - void reset() - { - m_count = 0; - } + void reset() { m_count = 0; } - double compute() const - { - return static_cast( m_count ); - } + double compute() const { return static_cast( m_count ); } - private: - int64_t m_count; +private: + int64_t m_count; }; class Sum { - public: - Sum() - { - reset(); - } +public: + Sum() { reset(); } - Sum( Sum && rhs ) = default; + Sum( Sum && rhs ) = default; - Sum & operator=( Sum && rhs ) = default; + Sum & operator=( Sum && rhs ) = default; - Sum & operator=( const Sum & rhs ) = delete; + Sum & operator=( const Sum & rhs ) = delete; - void add( double x ) - { - m_sum += x; - } + void add( double x ) { m_sum += x; } - void remove( double x ) - { - m_sum -= x; - } + void remove( double x ) { m_sum -= x; } - void reset() - { - m_sum = 0; - } + void reset() { m_sum = 0; } - double compute() const - { - return m_sum; - } + double compute() const { return m_sum; } - private: - double m_sum; +private: + double m_sum; }; class KahanSum { - public: - KahanSum() - { - reset(); - } - - void add( double x ) - { - double y = x - m_comp; - double tmp_sum = m_sum + y; - m_comp = tmp_sum - m_sum - y; - m_sum = tmp_sum; - } +public: + KahanSum() { reset(); } - void remove( double x ) - { - // same as adding negative - double y = -x - m_comp; - double tmp_sum = m_sum + y; - m_comp = tmp_sum - m_sum - y; - m_sum = tmp_sum; - } + void add( double x ) + { + double y = x - m_comp; + double tmp_sum = m_sum + y; + m_comp = tmp_sum - m_sum - y; + m_sum = tmp_sum; + } - void reset() - { - m_sum = m_comp = 0; - } + void remove( double x ) + { + // same as adding negative + double y = -x - m_comp; + double tmp_sum = m_sum + y; + m_comp = tmp_sum - m_sum - y; + m_sum = tmp_sum; + } - double compute() const - { - return m_sum; - } + void reset() { m_sum = m_comp = 0; } - private: - double m_sum; - double m_comp; + double compute() const { return m_sum; } +private: + double m_sum; + double m_comp; }; class Mean { - public: - Mean() - { - reset(); - } +public: + Mean() { reset(); } - void add( double x ) - { - m_count++; - m_mean += ( x - m_mean ) / m_count; - } + void add( double x ) + { + m_count++; + m_mean += ( x - m_mean ) / m_count; + } - void remove( double x ) - { - m_count--; - if( m_count > 0 ) - m_mean += ( m_mean - x ) / m_count; - else - m_mean = 0; - } + void remove( double x ) + { + m_count--; + if( m_count > 0 ) + m_mean += ( m_mean - x ) / m_count; + else + m_mean = 0; + } - void reset() - { - m_mean = m_count = 0; - } + void reset() { m_mean = m_count = 0; } - double compute() const - { - if( m_count > 0 ) - return m_mean; - return std::numeric_limits::quiet_NaN(); - } + double compute() const + { + if( m_count > 0 ) + return m_mean; + return std::numeric_limits::quiet_NaN(); + } - private: - double m_mean; - int64_t m_count; +private: + double m_mean; + int64_t m_count; }; class First { - public: - void add( double x ) - { - m_value_buffer.push( x ); - } +public: + void add( double x ) { m_value_buffer.push( x ); } - void remove( double x ) - { - m_value_buffer.pop_left(); - } + void remove( double x ) { m_value_buffer.pop_left(); } - void reset() - { - m_value_buffer.clear(); - } + void reset() { m_value_buffer.clear(); } - double compute() + double compute() + { + if( !m_value_buffer.empty() ) { - if( !m_value_buffer.empty() ) - { - return m_value_buffer[-1]; - } - return std::numeric_limits::quiet_NaN(); + return m_value_buffer[-1]; } + return std::numeric_limits::quiet_NaN(); + } - private: - VariableSizeWindowBuffer m_value_buffer; +private: + VariableSizeWindowBuffer m_value_buffer; }; class Last { - public: - Last() - { - reset(); - } +public: + Last() { reset(); } - void add( double x ) - { - m_last = x; - m_count++; - } + void add( double x ) + { + m_last = x; + m_count++; + } - void remove( double x ) - { - m_count--; - } + void remove( double x ) { m_count--; } - void reset() - { - m_count = 0; - } + void reset() { m_count = 0; } - double compute() - { - if( m_count > 0 ) - return m_last; - return std::numeric_limits::quiet_NaN(); - } + double compute() + { + if( m_count > 0 ) + return m_last; + return std::numeric_limits::quiet_NaN(); + } - private: - double m_last; - double m_count; +private: + double m_last; + double m_count; }; class Unique { - public: - - Unique() - { - m_powPrecision = 1; - } - - Unique( int64_t precision ) - { - m_powPrecision = pow( 10.0, ( double ) precision ); - } +public: + Unique() { m_powPrecision = 1; } - void add( double x ) - { - int64_t rounded_result = ( int64_t )( x * m_powPrecision ); - m_dict[rounded_result]++; - } + Unique( int64_t precision ) { m_powPrecision = pow( 10.0, (double)precision ); } - void remove( double x ) - { - int64_t rounded_result = ( int64_t )( x * m_powPrecision ); - auto it = m_dict.find( rounded_result ); - if( it -> second == 1 ) - m_dict.erase( rounded_result ); - else - it -> second--; - } + void add( double x ) + { + int64_t rounded_result = (int64_t)( x * m_powPrecision ); + m_dict[rounded_result]++; + } - void reset() - { - m_dict.clear(); - } + void remove( double x ) + { + int64_t rounded_result = (int64_t)( x * m_powPrecision ); + auto it = m_dict.find( rounded_result ); + if( it->second == 1 ) + m_dict.erase( rounded_result ); + else + it->second--; + } - double compute() const - { - return static_cast( m_dict.size() ); - } + void reset() { m_dict.clear(); } - private: + double compute() const { return static_cast( m_dict.size() ); } - std::unordered_map m_dict; - double m_powPrecision; +private: + std::unordered_map m_dict; + double m_powPrecision; }; class Product { - public: - Product() - { - reset(); - } +public: + Product() { reset(); } - void add( double x ) - { - m_count++; - if( x == 0 ) - m_nzero++; - else - m_prod *= x; - } + void add( double x ) + { + m_count++; + if( x == 0 ) + m_nzero++; + else + m_prod *= x; + } - void remove( double x ) - { - m_count--; - if( x == 0 ) - m_nzero--; - else - m_prod /= x; - } + void remove( double x ) + { + m_count--; + if( x == 0 ) + m_nzero--; + else + m_prod /= x; + } - void reset() - { - m_prod = 1.0; - m_count = m_nzero = 0; - } + void reset() + { + m_prod = 1.0; + m_count = m_nzero = 0; + } - double compute() const + double compute() const + { + if( m_count > 0 ) { - if( m_count > 0 ) - { - if( m_nzero == 0 ) - return m_prod; - else - return 0; - } + if( m_nzero == 0 ) + return m_prod; else - return std::numeric_limits::quiet_NaN(); + return 0; } + else + return std::numeric_limits::quiet_NaN(); + } - private: - - double m_prod; - int64_t m_count; - int64_t m_nzero; +private: + double m_prod; + int64_t m_count; + int64_t m_nzero; }; class WeightedMean { - public: - WeightedMean() - { - reset(); - } - - void add( double x, double w ) - { - m_weight += w; - if( m_weight > EPSILON ) - m_wmean += ( x * w - w * m_wmean ) / m_weight; - } +public: + WeightedMean() { reset(); } - void remove( double x, double w ) - { - m_weight -= w; - if( m_weight > EPSILON ) - m_wmean -= ( x * w - w * m_wmean ) / m_weight; - else - m_wmean = m_weight = 0; - } + void add( double x, double w ) + { + m_weight += w; + if( m_weight > EPSILON ) + m_wmean += ( x * w - w * m_wmean ) / m_weight; + } - void reset() - { + void remove( double x, double w ) + { + m_weight -= w; + if( m_weight > EPSILON ) + m_wmean -= ( x * w - w * m_wmean ) / m_weight; + else m_wmean = m_weight = 0; - } + } + + void reset() { m_wmean = m_weight = 0; } - double compute() const + double compute() const + { + if( m_weight > EPSILON ) { - if( m_weight > EPSILON ) - { - return m_wmean; - } - return std::numeric_limits::quiet_NaN(); + return m_wmean; } + return std::numeric_limits::quiet_NaN(); + } - private: - double m_wmean; - double m_weight; +private: + double m_wmean; + double m_weight; }; class Variance { - public: - Variance() - { - m_ddof = 1; - reset(); - } - - Variance( int64_t ddof ) : Variance() - { - m_ddof = ddof; - } +public: + Variance() + { + m_ddof = 1; + reset(); + } - void add( double x ) - { - m_count++; - m_dx = x - m_mean; - m_mean += m_dx / m_count; - m_unnormVar += ( x - m_mean ) * m_dx; - } + Variance( int64_t ddof ) + : Variance() + { + m_ddof = ddof; + } - void remove( double x ) - { - m_count--; - if( m_count == 0 ) - { - m_mean = m_unnormVar = 0; - return; - } - m_dx = x - m_mean; - m_mean -= m_dx / m_count; - m_unnormVar -= ( x - m_mean ) * m_dx; - } + void add( double x ) + { + m_count++; + m_dx = x - m_mean; + m_mean += m_dx / m_count; + m_unnormVar += ( x - m_mean ) * m_dx; + } - void reset() + void remove( double x ) + { + m_count--; + if( m_count == 0 ) { - m_mean = m_unnormVar = m_count = 0; + m_mean = m_unnormVar = 0; + return; } + m_dx = x - m_mean; + m_mean -= m_dx / m_count; + m_unnormVar -= ( x - m_mean ) * m_dx; + } - double compute() const - { - if( m_count > m_ddof ) - return ( m_unnormVar < 0 ? 0 : m_unnormVar / ( m_count - m_ddof ) ); + void reset() { m_mean = m_unnormVar = m_count = 0; } - return std::numeric_limits::quiet_NaN(); - } + double compute() const + { + if( m_count > m_ddof ) + return ( m_unnormVar < 0 ? 0 : m_unnormVar / ( m_count - m_ddof ) ); - private: + return std::numeric_limits::quiet_NaN(); + } - double m_mean; - double m_unnormVar; - double m_dx; - double m_count; - int64_t m_ddof; +private: + double m_mean; + double m_unnormVar; + double m_dx; + double m_count; + int64_t m_ddof; }; class WeightedVariance { - public: - WeightedVariance() - { - m_ddof = 1; - reset(); - } - - WeightedVariance( int64_t ddof ) : WeightedVariance() - { - m_ddof = ddof; - } +public: + WeightedVariance() + { + m_ddof = 1; + reset(); + } - void add( double x, double w ) - { - if( w <= 0 ) - return; - m_wsum += w; - m_dx = x - m_wmean; - m_wmean += ( w / m_wsum ) * m_dx; - m_unnormWVar += w * ( x - m_wmean ) * m_dx; - } + WeightedVariance( int64_t ddof ) + : WeightedVariance() + { + m_ddof = ddof; + } - void remove( double x, double w ) - { - m_wsum -= w; - if( m_wsum < EPSILON ) - { - m_wsum = m_wmean = m_unnormWVar = 0; - return; - } - m_dx = x - m_wmean; - m_wmean -= ( w / m_wsum ) * m_dx; - m_unnormWVar -= w * ( x - m_wmean ) * m_dx; - } + void add( double x, double w ) + { + if( w <= 0 ) + return; + m_wsum += w; + m_dx = x - m_wmean; + m_wmean += ( w / m_wsum ) * m_dx; + m_unnormWVar += w * ( x - m_wmean ) * m_dx; + } - void reset() + void remove( double x, double w ) + { + m_wsum -= w; + if( m_wsum < EPSILON ) { m_wsum = m_wmean = m_unnormWVar = 0; + return; } + m_dx = x - m_wmean; + m_wmean -= ( w / m_wsum ) * m_dx; + m_unnormWVar -= w * ( x - m_wmean ) * m_dx; + } - double compute() const - { - if( m_wsum > m_ddof ) - return ( m_unnormWVar < 0 ? 0 : m_unnormWVar / ( m_wsum - m_ddof ) ); + void reset() { m_wsum = m_wmean = m_unnormWVar = 0; } - return std::numeric_limits::quiet_NaN(); - } + double compute() const + { + if( m_wsum > m_ddof ) + return ( m_unnormWVar < 0 ? 0 : m_unnormWVar / ( m_wsum - m_ddof ) ); - private: + return std::numeric_limits::quiet_NaN(); + } - double m_wsum; - double m_wmean; - double m_unnormWVar; - double m_dx; - int64_t m_ddof; +private: + double m_wsum; + double m_wmean; + double m_unnormWVar; + double m_dx; + int64_t m_ddof; }; class Covariance { - public: - Covariance() - { - m_ddof = 1; - reset(); - } +public: + Covariance() + { + m_ddof = 1; + reset(); + } - Covariance( int64_t ddof ) : Covariance() - { - m_ddof = ddof; - } + Covariance( int64_t ddof ) + : Covariance() + { + m_ddof = ddof; + } - void add( double x, double y ) - { - m_count++; - m_dx = x - m_mux; - m_mux += m_dx / m_count; - m_muy += ( y - m_muy ) / m_count; - m_unnormCov += m_dx * ( y - m_muy ); - } + void add( double x, double y ) + { + m_count++; + m_dx = x - m_mux; + m_mux += m_dx / m_count; + m_muy += ( y - m_muy ) / m_count; + m_unnormCov += m_dx * ( y - m_muy ); + } - void remove( double x, double y ) + void remove( double x, double y ) + { + m_count--; + if( m_count == 0 ) { - m_count--; - if( m_count == 0 ) - { - m_mux = m_muy = m_unnormCov = 0; - return; - } - - m_dx = x - m_mux; - m_mux -= m_dx / m_count; - m_muy -= ( y - m_muy ) / m_count; - m_unnormCov -= m_dx * ( y - m_muy ); + m_mux = m_muy = m_unnormCov = 0; + return; } - void reset() - { - m_mux = m_muy = m_unnormCov = m_count = 0; - } + m_dx = x - m_mux; + m_mux -= m_dx / m_count; + m_muy -= ( y - m_muy ) / m_count; + m_unnormCov -= m_dx * ( y - m_muy ); + } - double compute() const - { - return ( m_count > m_ddof ? m_unnormCov / ( m_count - m_ddof ) : std::numeric_limits::quiet_NaN() ); - } + void reset() { m_mux = m_muy = m_unnormCov = m_count = 0; } - private: + double compute() const + { + return ( m_count > m_ddof ? m_unnormCov / ( m_count - m_ddof ) : std::numeric_limits::quiet_NaN() ); + } - double m_mux; - double m_muy; - double m_unnormCov; - double m_dx; - double m_count; - int64_t m_ddof; +private: + double m_mux; + double m_muy; + double m_unnormCov; + double m_dx; + double m_count; + int64_t m_ddof; }; class WeightedCovariance { - public: - WeightedCovariance() - { - m_ddof = 1; - reset(); - } - - WeightedCovariance( int64_t ddof ) : WeightedCovariance() - { - m_ddof = ddof; - } +public: + WeightedCovariance() + { + m_ddof = 1; + reset(); + } - void add( double x, double y, double w ) - { - if( w <= 0 ) - return; - m_wsum += w; - m_dx = x - m_wmux; - m_wmux += ( w / m_wsum ) * m_dx; - m_wmuy += ( w / m_wsum ) * ( y - m_wmuy ); - m_unnormWCov += w * m_dx * ( y - m_wmuy ); - } + WeightedCovariance( int64_t ddof ) + : WeightedCovariance() + { + m_ddof = ddof; + } - void remove( double x, double y, double w ) - { - m_wsum -= w; - if( m_wsum < EPSILON ) - { - m_wsum = m_wmux = m_wmuy = m_unnormWCov = 0; - return; - } - m_dx = x - m_wmux; - m_wmux -= ( w / m_wsum ) * m_dx; - m_wmuy -= ( w / m_wsum ) * ( y - m_wmuy ); - m_unnormWCov -= w * m_dx * ( y - m_wmuy ); - } + void add( double x, double y, double w ) + { + if( w <= 0 ) + return; + m_wsum += w; + m_dx = x - m_wmux; + m_wmux += ( w / m_wsum ) * m_dx; + m_wmuy += ( w / m_wsum ) * ( y - m_wmuy ); + m_unnormWCov += w * m_dx * ( y - m_wmuy ); + } - void reset() + void remove( double x, double y, double w ) + { + m_wsum -= w; + if( m_wsum < EPSILON ) { - m_wmux = m_wmuy = m_unnormWCov = m_wsum = 0; + m_wsum = m_wmux = m_wmuy = m_unnormWCov = 0; + return; } + m_dx = x - m_wmux; + m_wmux -= ( w / m_wsum ) * m_dx; + m_wmuy -= ( w / m_wsum ) * ( y - m_wmuy ); + m_unnormWCov -= w * m_dx * ( y - m_wmuy ); + } - double compute() const - { - return ( m_wsum > m_ddof ? m_unnormWCov / ( m_wsum - m_ddof ) : std::numeric_limits::quiet_NaN() ); - } + void reset() { m_wmux = m_wmuy = m_unnormWCov = m_wsum = 0; } - private: + double compute() const + { + return ( m_wsum > m_ddof ? m_unnormWCov / ( m_wsum - m_ddof ) : std::numeric_limits::quiet_NaN() ); + } - double m_wmux; - double m_wmuy; - double m_unnormWCov; - double m_dx; +private: + double m_wmux; + double m_wmuy; + double m_unnormWCov; + double m_dx; - double m_wsum; - int64_t m_ddof; + double m_wsum; + int64_t m_ddof; }; double corrCompute( double cov, double vx, double vy ) @@ -623,190 +523,186 @@ double corrCompute( double cov, double vx, double vy ) class Correlation { - public: - Correlation() - { - reset(); - } +public: + Correlation() { reset(); } - Correlation( int64_t arg ) : Correlation() { } + Correlation( int64_t arg ) + : Correlation() + { + } - void add( double x, double y ) - { - m_cov.add( x, y ); - m_vx.add( x ); - m_vy.add( y ); - } + void add( double x, double y ) + { + m_cov.add( x, y ); + m_vx.add( x ); + m_vy.add( y ); + } - void remove( double x, double y ) - { - m_cov.remove( x, y ); - m_vx.remove( x ); - m_vy.remove( y ); - } + void remove( double x, double y ) + { + m_cov.remove( x, y ); + m_vx.remove( x ); + m_vy.remove( y ); + } - void reset() - { - m_cov.reset(); - m_vx.reset(); - m_vy.reset(); - } + void reset() + { + m_cov.reset(); + m_vx.reset(); + m_vy.reset(); + } - double compute() const - { - return corrCompute( m_cov.compute(), m_vx.compute(), m_vy.compute() ); - } + double compute() const { return corrCompute( m_cov.compute(), m_vx.compute(), m_vy.compute() ); } - private: - Covariance m_cov; - Variance m_vx; - Variance m_vy; +private: + Covariance m_cov; + Variance m_vx; + Variance m_vy; }; class WeightedCorrelation { - public: - WeightedCorrelation() - { - reset(); - } - - WeightedCorrelation( int64_t arg ) : WeightedCorrelation() { } +public: + WeightedCorrelation() { reset(); } - void add( double x, double y, double w ) - { - m_cov.add( x, y, w ); - m_vx.add( x, w ); - m_vy.add( y, w ); - } + WeightedCorrelation( int64_t arg ) + : WeightedCorrelation() + { + } - void remove( double x, double y, double w ) - { - m_cov.remove( x, y, w ); - m_vx.remove( x, w ); - m_vy.remove( y, w ); - } + void add( double x, double y, double w ) + { + m_cov.add( x, y, w ); + m_vx.add( x, w ); + m_vy.add( y, w ); + } - void reset() - { - m_cov.reset(); - m_vx.reset(); - m_vy.reset(); - } + void remove( double x, double y, double w ) + { + m_cov.remove( x, y, w ); + m_vx.remove( x, w ); + m_vy.remove( y, w ); + } - double compute() const - { - return corrCompute( m_cov.compute(), m_vx.compute(), m_vy.compute() ); - } + void reset() + { + m_cov.reset(); + m_vx.reset(); + m_vy.reset(); + } + + double compute() const { return corrCompute( m_cov.compute(), m_vx.compute(), m_vy.compute() ); } - private: - WeightedCovariance m_cov; - WeightedVariance m_vx; - WeightedVariance m_vy; +private: + WeightedCovariance m_cov; + WeightedVariance m_vx; + WeightedVariance m_vy; }; class StandardError { - public: - StandardError() - { - m_ddof = 1; - reset(); - } +public: + StandardError() + { + m_ddof = 1; + reset(); + } - StandardError( int64_t ddof ) : StandardError() - { - m_ddof = ddof; - m_var = Variance( ddof ); - } + StandardError( int64_t ddof ) + : StandardError() + { + m_ddof = ddof; + m_var = Variance( ddof ); + } - void add( double x ) - { - m_count++; - m_var.add( x ); - } + void add( double x ) + { + m_count++; + m_var.add( x ); + } - void remove( double x ) - { - m_count--; - m_var.remove( x ); - } + void remove( double x ) + { + m_count--; + m_var.remove( x ); + } - void reset() - { - m_count = 0; - m_var.reset(); - } + void reset() + { + m_count = 0; + m_var.reset(); + } - double compute() const - { - return ( m_count > m_ddof ? sqrt( m_var.compute() / ( m_count - m_ddof ) ) : std::numeric_limits::quiet_NaN() ); - } + double compute() const + { + return ( m_count > m_ddof ? sqrt( m_var.compute() / ( m_count - m_ddof ) ) + : std::numeric_limits::quiet_NaN() ); + } - private: - Variance m_var; - int64_t m_ddof; - double m_count; +private: + Variance m_var; + int64_t m_ddof; + double m_count; }; class WeightedStandardError { - public: - WeightedStandardError() - { - m_ddof = 1; - reset(); - } - - WeightedStandardError( int64_t ddof ) : WeightedStandardError() - { - m_ddof = ddof; - m_var = WeightedVariance( ddof ); - } +public: + WeightedStandardError() + { + m_ddof = 1; + reset(); + } - void add( double x, double w ) - { - m_wsum += w; - m_var.add( x, w ); - } + WeightedStandardError( int64_t ddof ) + : WeightedStandardError() + { + m_ddof = ddof; + m_var = WeightedVariance( ddof ); + } - void remove( double x, double w ) - { - m_wsum -= w; - if( m_wsum < EPSILON ) - m_wsum = 0; - m_var.remove( x, w ); - } + void add( double x, double w ) + { + m_wsum += w; + m_var.add( x, w ); + } - void reset() - { + void remove( double x, double w ) + { + m_wsum -= w; + if( m_wsum < EPSILON ) m_wsum = 0; - m_var.reset(); - } - - double compute() const - { - return ( m_wsum > m_ddof && m_wsum > EPSILON ? sqrt( m_var.compute() / ( m_wsum - m_ddof ) ) : std::numeric_limits::quiet_NaN() ); - } + m_var.remove( x, w ); + } - private: - WeightedVariance m_var; - int64_t m_ddof; - double m_wsum; -}; + void reset() + { + m_wsum = 0; + m_var.reset(); + } + double compute() const + { + return ( m_wsum > m_ddof && m_wsum > EPSILON ? sqrt( m_var.compute() / ( m_wsum - m_ddof ) ) + : std::numeric_limits::quiet_NaN() ); + } +private: + WeightedVariance m_var; + int64_t m_ddof; + double m_wsum; +}; double skewCompute( double count, double mx, double mx3, double vx, bool bias ) { if( count <= 2 || vx < EPSILON ) - return std::numeric_limits::quiet_NaN(); + return std::numeric_limits::quiet_NaN(); double bias_skew = ( mx3 - 3 * mx * vx - mx * mx * mx ) / ( vx * sqrt( vx ) ); if( bias ) return bias_skew; - double factor = sqrt( count * ( count - 1 ) ) / ( count- 2 ); + double factor = sqrt( count * ( count - 1 ) ) / ( count - 2 ); return factor * bias_skew; } @@ -820,7 +716,7 @@ double kurtCompute( double count, double mx, double mx2, double mx3, double mx4, if( bias ) { if( excess ) - return bias_kurt-3; + return bias_kurt - 3; return bias_kurt; } @@ -830,340 +726,333 @@ double kurtCompute( double count, double mx, double mx2, double mx3, double mx4, ub_kurt -= 3 * gfactor; if( !excess ) ub_kurt += 3; - + return ub_kurt; } class Skew { - public: - Skew() - { - m_vx = Variance( 0 ); - m_bias = false; - reset(); - } - - Skew( bool bias ) : Skew() - { - m_bias = bias; - } +public: + Skew() + { + m_vx = Variance( 0 ); + m_bias = false; + reset(); + } - void add( double x ) - { - m_count++; - m_mx.add( x ); - m_mx3.add( x*x*x ); - m_vx.add( x ); - } + Skew( bool bias ) + : Skew() + { + m_bias = bias; + } - void remove( double x ) - { - m_count--; - m_mx.remove( x ); - m_mx3.remove( x*x*x ); - m_vx.remove( x ); - } + void add( double x ) + { + m_count++; + m_mx.add( x ); + m_mx3.add( x * x * x ); + m_vx.add( x ); + } - void reset() - { - m_mx.reset(); - m_mx3.reset(); - m_vx.reset(); - m_count = 0; - } + void remove( double x ) + { + m_count--; + m_mx.remove( x ); + m_mx3.remove( x * x * x ); + m_vx.remove( x ); + } - double compute() const - { - return skewCompute( m_count, m_mx.compute(), m_mx3.compute(), m_vx.compute(), m_bias ); - } + void reset() + { + m_mx.reset(); + m_mx3.reset(); + m_vx.reset(); + m_count = 0; + } - private: + double compute() const { return skewCompute( m_count, m_mx.compute(), m_mx3.compute(), m_vx.compute(), m_bias ); } - Mean m_mx; - Mean m_mx3; - Variance m_vx; - double m_count; - bool m_bias; +private: + Mean m_mx; + Mean m_mx3; + Variance m_vx; + double m_count; + bool m_bias; }; class WeightedSkew { - public: - WeightedSkew() - { - m_vx = WeightedVariance( 0 ); - m_bias = false; - reset(); - } - - WeightedSkew( bool bias ) : WeightedSkew() - { - m_bias = bias; - } +public: + WeightedSkew() + { + m_vx = WeightedVariance( 0 ); + m_bias = false; + reset(); + } - void add( double x, double w ) - { - m_count++; - m_mx.add( x, w ); - m_mx3.add( x*x*x, w ); - m_vx.add( x, w ); - } + WeightedSkew( bool bias ) + : WeightedSkew() + { + m_bias = bias; + } - void remove( double x, double w ) - { - m_count--; - m_mx.remove( x, w ); - m_mx3.remove( x*x*x, w ); - m_vx.remove( x, w ); - } + void add( double x, double w ) + { + m_count++; + m_mx.add( x, w ); + m_mx3.add( x * x * x, w ); + m_vx.add( x, w ); + } - void reset() - { - m_mx.reset(); - m_mx3.reset(); - m_vx.reset(); - m_count = 0; - } + void remove( double x, double w ) + { + m_count--; + m_mx.remove( x, w ); + m_mx3.remove( x * x * x, w ); + m_vx.remove( x, w ); + } - double compute() const - { - return skewCompute( m_count, m_mx.compute(), m_mx3.compute(), m_vx.compute(), m_bias ); - } + void reset() + { + m_mx.reset(); + m_mx3.reset(); + m_vx.reset(); + m_count = 0; + } - private: + double compute() const { return skewCompute( m_count, m_mx.compute(), m_mx3.compute(), m_vx.compute(), m_bias ); } - WeightedMean m_mx; - WeightedMean m_mx3; - WeightedVariance m_vx; - double m_count; - bool m_bias; +private: + WeightedMean m_mx; + WeightedMean m_mx3; + WeightedVariance m_vx; + double m_count; + bool m_bias; }; class Kurtosis { - public: - Kurtosis() - { - m_vx = Variance( 0 ); - m_bias = false; - m_excess = true; - reset(); - } - - Kurtosis( bool bias, bool excess ) : Kurtosis() - { - m_bias = bias; - m_excess = excess; - } - - void add( double x ) - { - m_count++; - double val = x; - m_mx.add( val ); - m_vx.add( val ); - val *= x; - m_mx2.add( val ); - val *= x; - m_mx3.add( val ); - val *= x; - m_mx4.add( val ); +public: + Kurtosis() + { + m_vx = Variance( 0 ); + m_bias = false; + m_excess = true; + reset(); + } - } + Kurtosis( bool bias, bool excess ) + : Kurtosis() + { + m_bias = bias; + m_excess = excess; + } - void remove( double x ) - { - m_count--; - double val = x; - m_mx.remove( val ); - m_vx.remove( val ); - val *= x; - m_mx2.remove( val ); - val *= x; - m_mx3.remove( val ); - val *= x; - m_mx4.remove( val ); - } + void add( double x ) + { + m_count++; + double val = x; + m_mx.add( val ); + m_vx.add( val ); + val *= x; + m_mx2.add( val ); + val *= x; + m_mx3.add( val ); + val *= x; + m_mx4.add( val ); + } - void reset() - { - m_mx.reset(); - m_mx2.reset(); - m_mx3.reset(); - m_mx4.reset(); - m_vx.reset(); - m_count = 0; - } + void remove( double x ) + { + m_count--; + double val = x; + m_mx.remove( val ); + m_vx.remove( val ); + val *= x; + m_mx2.remove( val ); + val *= x; + m_mx3.remove( val ); + val *= x; + m_mx4.remove( val ); + } - double compute() const - { - return kurtCompute( m_count, m_mx.compute(), m_mx2.compute(), m_mx3.compute(), m_mx4.compute(), m_vx.compute(), m_bias, m_excess ); - } + void reset() + { + m_mx.reset(); + m_mx2.reset(); + m_mx3.reset(); + m_mx4.reset(); + m_vx.reset(); + m_count = 0; + } - private: + double compute() const + { + return kurtCompute( m_count, m_mx.compute(), m_mx2.compute(), m_mx3.compute(), m_mx4.compute(), m_vx.compute(), + m_bias, m_excess ); + } - Mean m_mx; - Mean m_mx2; - Mean m_mx3; - Mean m_mx4; - Variance m_vx; - double m_count; - bool m_bias; - bool m_excess; +private: + Mean m_mx; + Mean m_mx2; + Mean m_mx3; + Mean m_mx4; + Variance m_vx; + double m_count; + bool m_bias; + bool m_excess; }; class WeightedKurtosis { - public: - WeightedKurtosis() - { - m_vx = WeightedVariance( 0 ); - m_bias = false; - m_excess = true; - reset(); - } - - WeightedKurtosis( bool bias, bool excess ) : WeightedKurtosis() - { - m_bias = bias; - m_excess = excess; - } +public: + WeightedKurtosis() + { + m_vx = WeightedVariance( 0 ); + m_bias = false; + m_excess = true; + reset(); + } - void add( double x, double w ) - { - m_count++; - double val = x; - m_mx.add( val, w ); - m_vx.add( val, w ); - val *= x; - m_mx2.add( val, w ); - val *= x; - m_mx3.add( val, w ); - val *= x; - m_mx4.add( val, w ); - } + WeightedKurtosis( bool bias, bool excess ) + : WeightedKurtosis() + { + m_bias = bias; + m_excess = excess; + } - void remove( double x, double w ) - { - m_count--; - double val = x; - m_mx.remove( val, w ); - m_vx.remove( val, w ); - val *= x; - m_mx2.remove( val, w ); - val *= x; - m_mx3.remove( val, w ); - val *= x; - m_mx4.remove( val, w ); - } + void add( double x, double w ) + { + m_count++; + double val = x; + m_mx.add( val, w ); + m_vx.add( val, w ); + val *= x; + m_mx2.add( val, w ); + val *= x; + m_mx3.add( val, w ); + val *= x; + m_mx4.add( val, w ); + } - void reset() - { - m_mx.reset(); - m_mx2.reset(); - m_mx3.reset(); - m_mx4.reset(); - m_vx.reset(); - m_count = 0; - } + void remove( double x, double w ) + { + m_count--; + double val = x; + m_mx.remove( val, w ); + m_vx.remove( val, w ); + val *= x; + m_mx2.remove( val, w ); + val *= x; + m_mx3.remove( val, w ); + val *= x; + m_mx4.remove( val, w ); + } - double compute() const - { - return kurtCompute( m_count, m_mx.compute(), m_mx2.compute(), m_mx3.compute(), m_mx4.compute(), m_vx.compute(), m_bias, m_excess ); - } + void reset() + { + m_mx.reset(); + m_mx2.reset(); + m_mx3.reset(); + m_mx4.reset(); + m_vx.reset(); + m_count = 0; + } - private: + double compute() const + { + return kurtCompute( m_count, m_mx.compute(), m_mx2.compute(), m_mx3.compute(), m_mx4.compute(), m_vx.compute(), + m_bias, m_excess ); + } - WeightedMean m_mx; - WeightedMean m_mx2; - WeightedMean m_mx3; - WeightedMean m_mx4; - WeightedVariance m_vx; - double m_count; - bool m_bias; - bool m_excess; +private: + WeightedMean m_mx; + WeightedMean m_mx2; + WeightedMean m_mx3; + WeightedMean m_mx4; + WeightedVariance m_vx; + double m_count; + bool m_bias; + bool m_excess; }; -template -using ost = boost::multi_index::multi_index_container, Comparator>>>; +template +using ost + = boost::multi_index::multi_index_container, Comparator>>>; class Quantile { enum Interpolate { - LINEAR, LOWER, HIGHER, MIDPOINT, NEAREST + LINEAR, + LOWER, + HIGHER, + MIDPOINT, + NEAREST }; - public: - Quantile( const std::vector & quants, int64_t interpolation ) - { - m_quants = quants; - m_interpolation = interpolation; - } +public: + Quantile( const std::vector & quants, int64_t interpolation ) + { + m_quants = quants; + m_interpolation = interpolation; + } - Quantile( Quantile && rhs ) - { - m_quants = rhs.m_quants; - m_interpolation = rhs.m_interpolation; - m_tree = std::move( rhs.m_tree ); - } + Quantile( Quantile && rhs ) + { + m_quants = rhs.m_quants; + m_interpolation = rhs.m_interpolation; + m_tree = std::move( rhs.m_tree ); + } - Quantile & operator=( Quantile && rhs ) - { - m_quants = rhs.m_quants; - m_interpolation = rhs.m_interpolation; - m_tree = std::move( rhs.m_tree ); + Quantile & operator=( Quantile && rhs ) + { + m_quants = rhs.m_quants; + m_interpolation = rhs.m_interpolation; + m_tree = std::move( rhs.m_tree ); - return *this; - } + return *this; + } - Quantile() = default; - Quantile( const Quantile & rhs ) = delete; - Quantile & operator=( const Quantile & rhs ) = delete; + Quantile() = default; + Quantile( const Quantile & rhs ) = delete; + Quantile & operator=( const Quantile & rhs ) = delete; - void add( double x ) - { - m_tree.insert( x ); - } + void add( double x ) { m_tree.insert( x ); } - void remove( double x ) - { - m_tree.erase( m_tree.find( x ) ); - } + void remove( double x ) { m_tree.erase( m_tree.find( x ) ); } - void reset() - { - m_tree.clear(); - } + void reset() { m_tree.clear(); } - double compute( int index ) const + double compute( int index ) const + { + // Compute which values to find + if( m_tree.size() == 0 ) { - // Compute which values to find - if( m_tree.size() == 0 ) - { - return std::numeric_limits::quiet_NaN(); - } + return std::numeric_limits::quiet_NaN(); + } - double target = std::get( m_quants[index]._data ) * ( m_tree.size() - 1 ); - int ft = floor( target ); - int ct = ceil( target ); - auto fIt = m_tree.get<0>().nth( ft ); - auto cIt = ( ft == ct ) ? fIt : std::next( fIt ); + double target = std::get( m_quants[index]._data ) * ( m_tree.size() - 1 ); + int ft = floor( target ); + int ct = ceil( target ); + auto fIt = m_tree.get<0>().nth( ft ); + auto cIt = ( ft == ct ) ? fIt : std::next( fIt ); - double qtl = 0.0; - switch ( m_interpolation ) - { + double qtl = 0.0; + switch( m_interpolation ) + { case LINEAR: - if ( ft == target ) + if( ft == target ) { qtl = *fIt; } else { - double lower = *fIt; + double lower = *fIt; double higher = *cIt; - qtl = ( 1 - target + ft ) * lower + ( 1 - ct + target ) * higher; + qtl = ( 1 - target + ft ) * lower + ( 1 - ct + target ) * higher; } break; case LOWER: @@ -1173,19 +1062,19 @@ class Quantile qtl = *cIt; break; case MIDPOINT: - if ( ft == target ) + if( ft == target ) { qtl = *fIt; } else { - double lower = *fIt; + double lower = *fIt; double higher = *cIt; - qtl = ( higher + lower ) / 2; + qtl = ( higher + lower ) / 2; } break; case NEAREST: - if ( target - ft < ct - target ) + if( target - ft < ct - target ) { qtl = *fIt; } @@ -1196,24 +1085,27 @@ class Quantile break; default: break; - } - return qtl; } + return qtl; + } - private: - ost> m_tree; - std::vector m_quants; - int64_t m_interpolation; +private: + ost> m_tree; + std::vector m_quants; + int64_t m_interpolation; }; class AscendingMinima { public: - AscendingMinima( bool max ) : m_max( max ) {} - AscendingMinima( AscendingMinima && rhs ) = default; + AscendingMinima( bool max ) + : m_max( max ) + { + } + AscendingMinima( AscendingMinima && rhs ) = default; AscendingMinima & operator=( AscendingMinima && rhs ) = default; - AscendingMinima() = default; + AscendingMinima() = default; AscendingMinima( const AscendingMinima & rhs ) = delete; void add( double x ) @@ -1233,10 +1125,7 @@ class AscendingMinima } } - void reset() - { - m_value_buffer.clear(); - } + void reset() { m_value_buffer.clear(); } double compute() { @@ -1248,7 +1137,7 @@ class AscendingMinima } private: - bool m_max; + bool m_max; VariableSizeWindowBuffer m_value_buffer; }; @@ -1256,647 +1145,642 @@ class Rank { enum RankMethod { - MIN, MAX, AVG + MIN, + MAX, + AVG }; enum NanOption { - KEEP, LAST + KEEP, + LAST }; - public: - Rank() = default; +public: + Rank() = default; - Rank( int64_t method, int64_t nanopt ) - { - m_method = method; - m_nanopt = nanopt; - } + Rank( int64_t method, int64_t nanopt ) + { + m_method = method; + m_nanopt = nanopt; + } - // no copy as always - Rank( Rank && rhs ) = default; - Rank & operator=( Rank && rhs ) = default; - Rank( const Rank & rhs ) = delete; - Rank & operator=( const Rank & rhs ) = delete; + // no copy as always + Rank( Rank && rhs ) = default; + Rank & operator=( Rank && rhs ) = default; + Rank( const Rank & rhs ) = delete; + Rank & operator=( const Rank & rhs ) = delete; - void add( double x ) + void add( double x ) + { + if( unlikely( isnan( x ) ) ) { - if( unlikely( isnan( x ) ) ) - { - if( m_nanopt == KEEP ) - m_lastval = std::numeric_limits::quiet_NaN(); - } - else - { - m_lastval = x; - if( m_method == MAX ) - m_maxtree.insert( x ); - else - m_mintree.insert( x ); - } + if( m_nanopt == KEEP ) + m_lastval = std::numeric_limits::quiet_NaN(); } - - void remove( double x ) + else { - if( likely( !isnan( x ) ) ) - { - if ( m_method == MAX ) - m_maxtree.erase ( m_maxtree.find( x ) ); - else - m_mintree.erase ( m_mintree.find( x ) ); - } + m_lastval = x; + if( m_method == MAX ) + m_maxtree.insert( x ); + else + m_mintree.insert( x ); } + } - void reset() + void remove( double x ) + { + if( likely( !isnan( x ) ) ) { if( m_method == MAX ) - m_maxtree.clear(); + m_maxtree.erase( m_maxtree.find( x ) ); else - m_mintree.clear(); + m_mintree.erase( m_mintree.find( x ) ); } + } - double compute() const + void reset() + { + if( m_method == MAX ) + m_maxtree.clear(); + else + m_mintree.clear(); + } + + double compute() const + { + // Verify tree is not empty and lastValue is valid + // Last value can only ever be NaN if the "keep" nan option is used + if( likely( !isnan( m_lastval ) && ( ( m_method == MAX && m_maxtree.size() > 0 ) || m_mintree.size() > 0 ) ) ) { - // Verify tree is not empty and lastValue is valid - // Last value can only ever be NaN if the "keep" nan option is used - if( likely( !isnan( m_lastval ) && ( ( m_method == MAX && m_maxtree.size() > 0 ) || m_mintree.size() > 0 ) ) ) + switch( m_method ) { - switch( m_method ) + case MIN: + { + if( m_mintree.size() == 1 ) + return 0; + return m_mintree.get<0>().find_rank( m_lastval ); + } + case MAX: + { + if( m_maxtree.size() == 1 ) + return 0; + return m_maxtree.size() - 1 - m_maxtree.get<0>().find_rank( m_lastval ); + } + case AVG: { - case MIN: - { - if ( m_mintree.size() == 1 ) - return 0; - return m_mintree.get<0>().find_rank( m_lastval ); - } - case MAX: - { - if ( m_maxtree.size() == 1 ) - return 0; - return m_maxtree.size() - 1 - m_maxtree.get<0>().find_rank( m_lastval ); - } - case AVG: - { - if ( m_mintree.size() == 1 ) - return 0; - - int min_rank = m_mintree.get<0>().find_rank( m_lastval ); - int max_rank = min_rank; - auto it = m_mintree.get<0>().nth( min_rank ); - it++; - for( ; it != m_mintree.end() && *it == m_lastval ; it++ ) max_rank++; // While this is in theory O(n), in reality this loop is only interated once, since there are likely no duplicate values or very few. - return ( double )( min_rank + max_rank ) / 2; - } - default: - break; + if( m_mintree.size() == 1 ) + return 0; + + int min_rank = m_mintree.get<0>().find_rank( m_lastval ); + int max_rank = min_rank; + auto it = m_mintree.get<0>().nth( min_rank ); + it++; + for( ; it != m_mintree.end() && *it == m_lastval; it++ ) + max_rank++; // While this is in theory O(n), in reality this loop is only interated once, since + // there are likely no duplicate values or very few. + return (double)( min_rank + max_rank ) / 2; } + default: + break; } - return std::numeric_limits::quiet_NaN(); } + return std::numeric_limits::quiet_NaN(); + } - private: - ost> m_mintree; - ost> m_maxtree; - double m_lastval = std::numeric_limits::quiet_NaN(); +private: + ost> m_mintree; + ost> m_maxtree; + double m_lastval = std::numeric_limits::quiet_NaN(); - int64_t m_method; - int64_t m_nanopt; + int64_t m_method; + int64_t m_nanopt; }; class ArgMinMax { - public: - ArgMinMax() = default; - ArgMinMax( bool max, bool recent ) : m_recent( recent ), m_monoQueue( max ) {} +public: + ArgMinMax() = default; + ArgMinMax( bool max, bool recent ) + : m_recent( recent ) + , m_monoQueue( max ) + { + } - ArgMinMax( ArgMinMax && rhs ) = default; - ArgMinMax & operator=( ArgMinMax && rhs ) = default; + ArgMinMax( ArgMinMax && rhs ) = default; + ArgMinMax & operator=( ArgMinMax && rhs ) = default; - // no copy - ArgMinMax( const ArgMinMax & rhs ) = delete; - ArgMinMax & operator=( const ArgMinMax & rhs ) = delete; + // no copy + ArgMinMax( const ArgMinMax & rhs ) = delete; + ArgMinMax & operator=( const ArgMinMax & rhs ) = delete; - void add( double x, DateTime t ) - { - m_monoQueue.add( x ); - auto & it = m_treemap[x]; - it.m_count++; - if( m_recent ) - it.m_lasttime = t; - else - it.m_alltimes.push( t ); - } + void add( double x, DateTime t ) + { + m_monoQueue.add( x ); + auto & it = m_treemap[x]; + it.m_count++; + if( m_recent ) + it.m_lasttime = t; + else + it.m_alltimes.push( t ); + } - void remove( double x ) - { - m_monoQueue.remove( x ); - auto it = m_treemap.find( x ); - it->second.m_count--; - if( !it->second.m_count ) // don't let map grow unbounded - m_treemap.erase( it ); - else if( !m_recent ) - it->second.m_alltimes.pop_left(); - } + void remove( double x ) + { + m_monoQueue.remove( x ); + auto it = m_treemap.find( x ); + it->second.m_count--; + if( !it->second.m_count ) // don't let map grow unbounded + m_treemap.erase( it ); + else if( !m_recent ) + it->second.m_alltimes.pop_left(); + } - void reset() - { - m_monoQueue.reset(); - m_treemap.clear(); - } + void reset() + { + m_monoQueue.reset(); + m_treemap.clear(); + } - DateTime compute() + DateTime compute() + { + if( m_treemap.size() > 0 ) { - if( m_treemap.size() > 0 ) - { - double arg_val = m_monoQueue.compute(); - if( m_recent ) - return m_treemap[arg_val].m_lasttime; - else - return m_treemap[arg_val].m_alltimes[-1]; - } - - return DateTime::fromNanoseconds( 0 ); + double arg_val = m_monoQueue.compute(); + if( m_recent ) + return m_treemap[arg_val].m_lasttime; + else + return m_treemap[arg_val].m_alltimes[-1]; } - private: - struct TreeData - { - TreeData & operator=( const TreeData & rhs ) = delete; - TreeData & operator=( TreeData && rhs ) = default; + return DateTime::fromNanoseconds( 0 ); + } - int m_count = 0; - DateTime m_lasttime; - VariableSizeWindowBuffer m_alltimes; - }; +private: + struct TreeData + { + TreeData & operator=( const TreeData & rhs ) = delete; + TreeData & operator=( TreeData && rhs ) = default; + + int m_count = 0; + DateTime m_lasttime; + VariableSizeWindowBuffer m_alltimes; + }; - bool m_recent; - AscendingMinima m_monoQueue; - std::map m_treemap; + bool m_recent; + AscendingMinima m_monoQueue; + std::map m_treemap; }; class EMA { - public: - EMA() = default; +public: + EMA() = default; - EMA( double alpha, bool ignore_na, int64_t horizon, bool adjust ) - { - m_alpha = alpha; - m_ignore_na = ignore_na; - reset(); - } + EMA( double alpha, bool ignore_na, int64_t horizon, bool adjust ) + { + m_alpha = alpha; + m_ignore_na = ignore_na; + reset(); + } - EMA( EMA && rhs ) = default; + EMA( EMA && rhs ) = default; - EMA & operator=( EMA && rhs ) = default; + EMA & operator=( EMA && rhs ) = default; - void add( double x ) + void add( double x ) + { + if( unlikely( m_first ) && !isnan( x ) ) { - if( unlikely( m_first ) && !isnan( x ) ) - { - m_ema = x; - m_first = false; - } - else if( unlikely( isnan( x ) ) && !m_ignore_na && likely( !m_first ) ) + m_ema = x; + m_first = false; + } + else if( unlikely( isnan( x ) ) && !m_ignore_na && likely( !m_first ) ) + { + m_offset++; + } + else if( likely( !isnan( x ) ) ) + { + double delta = x - m_ema; + if( m_offset == 1 ) { - m_offset++; + m_ema += m_alpha * delta; } - else if( likely( !isnan( x ) ) ) + else { - double delta = x - m_ema; - if( m_offset == 1 ) - { - m_ema += m_alpha * delta; - } - else - { - m_ema = ( m_ema * pow( ( 1 - m_alpha ), m_offset ) + m_alpha * x ) / - ( pow( 1 - m_alpha, m_offset ) + m_alpha ); - m_offset = 1; - } + m_ema = ( m_ema * pow( ( 1 - m_alpha ), m_offset ) + m_alpha * x ) + / ( pow( 1 - m_alpha, m_offset ) + m_alpha ); + m_offset = 1; } } + } - void remove( double x ) { } - - void reset() - { - m_ema = 0; - m_offset = 1; - m_first = true; - } + void remove( double x ) {} - double compute() const - { - return unlikely( m_first ) ? std::numeric_limits::quiet_NaN() : m_ema; - } + void reset() + { + m_ema = 0; + m_offset = 1; + m_first = true; + } - private: + double compute() const { return unlikely( m_first ) ? std::numeric_limits::quiet_NaN() : m_ema; } - double m_ema; - int64_t m_offset; - bool m_first; +private: + double m_ema; + int64_t m_offset; + bool m_first; - double m_alpha; - bool m_ignore_na; + double m_alpha; + bool m_ignore_na; }; class AdjustedEMA { - public: - AdjustedEMA() = default; - - AdjustedEMA( double alpha, bool ignore_na, int64_t horizon, bool adjust ) - { - m_decay = 1 - alpha; - m_ignore_na = ignore_na; - m_horizon = horizon; - reset(); - } +public: + AdjustedEMA() = default; - AdjustedEMA( AdjustedEMA && rhs ) = default; + AdjustedEMA( double alpha, bool ignore_na, int64_t horizon, bool adjust ) + { + m_decay = 1 - alpha; + m_ignore_na = ignore_na; + m_horizon = horizon; + reset(); + } - AdjustedEMA & operator=( AdjustedEMA && rhs ) = default; + AdjustedEMA( AdjustedEMA && rhs ) = default; - void add( double x ) - { - if( likely( !isnan( x ) ) ) - { - double decay_factor = ( m_ignore_na ? m_decay : pow( m_decay, m_offset ) ); - m_ema *= decay_factor; - m_norm *= decay_factor; - m_offset = 1; - m_ema += x; - m_norm++; - } - else - { - m_offset++; - m_nan_count++; - } - } + AdjustedEMA & operator=( AdjustedEMA && rhs ) = default; - void remove( double x ) + void add( double x ) + { + if( likely( !isnan( x ) ) ) { - if( likely( !isnan( x ) ) ) - { - double lookback = ( m_ignore_na ? m_horizon - m_nan_count : m_horizon - m_offset + 1 ); - double decay_factor = pow( m_decay, lookback ); - m_norm -= decay_factor; - m_ema -= x * decay_factor; - - // EMA may go to zero with a non-zero normalizer due to lots of 0 values in the time series: they are separate conditions - if( abs( m_ema ) < EPSILON ) - m_ema = 0; - } - else - m_nan_count--; + double decay_factor = ( m_ignore_na ? m_decay : pow( m_decay, m_offset ) ); + m_ema *= decay_factor; + m_norm *= decay_factor; + m_offset = 1; + m_ema += x; + m_norm++; } - - void reset() + else { - m_ema = m_norm = m_nan_count = 0; - m_offset = 1; + m_offset++; + m_nan_count++; } + } - double compute() const + void remove( double x ) + { + if( likely( !isnan( x ) ) ) { - if( m_norm > 0 ) // may be 0 if calc is triggered before any ticks are received - return m_ema / m_norm; + double lookback = ( m_ignore_na ? m_horizon - m_nan_count : m_horizon - m_offset + 1 ); + double decay_factor = pow( m_decay, lookback ); + m_norm -= decay_factor; + m_ema -= x * decay_factor; - return std::numeric_limits::quiet_NaN(); + // EMA may go to zero with a non-zero normalizer due to lots of 0 values in the time series: they are + // separate conditions + if( abs( m_ema ) < EPSILON ) + m_ema = 0; } + else + m_nan_count--; + } + + void reset() + { + m_ema = m_norm = m_nan_count = 0; + m_offset = 1; + } - private: + double compute() const + { + if( m_norm > 0 ) // may be 0 if calc is triggered before any ticks are received + return m_ema / m_norm; - double m_ema; - double m_norm; - int64_t m_offset; - double m_nan_count; + return std::numeric_limits::quiet_NaN(); + } - double m_decay; - bool m_ignore_na; - int64_t m_horizon; +private: + double m_ema; + double m_norm; + int64_t m_offset; + double m_nan_count; + + double m_decay; + bool m_ignore_na; + int64_t m_horizon; }; class AlphaDebiasEMA { - public: - AlphaDebiasEMA() = default; +public: + AlphaDebiasEMA() = default; - AlphaDebiasEMA( double alpha, bool ignore_na, int64_t horizon, bool adjust ) - { - m_decay = 1 - alpha; - m_ignore_na = ignore_na; - m_horizon = horizon; - m_adjust = adjust; - reset(); - } + AlphaDebiasEMA( double alpha, bool ignore_na, int64_t horizon, bool adjust ) + { + m_decay = 1 - alpha; + m_ignore_na = ignore_na; + m_horizon = horizon; + m_adjust = adjust; + reset(); + } - AlphaDebiasEMA( AlphaDebiasEMA && rhs ) = default; + AlphaDebiasEMA( AlphaDebiasEMA && rhs ) = default; - AlphaDebiasEMA & operator=( AlphaDebiasEMA && rhs ) = default; + AlphaDebiasEMA & operator=( AlphaDebiasEMA && rhs ) = default; - void add( double x ) + void add( double x ) + { + if( m_first && likely( !isnan( x ) ) ) { - if( m_first && likely( !isnan( x ) ) ) - { - m_wsum = 1; - m_sqsum = 1; - m_first = false; - } - else if( likely( !isnan( x ) ) ) - { - double decay_factor = ( m_ignore_na ? m_decay : pow( m_decay, m_offset ) ); - m_wsum *= decay_factor; - m_sqsum *= decay_factor * decay_factor; - m_offset = 1; - - double w0; - if( m_adjust ) - w0 = 1.0; - else - w0 = 1 - m_decay; - m_sqsum += w0 * w0; - m_wsum += w0; - if( !m_adjust ) - { - double correction = decay_factor + w0; - m_wsum /= correction; - m_sqsum /= ( correction * correction ); - } - } - else if ( likely( !m_first ) ) - { - m_offset++; - m_nan_count++; - } + m_wsum = 1; + m_sqsum = 1; + m_first = false; } - - void remove( double x ) + else if( likely( !isnan( x ) ) ) { - if( likely( !isnan( x ) )) + double decay_factor = ( m_ignore_na ? m_decay : pow( m_decay, m_offset ) ); + m_wsum *= decay_factor; + m_sqsum *= decay_factor * decay_factor; + m_offset = 1; + + double w0; + if( m_adjust ) + w0 = 1.0; + else + w0 = 1 - m_decay; + m_sqsum += w0 * w0; + m_wsum += w0; + if( !m_adjust ) { - double lookback = ( m_ignore_na ? m_horizon - m_nan_count : m_horizon - m_offset + 1 ); - double wh = pow( m_decay, lookback ); - if( !m_adjust ) - wh *= ( 1- m_decay ); - m_sqsum -= wh * wh; - m_wsum -= wh; - if( m_wsum < EPSILON || m_sqsum < EPSILON ) - { - m_wsum = 0; - m_sqsum = 0; - } + double correction = decay_factor + w0; + m_wsum /= correction; + m_sqsum /= ( correction * correction ); } - else - m_nan_count--; } - - void reset() + else if( likely( !m_first ) ) { - m_wsum = m_sqsum = m_nan_count = 0; - m_offset = 1; - m_first = true; + m_offset++; + m_nan_count++; } + } - double compute() const - { - double wsum_sq = m_wsum * m_wsum; - if( abs( wsum_sq - m_sqsum ) > EPSILON ) - return wsum_sq / ( wsum_sq - m_sqsum ); - else - return std::numeric_limits::quiet_NaN(); + void remove( double x ) + { + if( likely( !isnan( x ) ) ) + { + double lookback = ( m_ignore_na ? m_horizon - m_nan_count : m_horizon - m_offset + 1 ); + double wh = pow( m_decay, lookback ); + if( !m_adjust ) + wh *= ( 1 - m_decay ); + m_sqsum -= wh * wh; + m_wsum -= wh; + if( m_wsum < EPSILON || m_sqsum < EPSILON ) + { + m_wsum = 0; + m_sqsum = 0; + } } + else + m_nan_count--; + } - private: + void reset() + { + m_wsum = m_sqsum = m_nan_count = 0; + m_offset = 1; + m_first = true; + } - double m_wsum; - double m_sqsum; - int64_t m_offset; - double m_nan_count; - bool m_first; + double compute() const + { + double wsum_sq = m_wsum * m_wsum; + if( abs( wsum_sq - m_sqsum ) > EPSILON ) + return wsum_sq / ( wsum_sq - m_sqsum ); + else + return std::numeric_limits::quiet_NaN(); + } - double m_decay; - int64_t m_horizon; - bool m_ignore_na, m_adjust; +private: + double m_wsum; + double m_sqsum; + int64_t m_offset; + double m_nan_count; + bool m_first; + + double m_decay; + int64_t m_horizon; + bool m_ignore_na, m_adjust; }; class HalflifeEMA { - public: - HalflifeEMA() = default; - - HalflifeEMA( TimeDelta halflife, DateTime start ) - { - m_decay_factor = log( 0.5 ) / halflife.asNanoseconds(); - m_last_tick = start; - reset(); - } - - HalflifeEMA( HalflifeEMA && rhs ) = default; +public: + HalflifeEMA() = default; - HalflifeEMA & operator=( HalflifeEMA && rhs ) = default; + HalflifeEMA( TimeDelta halflife, DateTime start ) + { + m_decay_factor = log( 0.5 ) / halflife.asNanoseconds(); + m_last_tick = start; + reset(); + } - void add( double x, DateTime now ) - { - if( likely( !isnan( x ) ) ) - { - TimeDelta delta_t = now - m_last_tick; - double decay = exp( m_decay_factor * delta_t.asNanoseconds() ); - m_ema = decay * m_ema + x; - m_norm = decay * m_norm + 1.0; - m_last_tick = now; - } - } + HalflifeEMA( HalflifeEMA && rhs ) = default; - void reset() - { - m_ema = m_norm = 0; - } + HalflifeEMA & operator=( HalflifeEMA && rhs ) = default; - double compute() const + void add( double x, DateTime now ) + { + if( likely( !isnan( x ) ) ) { - return m_ema / m_norm; + TimeDelta delta_t = now - m_last_tick; + double decay = exp( m_decay_factor * delta_t.asNanoseconds() ); + m_ema = decay * m_ema + x; + m_norm = decay * m_norm + 1.0; + m_last_tick = now; } + } - private: + void reset() { m_ema = m_norm = 0; } - double m_ema; - double m_norm; - double m_decay_factor; - DateTime m_last_tick; + double compute() const { return m_ema / m_norm; } +private: + double m_ema; + double m_norm; + double m_decay_factor; + DateTime m_last_tick; }; class HalflifeDebiasEMA { - public: - HalflifeDebiasEMA() = default; - - HalflifeDebiasEMA( TimeDelta halflife, DateTime start ) - { - m_decay_factor = log( 0.5 ) / halflife.asNanoseconds(); - m_last_tick = start; - reset(); - } +public: + HalflifeDebiasEMA() = default; - HalflifeDebiasEMA( HalflifeDebiasEMA && rhs ) = default; + HalflifeDebiasEMA( TimeDelta halflife, DateTime start ) + { + m_decay_factor = log( 0.5 ) / halflife.asNanoseconds(); + m_last_tick = start; + reset(); + } - HalflifeDebiasEMA & operator=( HalflifeDebiasEMA && rhs ) = default; + HalflifeDebiasEMA( HalflifeDebiasEMA && rhs ) = default; - void add( double x, DateTime now ) - { - if( likely( !isnan( x ) ) ) - { - TimeDelta delta_t = now - m_last_tick; - double decay = exp( m_decay_factor * delta_t.asNanoseconds() ); - m_sqsum = decay * decay * m_sqsum + 1.0; - m_wsum = decay * m_wsum + 1.0; - m_last_tick = now; - } - } + HalflifeDebiasEMA & operator=( HalflifeDebiasEMA && rhs ) = default; - void reset() + void add( double x, DateTime now ) + { + if( likely( !isnan( x ) ) ) { - m_wsum = m_sqsum = 0; + TimeDelta delta_t = now - m_last_tick; + double decay = exp( m_decay_factor * delta_t.asNanoseconds() ); + m_sqsum = decay * decay * m_sqsum + 1.0; + m_wsum = decay * m_wsum + 1.0; + m_last_tick = now; } + } - double compute() const - { - double wsum_sq = m_wsum * m_wsum; - if( wsum_sq != m_sqsum ) - return wsum_sq / ( wsum_sq - m_sqsum ); - else - return std::numeric_limits::quiet_NaN(); - } + void reset() { m_wsum = m_sqsum = 0; } - private: + double compute() const + { + double wsum_sq = m_wsum * m_wsum; + if( wsum_sq != m_sqsum ) + return wsum_sq / ( wsum_sq - m_sqsum ); + else + return std::numeric_limits::quiet_NaN(); + } - double m_wsum; - double m_sqsum; - double m_decay_factor; - DateTime m_last_tick; +private: + double m_wsum; + double m_sqsum; + double m_decay_factor; + DateTime m_last_tick; }; // NaN handling generic code for the DataValidator class struct NanCheck { - template ::value, bool> = true> + template::value, bool> = true> static inline bool any_nan( FloatingT val, V... args ) { return isnan( val ) || any_nan( args... ); } - template ::value, bool> = false> + template::value, bool> = false> static inline bool any_nan( NonFloatingT val, V... args ) { return any_nan( args... ); } - static inline bool any_nan() - { - return false; - } + static inline bool any_nan() { return false; } }; // Validates min_data_points and takes care of NaN handling template class DataValidator { - public: - DataValidator() = default; +public: + DataValidator() = default; - DataValidator( DataValidator && rhs ) = default; + DataValidator( DataValidator && rhs ) = default; - DataValidator( const DataValidator & rhs ) = default; + DataValidator( const DataValidator & rhs ) = default; - DataValidator & operator=( DataValidator && rhs ) = default; + DataValidator & operator=( DataValidator && rhs ) = default; - DataValidator & operator=( const DataValidator & rhs ) = default; + DataValidator & operator=( const DataValidator & rhs ) = default; - template - DataValidator( int64_t min_data_points, bool ignore_na, V... args ) : - m_mdp( min_data_points ), m_igna( ignore_na ), m_stat ( args... ) { } + template + DataValidator( int64_t min_data_points, bool ignore_na, V... args ) + : m_mdp( min_data_points ) + , m_igna( ignore_na ) + , m_stat( args... ) + { + } - void add( double x ) + void add( double x ) + { + if( isnan( x ) ) { - if( isnan( x ) ) - { - m_nans++; - if( m_process_na || ( m_consider_na && !m_igna ) ) - m_stat.add( x ); - } - else - { - m_points++; + m_nans++; + if( m_process_na || ( m_consider_na && !m_igna ) ) m_stat.add( x ); - } } + else + { + m_points++; + m_stat.add( x ); + } + } - template - void add( V... args ) + template + void add( V... args ) + { + if( NanCheck::any_nan( args... ) ) + m_nans++; + else { - if( NanCheck::any_nan( args...) ) - m_nans++; - else - { - m_points++; - m_stat.add( args... ); - } + m_points++; + m_stat.add( args... ); } + } - template - void remove( V... args ) + template + void remove( V... args ) + { + if( NanCheck::any_nan( args... ) ) { - if( NanCheck::any_nan( args...) ) - { - m_nans--; - if( m_process_na || ( m_consider_na && !m_igna ) ) - m_stat.remove( args... ); - } - else - { - m_points--; + m_nans--; + if( m_process_na || ( m_consider_na && !m_igna ) ) m_stat.remove( args... ); - } } - - template - double compute( V... args ) + else { - if( ( !m_igna && (m_nans > 0 && !m_consider_na ) ) || m_points < m_mdp ) - return std::numeric_limits::quiet_NaN(); - - return m_stat.compute( args... ); + m_points--; + m_stat.remove( args... ); } + } - DateTime compute_dt() // needed for argmin/max - { - if( ( !m_igna && m_nans > 0 ) || m_points < m_mdp ) - return DateTime::fromNanoseconds( 0 ); + template + double compute( V... args ) + { + if( ( !m_igna && ( m_nans > 0 && !m_consider_na ) ) || m_points < m_mdp ) + return std::numeric_limits::quiet_NaN(); - return m_stat.compute(); - } + return m_stat.compute( args... ); + } - void reset() - { - m_nans = 0; - m_points = 0; - m_stat.reset(); - } + DateTime compute_dt() // needed for argmin/max + { + if( ( !m_igna && m_nans > 0 ) || m_points < m_mdp ) + return DateTime::fromNanoseconds( 0 ); - private: - int64_t m_nans = 0; - int64_t m_points = 0; - int64_t m_mdp = 0; - bool m_igna = false; - T m_stat; - static constexpr bool m_process_na = ( std::is_same::value || std::is_same::value || std::is_same::value - || std::is_same::value || std::is_same::value || std::is_same::value ); - static constexpr bool m_consider_na = ( std::is_same::value || std::is_same::value ); -}; + return m_stat.compute(); + } + + void reset() + { + m_nans = 0; + m_points = 0; + m_stat.reset(); + } +private: + int64_t m_nans = 0; + int64_t m_points = 0; + int64_t m_mdp = 0; + bool m_igna = false; + T m_stat; + static constexpr bool m_process_na + = ( std::is_same::value || std::is_same::value || std::is_same::value + || std::is_same::value || std::is_same::value + || std::is_same::value ); + static constexpr bool m_consider_na = ( std::is_same::value || std::is_same::value ); +}; // Generic window update nodes @@ -1911,23 +1795,28 @@ DECLARE_CPPNODE( _generic_tick_window_updates ) TS_INPUT( Generic, reset ); TS_INPUT( Generic, recalc ); - STATE_VAR( bool, s_first{true} ); - STATE_VAR( int64_t, s_last_call{0} ); - STATE_VAR( int64_t, s_last_count{0} ); - STATE_VAR( bool, s_pending_recalc{false} ); + STATE_VAR( bool, s_first{ true } ); + STATE_VAR( int64_t, s_last_call{ 0 } ); + STATE_VAR( int64_t, s_last_count{ 0 } ); + STATE_VAR( bool, s_pending_recalc{ false } ); - STATE_VAR( FixedSizeWindowBuffer, s_value_buffer{interval} ); + STATE_VAR( FixedSizeWindowBuffer, s_value_buffer{ interval } ); STATE_VAR( std::vector, s_removals{} ); TS_NAMED_OUTPUT_RENAMED( std::vector, additions, additions_ ); TS_NAMED_OUTPUT_RENAMED( std::vector, removals, removals_ ); - CSP csp; - const char * name() const override { return "_tick_window_updates"; } + CSP csp; + const char * name() const override + { + return "_tick_window_updates"; + } public: - _generic_tick_window_updates( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _generic_tick_window_updates( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } START() { @@ -1954,29 +1843,29 @@ DECLARE_CPPNODE( _generic_tick_window_updates ) if( csp.ticked( sampler ) ) { // Handle removals if needed - if( s_value_buffer.full() && s_removals.size() < ( size_t )s_last_count ) + if( s_value_buffer.full() && s_removals.size() < (size_t)s_last_count ) { T v = s_value_buffer.pop_left(); s_removals.push_back( v ); } - NodeT * node = static_cast( this ); // CRTP + NodeT * node = static_cast( this ); // CRTP if( csp.ticked( x ) ) { // Ensure size is consistent for NumPy case - do nothing for floats - node -> validateShape(); + node->validateShape(); s_value_buffer.push( x ); } else { - node -> checkValid(); - s_value_buffer.push( node -> createNan() ); + node->checkValid(); + s_value_buffer.push( node->createNan() ); } } - if( csp.ticked( trigger ) || ( s_first && csp.ticked( x ) && csp.ticked( sampler ) ) ) + if( csp.ticked( trigger ) || ( s_first && csp.ticked( x ) && csp.ticked( sampler ) ) ) { - s_first = false; + s_first = false; int64_t total_count = csp.count( sampler ); // Handle removals @@ -1989,33 +1878,33 @@ DECLARE_CPPNODE( _generic_tick_window_updates ) if( s_pending_recalc && !s_value_buffer.empty() ) { // Copy entire buffer to additions for fresh calculation - std::vector* additions = &additions_.reserveSpace(); - additions -> reserve( s_value_buffer.count() ); + std::vector * additions = &additions_.reserveSpace(); + additions->reserve( s_value_buffer.count() ); s_value_buffer.copy_values( additions ); s_pending_recalc = false; } else { // Handle incremental additions since last call - size_t ticks = total_count - s_last_call; - size_t sz = s_value_buffer.count(); + size_t ticks = total_count - s_last_call; + size_t sz = s_value_buffer.count(); int64_t add_cutoff = std::min( ticks, sz ); - int64_t offset = add_cutoff-1; + int64_t offset = add_cutoff - 1; - std::vector* additions = nullptr; + std::vector * additions = nullptr; while( offset >= 0 ) { if( !additions ) // reserve memory { additions = &additions_.reserveSpace(); - additions -> clear(); + additions->clear(); } - additions -> push_back( s_value_buffer[offset] ); + additions->push_back( s_value_buffer[offset] ); offset--; } } - s_last_call = total_count; + s_last_call = total_count; s_last_count = s_value_buffer.count(); // needed for no improper removals } } @@ -2025,7 +1914,6 @@ template DECLARE_CPPNODE( _generic_time_window_updates ) { protected: - TS_INPUT( T, x ); SCALAR_INPUT( TimeDelta, interval ); TS_INPUT( Generic, trigger ); @@ -2034,19 +1922,16 @@ DECLARE_CPPNODE( _generic_time_window_updates ) TS_INPUT( Generic, recalc ); STATE_VAR( DateTime, s_last_call ); - STATE_VAR( bool, s_pending_recalc{false} ); - STATE_VAR( int64_t, s_last_tick{0} ); + STATE_VAR( bool, s_pending_recalc{ false } ); + STATE_VAR( int64_t, s_last_tick{ 0 } ); - STATE_VAR( size_t, s_pending_removals{0} ); + STATE_VAR( size_t, s_pending_removals{ 0 } ); // This keeps track of how many additions have been published without corresponding removals - // Its to fix a rare bug when trigger is separate from the data, and the data has multiple ticks at the same timestamp - // In this case the first data tick gets picked up on trigger and sent with additions, but the second tick does NOT - // get picked up, because trigger already finished. Then when its time to remove, the value that wasnt sent on additions - // gets added to removed because its past the timestamp... example: - // import csp - // from datetime import datetime, timedelta - // import csp.stats - // data = [ + // Its to fix a rare bug when trigger is separate from the data, and the data has multiple ticks at the same + // timestamp In this case the first data tick gets picked up on trigger and sent with additions, but the second tick + // does NOT get picked up, because trigger already finished. Then when its time to remove, the value that wasnt + // sent on additions gets added to removed because its past the timestamp... example: import csp from datetime + // import datetime, timedelta import csp.stats data = [ // (datetime(2023, 2, 17,17, 17, 34), 23.0), // (datetime(2023, 2, 17,17, 22, 43), 22.0), // (datetime(2023, 2, 17,17, 22, 43), 22.0), @@ -2071,8 +1956,8 @@ DECLARE_CPPNODE( _generic_time_window_updates ) // // We use a simple counter to work around this issue, we wont output a removal if there are no pending remaining - STATE_VAR( bool, s_first{true} ); - STATE_VAR( bool, s_expanding{false} ); + STATE_VAR( bool, s_first{ true } ); + STATE_VAR( bool, s_expanding{ false } ); STATE_VAR( VariableSizeWindowBuffer, s_value_buffer{} ); STATE_VAR( VariableSizeWindowBuffer, s_time_buffer{} ); @@ -2082,13 +1967,17 @@ DECLARE_CPPNODE( _generic_time_window_updates ) TS_NAMED_OUTPUT_RENAMED( std::vector, additions, additions_ ); TS_NAMED_OUTPUT_RENAMED( std::vector, removals, removals_ ); - CSP csp; - const char * name() const override { return "_time_window_updates"; } + CSP csp; + const char * name() const override + { + return "_time_window_updates"; + } public: - - _generic_time_window_updates( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _generic_time_window_updates( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } START() { @@ -2120,10 +2009,10 @@ DECLARE_CPPNODE( _generic_time_window_updates ) if( csp.ticked( sampler ) ) { - NodeT * node = static_cast( this ); // CRTP + NodeT * node = static_cast( this ); // CRTP if( csp.ticked( x ) ) { - node -> validateShape(); + node->validateShape(); if( s_expanding ) s_additions.push_back( x ); else @@ -2131,11 +2020,11 @@ DECLARE_CPPNODE( _generic_time_window_updates ) } else { - node -> checkValid(); + node->checkValid(); if( s_expanding ) - s_additions.push_back( node -> createNan() ); + s_additions.push_back( node->createNan() ); else - s_value_buffer.push( node -> createNan() ); + s_value_buffer.push( node->createNan() ); } if( !s_expanding ) s_time_buffer.push( now() ); @@ -2148,7 +2037,7 @@ DECLARE_CPPNODE( _generic_time_window_updates ) if( s_expanding ) { // fast track expanding window calculations - // we just need to swap in all additions with no checks + // we just need to swap in all additions with no checks if( !s_additions.empty() ) { std::swap( additions_.reserveSpace(), s_additions ); @@ -2156,24 +2045,24 @@ DECLARE_CPPNODE( _generic_time_window_updates ) } return; } - + DateTime threshold = now() - interval; // Handle removals - int64_t count = csp.count( sampler ); - std::vector* removals = nullptr; + int64_t count = csp.count( sampler ); + std::vector * removals = nullptr; while( !s_value_buffer.empty() && s_time_buffer[-1] <= threshold ) { DateTime time = s_time_buffer.pop_left(); - T val = s_value_buffer.pop_left(); + T val = s_value_buffer.pop_left(); if( !s_pending_recalc && time <= s_last_call && s_pending_removals ) { if( !removals ) // reserve memory { removals = &removals_.reserveSpace(); - removals -> clear(); + removals->clear(); } - removals -> push_back( val ); + removals->push_back( val ); --s_pending_removals; } } @@ -2182,27 +2071,27 @@ DECLARE_CPPNODE( _generic_time_window_updates ) if( s_pending_recalc && !s_value_buffer.empty() ) { // Copy entire buffer to additions for fresh calculation - std::vector* additions = &additions_.reserveSpace(); - additions -> reserve( s_value_buffer.count() ); + std::vector * additions = &additions_.reserveSpace(); + additions->reserve( s_value_buffer.count() ); s_value_buffer.copy_values( additions ); - s_pending_recalc = false; - s_pending_removals = additions -> size(); + s_pending_recalc = false; + s_pending_removals = additions->size(); } else { // Handle additions incrementally - int64_t sz = s_time_buffer.count(); + int64_t sz = s_time_buffer.count(); int64_t offset = std::min( count - s_last_tick, sz ) - 1; - std::vector* additions = nullptr; + std::vector * additions = nullptr; while( offset >= 0 ) { if( !additions ) // reserve memory { additions = &additions_.reserveSpace(); - additions -> clear(); + additions->clear(); } - additions -> push_back( s_value_buffer[offset] ); + additions->push_back( s_value_buffer[offset] ); offset--; ++s_pending_removals; } @@ -2218,7 +2107,6 @@ template DECLARE_CPPNODE( _generic_cross_sectional ) { protected: - TS_INPUT( std::vector, additions ); TS_INPUT( std::vector, removals ); TS_INPUT( Generic, trigger ); @@ -2226,13 +2114,17 @@ DECLARE_CPPNODE( _generic_cross_sectional ) STATE_VAR( VariableSizeWindowBuffer, s_window{} ); TS_OUTPUT( OutputT ); - CSP csp; - const char * name() const override { return "_cross_sectional"; } + CSP csp; + const char * name() const override + { + return "_cross_sectional"; + } public: - - _generic_cross_sectional( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - {} + _generic_cross_sectional( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) + : csp::CppNode( engine, nodedef ) + { + } INVOKE() { @@ -2243,10 +2135,10 @@ DECLARE_CPPNODE( _generic_cross_sectional ) if( csp.ticked( additions ) ) s_window.extend( additions.lastValue() ); if( csp.ticked( trigger ) ) - static_cast( this ) -> computeCrossSectional(); // CRTP + static_cast( this )->computeCrossSectional(); // CRTP } }; -} +} // namespace csp::cppnodes #endif // _IN_CSP_CPPNODES_STATSIMPL_H diff --git a/cpp/csp/engine/AdapterManager.cpp b/cpp/csp/engine/AdapterManager.cpp index 1f1a3999f..e99067fba 100644 --- a/cpp/csp/engine/AdapterManager.cpp +++ b/cpp/csp/engine/AdapterManager.cpp @@ -4,24 +4,24 @@ namespace csp { -ManagedSimInputAdapter::ManagedSimInputAdapter( csp::Engine * engine, - const CspTypePtr & type, - AdapterManager *manager, - PushMode pushMode ) : InputAdapter( engine, type, pushMode ), - m_manager( manager ), - m_lastCycleCount( 0 ) +ManagedSimInputAdapter::ManagedSimInputAdapter( csp::Engine * engine, const CspTypePtr & type, AdapterManager * manager, + PushMode pushMode ) + : InputAdapter( engine, type, pushMode ) + , m_manager( manager ) + , m_lastCycleCount( 0 ) { } -AdapterManager::AdapterManager( csp::Engine * engine ) : m_engine( engine ), m_statusAdapter( nullptr ), m_started( false ) +AdapterManager::AdapterManager( csp::Engine * engine ) + : m_engine( engine ) + , m_statusAdapter( nullptr ) + , m_started( false ) { - if( !m_engine -> isRootEngine() ) + if( !m_engine->isRootEngine() ) CSP_THROW( NotImplemented, "AdapterManager support is not currently available in dynamic graphs" ); } -AdapterManager::~AdapterManager() -{ -} +AdapterManager::~AdapterManager() {} void AdapterManager::start( DateTime starttime, DateTime endtime ) { @@ -31,13 +31,11 @@ void AdapterManager::start( DateTime starttime, DateTime endtime ) scheduleTimerCB( starttime ); } -void AdapterManager::stop() -{ -} +void AdapterManager::stop() {} void AdapterManager::processSimTimerCB() { - DateTime next = processNextSimTimeSlice( rootEngine() -> now() ); + DateTime next = processNextSimTimeSlice( rootEngine()->now() ); if( !next.isNone() ) scheduleTimerCB( next ); } @@ -45,15 +43,15 @@ void AdapterManager::processSimTimerCB() StatusAdapter * AdapterManager::createStatusAdapter( CspTypePtr & type, PushMode pushMode ) { if( !m_statusAdapter ) - m_statusAdapter = m_engine -> createOwnedObject( type, pushMode, statusPushGroup() ); + m_statusAdapter = m_engine->createOwnedObject( type, pushMode, statusPushGroup() ); return m_statusAdapter; } -void AdapterManager::pushStatus( int64_t level, int64_t errCode, const std::string & errMsg, PushBatch *batch ) const +void AdapterManager::pushStatus( int64_t level, int64_t errCode, const std::string & errMsg, PushBatch * batch ) const { if( m_statusAdapter ) - m_statusAdapter -> pushStatus( level, errCode, errMsg, batch ); + m_statusAdapter->pushStatus( level, errCode, errMsg, batch ); } -} +} // namespace csp diff --git a/cpp/csp/engine/AdapterManager.h b/cpp/csp/engine/AdapterManager.h index a0c1531ee..d7ba63120 100644 --- a/cpp/csp/engine/AdapterManager.h +++ b/cpp/csp/engine/AdapterManager.h @@ -19,73 +19,76 @@ class Engine; class ManagedSimInputAdapter : public InputAdapter { public: - ManagedSimInputAdapter( csp::Engine *engine, const CspTypePtr &type, AdapterManager *manager, PushMode pushMode ); + ManagedSimInputAdapter( csp::Engine * engine, const CspTypePtr & type, AdapterManager * manager, + PushMode pushMode ); - template< typename T > - bool pushTick( const T &value ); + template + bool pushTick( const T & value ); - template< typename T > + template bool pushNullTick(); private: template - void scheduleDelayedValue(std::optional&& value); + void scheduleDelayedValue( std::optional && value ); - template< typename T > + template bool consumeDelayedValue(); AdapterManager * m_manager; - std::uint64_t m_lastCycleCount; + std::uint64_t m_lastCycleCount; }; -template< typename T > -inline bool ManagedSimInputAdapter::pushTick( const T &value ) +template +inline bool ManagedSimInputAdapter::pushTick( const T & value ) { if( pushMode() == PushMode::NON_COLLAPSING ) { - auto cycleCount = rootEngine() -> cycleCount(); + auto cycleCount = rootEngine()->cycleCount(); if( m_lastCycleCount == cycleCount || !consumeTick( value ) ) { - //for non-collapsing data with duplicate timestamps, - //we schedule the data to further engine cycles - rootEngine() -> scheduleCallback( rootEngine() -> now(), [ this, value ]() -> const InputAdapter* - { - auto cycleCount = rootEngine() -> cycleCount(); - if(m_lastCycleCount == cycleCount) - return this; - - m_lastCycleCount = cycleCount; - consumeTick(value); - return nullptr; - } ); + // for non-collapsing data with duplicate timestamps, + // we schedule the data to further engine cycles + rootEngine()->scheduleCallback( rootEngine()->now(), + [this, value]() -> const InputAdapter * + { + auto cycleCount = rootEngine()->cycleCount(); + if( m_lastCycleCount == cycleCount ) + return this; + + m_lastCycleCount = cycleCount; + consumeTick( value ); + return nullptr; + } ); } m_lastCycleCount = cycleCount; } else consumeTick( value ); - //Note we always return true since we rescheduled our own timer above if need be + // Note we always return true since we rescheduled our own timer above if need be return true; } -template< typename T > +template bool ManagedSimInputAdapter::pushNullTick() { if( pushMode() == PushMode::NON_COLLAPSING ) { - auto cycleCount = rootEngine() -> cycleCount(); + auto cycleCount = rootEngine()->cycleCount(); if( m_lastCycleCount == cycleCount ) { - rootEngine() -> scheduleCallback( rootEngine() -> now(), [ this ]() -> const InputAdapter* - { - auto cycleCount = rootEngine() -> cycleCount(); - if( m_lastCycleCount == cycleCount ) - return this; - - m_lastCycleCount = cycleCount; - return nullptr; - } ); + rootEngine()->scheduleCallback( rootEngine()->now(), + [this]() -> const InputAdapter * + { + auto cycleCount = rootEngine()->cycleCount(); + if( m_lastCycleCount == cycleCount ) + return this; + + m_lastCycleCount = cycleCount; + return nullptr; + } ); } m_lastCycleCount = cycleCount; } @@ -99,41 +102,41 @@ class AdapterManager : public EngineOwned AdapterManager( csp::Engine * ); virtual ~AdapterManager(); - virtual const char *name() const = 0; + virtual const char * name() const = 0; - //derivations should call base start from their start call + // derivations should call base start from their start call virtual void start( DateTime starttime, DateTime endtime ); virtual void stop(); - //for sim inputs, this should process all entries with time "time" and return the time of the next - //available tick, or return DateTime::NONE() if there is no more data. Initial call will be with starttime - //subsequent calls will be with the previously returned DateTime + // for sim inputs, this should process all entries with time "time" and return the time of the next + // available tick, or return DateTime::NONE() if there is no more data. Initial call will be with starttime + // subsequent calls will be with the previously returned DateTime virtual DateTime processNextSimTimeSlice( DateTime time ) = 0; - Engine * engine() { return m_engine; } - const Engine * engine() const { return m_engine; } + Engine * engine() { return m_engine; } + const Engine * engine() const { return m_engine; } + + RootEngine * rootEngine() { return m_engine->rootEngine(); } + const RootEngine * rootEngine() const { return m_engine->rootEngine(); } - RootEngine * rootEngine() { return m_engine -> rootEngine(); } - const RootEngine * rootEngine() const { return m_engine -> rootEngine(); } + DateTime starttime() const { return m_starttime; } + DateTime endtime() const { return m_endtime; } - DateTime starttime() const { return m_starttime; } - DateTime endtime() const { return m_endtime; } - - void setStarted() { m_started = true; } - bool started() const { return m_started; } + void setStarted() { m_started = true; } + bool started() const { return m_started; } - StatusAdapter *createStatusAdapter( CspTypePtr &type, PushMode pushMode ); - void pushStatus( int64_t level, int64_t errCode, const std::string &errMsg, PushBatch *batch = nullptr ) const; + StatusAdapter * createStatusAdapter( CspTypePtr & type, PushMode pushMode ); + void pushStatus( int64_t level, int64_t errCode, const std::string & errMsg, PushBatch * batch = nullptr ) const; protected: - //for adapters that want status adapter synced to a PushGroup + // for adapters that want status adapter synced to a PushGroup virtual PushGroup * statusPushGroup() { return nullptr; } void scheduleTimerCB( DateTime next ); - + void processSimTimerCB(); - csp::Engine * m_engine; + csp::Engine * m_engine; DateTime m_starttime; DateTime m_endtime; StatusAdapter * m_statusAdapter; @@ -144,15 +147,19 @@ inline void AdapterManager::scheduleTimerCB( DateTime next ) { try { - rootEngine() -> scheduleCallback( next, [ this ](){ processSimTimerCB(); return nullptr; } ); + rootEngine()->scheduleCallback( next, + [this]() + { + processSimTimerCB(); + return nullptr; + } ); } - catch( const ValueError &err ) + catch( const ValueError & err ) { CSP_THROW( ValueError, "AdapterManager " << name() << " scheduler error: " << err.description() ); } - } -} +} // namespace csp #endif diff --git a/cpp/csp/engine/AlarmInputAdapter.h b/cpp/csp/engine/AlarmInputAdapter.h index 28b70f704..3714a5bc7 100644 --- a/cpp/csp/engine/AlarmInputAdapter.h +++ b/cpp/csp/engine/AlarmInputAdapter.h @@ -11,7 +11,8 @@ template class AlarmInputAdapter final : public InputAdapter { public: - AlarmInputAdapter( Engine * engine, CspTypePtr & type ) : InputAdapter( engine, type, PushMode::NON_COLLAPSING ) + AlarmInputAdapter( Engine * engine, CspTypePtr & type ) + : InputAdapter( engine, type, PushMode::NON_COLLAPSING ) { } @@ -19,51 +20,48 @@ class AlarmInputAdapter final : public InputAdapter void stop() override { for( auto & handle : m_pendingHandles ) - rootEngine() -> cancelCallback( handle ); + rootEngine()->cancelCallback( handle ); m_pendingHandles.clear(); } Scheduler::Handle scheduleAlarm( TimeDelta delta, const T & value ) { - return scheduleAlarm( rootEngine() -> now() + delta, value ); + return scheduleAlarm( rootEngine()->now() + delta, value ); } Scheduler::Handle scheduleAlarm( DateTime time, const T & value ) { - auto handle = rootEngine() -> reserveSchedulerHandle(); - auto it = m_pendingHandles.insert( m_pendingHandles.end(), handle ); - handle = rootEngine() -> scheduleCallback( handle, time, - [this, value, it]() -> const InputAdapter * - { - if( !this -> consumeTick( value ) ) - return this; + auto handle = rootEngine()->reserveSchedulerHandle(); + auto it = m_pendingHandles.insert( m_pendingHandles.end(), handle ); + handle = rootEngine()->scheduleCallback( handle, time, + [this, value, it]() -> const InputAdapter * + { + if( !this->consumeTick( value ) ) + return this; - m_pendingHandles.erase( it ); - return nullptr; - } ); - (*it) = handle; + m_pendingHandles.erase( it ); + return nullptr; + } ); + ( *it ) = handle; return handle; } Scheduler::Handle rescheduleAlarm( Scheduler::Handle handle, TimeDelta delta ) { - return rescheduleAlarm( handle, rootEngine() -> now() + delta ); + return rescheduleAlarm( handle, rootEngine()->now() + delta ); } Scheduler::Handle rescheduleAlarm( Scheduler::Handle handle, DateTime time ) { - return rootEngine() -> rescheduleCallback( handle, time ); + return rootEngine()->rescheduleCallback( handle, time ); } - void cancelAlarm( Scheduler::Handle handle ) - { - rootEngine() -> cancelCallback( handle ); - } + void cancelAlarm( Scheduler::Handle handle ) { rootEngine()->cancelCallback( handle ); } private: std::list m_pendingHandles; }; -}; +}; // namespace csp #endif diff --git a/cpp/csp/engine/BasketInfo.cpp b/cpp/csp/engine/BasketInfo.cpp index f43e425e9..de96d6148 100644 --- a/cpp/csp/engine/BasketInfo.cpp +++ b/cpp/csp/engine/BasketInfo.cpp @@ -5,20 +5,20 @@ namespace csp { -InputBasketInfo::InputBasketInfo( RootEngine * rootEngine, - size_t size, bool isDynamic ) : m_inputs( nullptr ), - m_size( size ), - m_lastCycleCount( 0 ), - m_rootEngine( rootEngine ), - m_valid( false ), - m_isDynamic( isDynamic ) +InputBasketInfo::InputBasketInfo( RootEngine * rootEngine, size_t size, bool isDynamic ) + : m_inputs( nullptr ) + , m_size( size ) + , m_lastCycleCount( 0 ) + , m_rootEngine( rootEngine ) + , m_valid( false ) + , m_isDynamic( isDynamic ) { - //For dynamic baskets, the shape time series is stored at index -1 - //so that size / iteration still works cleanly + // For dynamic baskets, the shape time series is stored at index -1 + // so that size / iteration still works cleanly auto numElements = m_size + ( isDynamic ? 1 : 0 ); if( numElements > 0 ) { - m_inputs = ( TimeSeriesProvider const ** ) malloc( sizeof( TimeSeriesProvider * ) * numElements ); + m_inputs = (TimeSeriesProvider const **)malloc( sizeof( TimeSeriesProvider * ) * numElements ); memset( m_inputs, 0, sizeof( TimeSeriesProvider * ) * numElements ); if( isDynamic ) ++m_inputs; @@ -38,37 +38,39 @@ int64_t DynamicInputBasketInfo::addDynamicKey( const DialectGenericType & key, c int64_t elemId = m_size; ++m_size; - //we always add to the end of the basket, removes swap out last elem into removed slot + // we always add to the end of the basket, removes swap out last elem into removed slot if( elemId >= m_capacity ) { CSP_ASSERT( elemId == m_capacity ); m_capacity = std::max( 1, m_capacity * 2 ); - m_inputs = ( TimeSeriesProvider const ** ) realloc( m_inputs - 1, sizeof( TimeSeriesProvider * ) * ( m_capacity + 1 ) ); + m_inputs + = (TimeSeriesProvider const **)realloc( m_inputs - 1, sizeof( TimeSeriesProvider * ) * ( m_capacity + 1 ) ); ++m_inputs; std::fill( m_inputs + elemId, m_inputs + m_capacity, nullptr ); } else - CSP_ASSERT( !m_inputs[ elemId ] ); + CSP_ASSERT( !m_inputs[elemId] ); - m_inputs[ elemId ] = ts; - const_cast( ts ) -> setTickCountPolicy( m_tickCountPolicy ); - const_cast( ts ) -> setTickTimeWindowPolicy( m_timeWindowPolicy ); + m_inputs[elemId] = ts; + const_cast( ts )->setTickCountPolicy( m_tickCountPolicy ); + const_cast( ts )->setTickTimeWindowPolicy( m_timeWindowPolicy ); - if( likely( ( bool ) m_changeCallback ) ) + if( likely( (bool)m_changeCallback ) ) m_changeCallback( key, true, elemId, -1 ); return elemId; } -void DynamicInputBasketInfo::removeDynamicKey( uint64_t engineCycle, const DialectGenericType & key, int64_t removeId, int64_t replaceId ) +void DynamicInputBasketInfo::removeDynamicKey( uint64_t engineCycle, const DialectGenericType & key, int64_t removeId, + int64_t replaceId ) { CSP_ASSERT( m_size > 0 ); - //we need to update ticked items if replaceId is set ( being moved ) AND it ticked. Note that we dont have to reset removeId - //since we dont allow removing a key same cycle as it is ticked. - //We check if replaceId has a tick this cycle to avoid doing a potentially expensive tickeditems scan if we dont need to - if( replaceId != -1 && elem( replaceId ) -> lastCycleCount() == engineCycle ) + // we need to update ticked items if replaceId is set ( being moved ) AND it ticked. Note that we dont have to + // reset removeId since we dont allow removing a key same cycle as it is ticked. We check if replaceId has a tick + // this cycle to avoid doing a potentially expensive tickeditems scan if we dont need to + if( replaceId != -1 && elem( replaceId )->lastCycleCount() == engineCycle ) { - //need to scan unfortunately, we had a long debate back and forth on this, we may revisit in the future + // need to scan unfortunately, we had a long debate back and forth on this, we may revisit in the future for( auto & id : m_tickedInputs ) { if( id == replaceId ) @@ -81,36 +83,36 @@ void DynamicInputBasketInfo::removeDynamicKey( uint64_t engineCycle, const Diale --m_size; - m_inputs[ removeId ] = nullptr; + m_inputs[removeId] = nullptr; if( replaceId >= 0 ) { - m_inputs[ removeId ] = m_inputs[ replaceId ]; - m_inputs[ replaceId ] = nullptr; + m_inputs[removeId] = m_inputs[replaceId]; + m_inputs[replaceId] = nullptr; } - if( likely( ( bool ) m_changeCallback ) ) + if( likely( (bool)m_changeCallback ) ) m_changeCallback( key, false, removeId, replaceId ); } - -OutputBasketInfo::OutputBasketInfo( CspTypePtr & type, Node * node, - size_t size, bool isDynamic ) : m_outputs( nullptr ), - m_size( size ), - m_isDynamic( isDynamic ) +OutputBasketInfo::OutputBasketInfo( CspTypePtr & type, Node * node, size_t size, bool isDynamic ) + : m_outputs( nullptr ) + , m_size( size ) + , m_isDynamic( isDynamic ) { if( m_size > 0 ) { - //for static baskets we do a single alloc so all time series are in the same memory block + // for static baskets we do a single alloc so all time series are in the same memory block size_t allocSize = sizeof( TimeSeriesProvider * ) * m_size + sizeof( TimeSeriesProvider ) * size; - void * mem = malloc( allocSize ); - m_outputs = ( TimeSeriesProvider * * ) mem; - //location of first contiguous TSP - TimeSeriesProvider * p = reinterpret_cast( reinterpret_cast( mem ) + sizeof( TimeSeriesProvider * ) * m_size ); + void * mem = malloc( allocSize ); + m_outputs = (TimeSeriesProvider **)mem; + // location of first contiguous TSP + TimeSeriesProvider * p = reinterpret_cast( reinterpret_cast( mem ) + + sizeof( TimeSeriesProvider * ) * m_size ); for( int64_t elemId = 0; elemId < m_size; ++elemId ) { - new ( p ) TimeSeriesProvider(); - m_outputs[ elemId ] = p; - m_outputs[ elemId ] -> init( type, node ); + new( p ) TimeSeriesProvider(); + m_outputs[elemId] = p; + m_outputs[elemId]->init( type, node ); ++p; } } @@ -119,16 +121,16 @@ OutputBasketInfo::OutputBasketInfo( CspTypePtr & type, Node * node, OutputBasketInfo::~OutputBasketInfo() { for( int64_t i = 0; i < m_size; ++i ) - m_outputs[i] -> ~TimeSeriesProvider(); + m_outputs[i]->~TimeSeriesProvider(); free( m_outputs ); } - -DynamicOutputBasketInfo::DynamicOutputBasketInfo( CspTypePtr & type, Node * node ) : OutputBasketInfo( type, node, 0, true ), - m_elemType( type ), - m_parentNode( node ), - m_capacity( 0 ), - m_ownTs( true ) +DynamicOutputBasketInfo::DynamicOutputBasketInfo( CspTypePtr & type, Node * node ) + : OutputBasketInfo( type, node, 0, true ) + , m_elemType( type ) + , m_parentNode( node ) + , m_capacity( 0 ) + , m_ownTs( true ) { static CspTypePtr s_shapeType = std::make_shared( csp::autogen::DynamicBasketEvent::meta() ); m_shapeTs.init( s_shapeType, node ); @@ -136,41 +138,40 @@ DynamicOutputBasketInfo::DynamicOutputBasketInfo( CspTypePtr & type, Node * node DynamicOutputBasketInfo::~DynamicOutputBasketInfo() { - //we need to free indiviual timeseries here since we allocate TS's independently in dynamic baskets - //static baskets do a single allocation for all ts - //we may be a dynamic graph output basket, in which case we dont own the time series ( see comment in header ) + // we need to free indiviual timeseries here since we allocate TS's independently in dynamic baskets + // static baskets do a single allocation for all ts + // we may be a dynamic graph output basket, in which case we dont own the time series ( see comment in header ) if( m_ownTs ) { for( int64_t i = 0; i < m_capacity; ++i ) delete m_outputs[i]; } - //set size to 0 so that ~OutputBasketInfo doesnt try to destroy providers again + // set size to 0 so that ~OutputBasketInfo doesnt try to destroy providers again m_size = m_capacity = 0; } void DynamicOutputBasketInfo::linkInputBasket( Node * node, INOUT_ID_TYPE inputIdx ) { - node -> inputBasket( inputIdx ) -> setElem( -1, &m_shapeTs ); + node->inputBasket( inputIdx )->setElem( -1, &m_shapeTs ); m_shapeTs.addConsumer( node, InputId( inputIdx ) ); } void DynamicOutputBasketInfo::addShapeChange( const DialectGenericType & key, bool added ) { - if( m_parentNode -> rootEngine() -> cycleCount() != m_shapeTs.lastCycleCount() ) + if( m_parentNode->rootEngine()->cycleCount() != m_shapeTs.lastCycleCount() ) { auto events = autogen::DynamicBasketEvents::create(); - events -> set_events( {} ); - m_shapeTs.outputTickTyped( m_parentNode -> rootEngine() -> cycleCount(), - m_parentNode -> rootEngine() -> now(), - events, false ); + events->set_events( {} ); + m_shapeTs.outputTickTyped( m_parentNode->rootEngine()->cycleCount(), + m_parentNode->rootEngine()->now(), events, false ); } - auto & events = m_shapeTs.lastValueTyped() -> events(); + auto & events = m_shapeTs.lastValueTyped()->events(); auto event = autogen::DynamicBasketEvent::create(); - event -> set_key( key ); - event -> set_added( added ); + event->set_key( key ); + event->set_added( added ); const_cast &>( events ).emplace_back( event ); } @@ -180,10 +181,10 @@ bool DynamicOutputBasketInfo::addCapacity() if( m_size == m_capacity ) { m_capacity = std::max( 1, m_capacity * 2 ); - //in dynamic baskets we cant use the "single alloc" optimization we do for static baskets since - //input baskets already refer to the TSPs by pointer ( which can be changed on realloc ) - //so we do a regular alloc of array of ptrs, then allocate each item independently - m_outputs = ( TimeSeriesProvider ** ) realloc( m_outputs, sizeof( TimeSeriesProvider * ) * ( m_capacity ) ); + // in dynamic baskets we cant use the "single alloc" optimization we do for static baskets since + // input baskets already refer to the TSPs by pointer ( which can be changed on realloc ) + // so we do a regular alloc of array of ptrs, then allocate each item independently + m_outputs = (TimeSeriesProvider **)realloc( m_outputs, sizeof( TimeSeriesProvider * ) * ( m_capacity ) ); memset( m_outputs + m_size, 0, sizeof( TimeSeriesProvider * ) * ( m_capacity - m_size ) ); return true; } @@ -193,13 +194,14 @@ bool DynamicOutputBasketInfo::addCapacity() void DynamicOutputBasketInfo::propagateAddKey( const DialectGenericType & key, const TimeSeriesProvider * ts ) { - m_shapeTs.propagator().apply( [ts, &key]( Consumer * node, const InputId & id ) - { - static_cast( node ) -> addDynamicInputBasketKey( id.id, key, ts ); - //invoke Consumer::handleEvent not Node::handleEvent, we dont want shape tick to get into - //tickedInputs of basket - static_cast( node ) -> Consumer::handleEvent( id ); - } ); + m_shapeTs.propagator().apply( + [ts, &key]( Consumer * node, const InputId & id ) + { + static_cast( node )->addDynamicInputBasketKey( id.id, key, ts ); + // invoke Consumer::handleEvent not Node::handleEvent, we dont want shape tick to get into + // tickedInputs of basket + static_cast( node )->Consumer::handleEvent( id ); + } ); } int64_t DynamicOutputBasketInfo::addDynamicTs( const DialectGenericType & key, const TimeSeriesProvider * ts ) @@ -209,13 +211,12 @@ int64_t DynamicOutputBasketInfo::addDynamicTs( const DialectGenericType & key, c m_ownTs = false; addCapacity(); - int64_t elemId = m_size++; - m_outputs[ elemId ] = const_cast( ts ); + int64_t elemId = m_size++; + m_outputs[elemId] = const_cast( ts ); addShapeChange( key, true ); propagateAddKey( key, ts ); return elemId; - } int64_t DynamicOutputBasketInfo::addDynamicKey( const DialectGenericType & key ) @@ -227,14 +228,14 @@ int64_t DynamicOutputBasketInfo::addDynamicKey( const DialectGenericType & key ) int64_t elemId = m_size++; - if( !m_outputs[ elemId ] ) + if( !m_outputs[elemId] ) { - m_outputs[ elemId ] = new TimeSeriesProvider(); - m_outputs[ elemId ] -> init( m_elemType, m_parentNode ); + m_outputs[elemId] = new TimeSeriesProvider(); + m_outputs[elemId]->init( m_elemType, m_parentNode ); } addShapeChange( key, true ); - propagateAddKey( key, m_outputs[ elemId ] ); + propagateAddKey( key, m_outputs[elemId] ); return elemId; } @@ -243,29 +244,30 @@ int64_t DynamicOutputBasketInfo::removeDynamicKey( const DialectGenericType & ke CSP_ASSERT( m_size > 0 ); int64_t replaceId = -1; - m_outputs[ elemId ] -> reset(); + m_outputs[elemId]->reset(); if( elemId != m_size - 1 ) { - std::swap( m_outputs[ elemId ], m_outputs[ m_size - 1 ] ); + std::swap( m_outputs[elemId], m_outputs[m_size - 1] ); replaceId = m_size - 1; - //We need to update the input ids associated with this TS now that it was moved - m_outputs[ elemId ] -> propagator().apply( [elemId]( Consumer *, const InputId & id ) - { const_cast( id ).elemId = elemId; } ); + // We need to update the input ids associated with this TS now that it was moved + m_outputs[elemId]->propagator().apply( [elemId]( Consumer *, const InputId & id ) + { const_cast( id ).elemId = elemId; } ); } --m_size; addShapeChange( key, false ); - m_shapeTs.propagator().apply( [&key,elemId,replaceId]( Consumer * node, const InputId & id ) - { - static_cast( node ) -> removeDynamicInputBasketKey( id.id, key, elemId, replaceId ); - static_cast( node ) -> Consumer::handleEvent( id ); - } ); + m_shapeTs.propagator().apply( + [&key, elemId, replaceId]( Consumer * node, const InputId & id ) + { + static_cast( node )->removeDynamicInputBasketKey( id.id, key, elemId, replaceId ); + static_cast( node )->Consumer::handleEvent( id ); + } ); return replaceId; } -} +} // namespace csp diff --git a/cpp/csp/engine/BasketInfo.h b/cpp/csp/engine/BasketInfo.h index 4fa50cb8a..43e763e49 100644 --- a/cpp/csp/engine/BasketInfo.h +++ b/cpp/csp/engine/BasketInfo.h @@ -20,7 +20,7 @@ class InputBasketInfo InputBasketInfo( RootEngine * rootEngine, size_t size, bool isDynamic = false ); ~InputBasketInfo(); - InputBasketInfo( const InputBasketInfo & ) = delete; + InputBasketInfo( const InputBasketInfo & ) = delete; InputBasketInfo & operator=( const InputBasketInfo & ) = delete; bool isDynamicBasket() const { return m_isDynamic; } @@ -28,13 +28,13 @@ class InputBasketInfo const TimeSeriesProvider * elem( int64_t elemId ) const { CSP_ASSERT( ( elemId >= 0 && elemId < m_size ) || ( elemId == -1 && m_isDynamic ) ); - return m_inputs[ elemId ]; + return m_inputs[elemId]; } void setElem( int64_t elemId, const TimeSeriesProvider * ts ) { - CSP_ASSERT( !m_inputs[ elemId ] ); - m_inputs[ elemId ] = ts; + CSP_ASSERT( !m_inputs[elemId] ); + m_inputs[elemId] = ts; } size_t size() const { return m_size; } @@ -42,10 +42,10 @@ class InputBasketInfo void handleEvent( int64_t elemId ) { const auto * ts = elem( elemId ); - if( ts -> lastCycleCount() > m_lastCycleCount ) + if( ts->lastCycleCount() > m_lastCycleCount ) { m_tickedInputs.clear(); - m_lastCycleCount = ts -> lastCycleCount(); + m_lastCycleCount = ts->lastCycleCount(); } m_tickedInputs.emplace_back( elemId ); @@ -56,79 +56,95 @@ class InputBasketInfo if( likely( m_valid ) ) return m_valid; - //Unforunately we have to loop over every input if we're not valid yet. Originally we were counting - //valid in handleEvent but this breaks if the basket is passive + // Unforunately we have to loop over every input if we're not valid yet. Originally we were counting + // valid in handleEvent but this breaks if the basket is passive for( auto it = m_inputs, itEnd = m_inputs + m_size; it != itEnd; ++it ) { - if( !(*it) -> valid() ) + if( !( *it )->valid() ) return false; } m_valid = true; return true; } - //num ticked this cycle - bool ticked() const { return hasTickedInputs(); } + // num ticked this cycle + bool ticked() const { return hasTickedInputs(); } struct input_iterator { - input_iterator() : m_it( nullptr ), m_itEnd( nullptr ), m_elemId(0) {} + input_iterator() + : m_it( nullptr ) + , m_itEnd( nullptr ) + , m_elemId( 0 ) + { + } input_iterator( TimeSeriesProvider const * const * itBegin, TimeSeriesProvider const * const * itEnd, - int64_t startId = 0 ) : m_elemId( startId ) + int64_t startId = 0 ) + : m_elemId( startId ) { m_it = itBegin; m_itEnd = itEnd; } - const TimeSeriesProvider * get() const { return *m_it; } - const TimeSeriesProvider * operator ->() const { return get(); } - const TimeSeriesProvider & ts() const { return *get(); } + const TimeSeriesProvider * get() const { return *m_it; } + const TimeSeriesProvider * operator->() const { return get(); } + const TimeSeriesProvider & ts() const { return *get(); } operator bool() const { return m_it != m_itEnd; } - input_iterator & operator++() { ++m_it; ++m_elemId; return *this; } - int64_t elemId() const { return m_elemId; } + input_iterator & operator++() + { + ++m_it; + ++m_elemId; + return *this; + } + int64_t elemId() const { return m_elemId; } private: TimeSeriesProvider const * const * m_it; TimeSeriesProvider const * const * m_itEnd; - int64_t m_elemId; + int64_t m_elemId; }; struct ticked_iterator { - ticked_iterator( TimeSeriesProvider const * const * inputs, - TickedInputs::const_iterator it, - TickedInputs::const_iterator itEnd ) : m_inputs( inputs ), - m_it( it ), - m_itEnd( itEnd ) - {} + ticked_iterator( TimeSeriesProvider const * const * inputs, TickedInputs::const_iterator it, + TickedInputs::const_iterator itEnd ) + : m_inputs( inputs ) + , m_it( it ) + , m_itEnd( itEnd ) + { + } - operator bool() const { return m_it != m_itEnd; } - ticked_iterator & operator++() { ++m_it; return *this; } + operator bool() const { return m_it != m_itEnd; } + ticked_iterator & operator++() + { + ++m_it; + return *this; + } - const TimeSeriesProvider * get() const { return m_inputs[ *m_it ]; } - const TimeSeriesProvider * operator -> () const { return get(); } - const TimeSeriesProvider & ts() const { return *get(); } + const TimeSeriesProvider * get() const { return m_inputs[*m_it]; } + const TimeSeriesProvider * operator->() const { return get(); } + const TimeSeriesProvider & ts() const { return *get(); } - int64_t elemId() const { return *m_it; } + int64_t elemId() const { return *m_it; } private: TimeSeriesProvider const * const * m_inputs; - TickedInputs::const_iterator m_it; - TickedInputs::const_iterator m_itEnd; + TickedInputs::const_iterator m_it; + TickedInputs::const_iterator m_itEnd; }; struct valid_iterator { - valid_iterator( TimeSeriesProvider const * const * itBegin, - TimeSeriesProvider const * const * itEnd ) : m_it( itBegin ), - m_itEnd( itEnd ), - m_elemId( 0 ) + valid_iterator( TimeSeriesProvider const * const * itBegin, TimeSeriesProvider const * const * itEnd ) + : m_it( itBegin ) + , m_itEnd( itEnd ) + , m_elemId( 0 ) { - if( m_it != m_itEnd && unlikely( !(*m_it ) -> valid() ) ) - ++(*this); + if( m_it != m_itEnd && unlikely( !( *m_it )->valid() ) ) + ++( *this ); } operator bool() const { return m_it != m_itEnd; } @@ -139,14 +155,14 @@ class InputBasketInfo { ++m_it; ++m_elemId; - } while( m_it != m_itEnd && !( (*m_it) -> valid() ) ); + } while( m_it != m_itEnd && !( ( *m_it )->valid() ) ); return *this; } - int64_t elemId() const { return m_elemId; } - const TimeSeriesProvider * get() const { return *m_it; } - const TimeSeriesProvider * operator -> () const { return get(); } - const TimeSeriesProvider & ts() const { return *get(); } + int64_t elemId() const { return m_elemId; } + const TimeSeriesProvider * get() const { return *m_it; } + const TimeSeriesProvider * operator->() const { return get(); } + const TimeSeriesProvider & ts() const { return *get(); } private: TimeSeriesProvider const * const * m_it; @@ -163,9 +179,9 @@ class InputBasketInfo return ticked_iterator( m_inputs, m_tickedInputs.end(), m_tickedInputs.end() ); } - valid_iterator begin_valid() const { return valid_iterator( m_inputs, m_inputs + m_size ); } + valid_iterator begin_valid() const { return valid_iterator( m_inputs, m_inputs + m_size ); } - input_iterator begin_inputs( bool include_hidden ) const + input_iterator begin_inputs( bool include_hidden ) const { if( include_hidden && m_isDynamic ) return input_iterator( m_inputs - 1, m_inputs + m_size, -1 ); @@ -173,12 +189,9 @@ class InputBasketInfo } protected: - bool hasTickedInputs() const - { - return m_lastCycleCount == m_rootEngine -> cycleCount(); - } + bool hasTickedInputs() const { return m_lastCycleCount == m_rootEngine->cycleCount(); } - //not owned + // not owned TimeSeriesProvider const ** m_inputs; TickedInputs m_tickedInputs; INOUT_ELEMID_TYPE m_size; @@ -191,33 +204,36 @@ class InputBasketInfo class DynamicInputBasketInfo : public InputBasketInfo { public: - using ChangeCallback = std::function; + using ChangeCallback + = std::function; using Keys = std::vector; - DynamicInputBasketInfo( RootEngine * rootEngine ) : InputBasketInfo( rootEngine, 0, true ), - m_capacity(0), m_tickCountPolicy( 1 ) + DynamicInputBasketInfo( RootEngine * rootEngine ) + : InputBasketInfo( rootEngine, 0, true ) + , m_capacity( 0 ) + , m_tickCountPolicy( 1 ) { - //Note that dynamic baskets are always valid, no need to compute it + // Note that dynamic baskets are always valid, no need to compute it m_valid = true; } ~DynamicInputBasketInfo() {} - DynamicInputBasketInfo( const DynamicInputBasketInfo & ) = delete; + DynamicInputBasketInfo( const DynamicInputBasketInfo & ) = delete; DynamicInputBasketInfo & operator=( const DynamicInputBasketInfo & ) = delete; - void setChangeCallback( ChangeCallback cb ) { m_changeCallback = std::move( cb ); } const TimeSeriesProvider * shapeTs() const { return m_inputs[-1]; } int64_t addDynamicKey( const DialectGenericType & key, const TimeSeriesProvider * ts ); - //removeID is element to remove, replaceId ( if >= 0 ) is element to take its place ( compaction ) + // removeID is element to remove, replaceId ( if >= 0 ) is element to take its place ( compaction ) void removeDynamicKey( uint64_t engineCycle, const DialectGenericType & key, int64_t removeId, int64_t replaceId ); - //for dynamic input baskets we maintain bufffering policy on the dynamic basket so that they get applied to newly added inputs - void setTickCountPolicy( int32_t tickCount ) { m_tickCountPolicy = tickCount; } + // for dynamic input baskets we maintain bufffering policy on the dynamic basket so that they get applied to newly + // added inputs + void setTickCountPolicy( int32_t tickCount ) { m_tickCountPolicy = tickCount; } void setTickTimeWindowPolicy( TimeDelta window ) { m_timeWindowPolicy = window; } private: @@ -233,13 +249,13 @@ class OutputBasketInfo OutputBasketInfo( CspTypePtr & type, Node * node, size_t size, bool isDynamic = false ); ~OutputBasketInfo(); - OutputBasketInfo( const OutputBasketInfo & rhs ) = delete; + OutputBasketInfo( const OutputBasketInfo & rhs ) = delete; OutputBasketInfo & operator=( const OutputBasketInfo & rhs ) = delete; TimeSeriesProvider * elem( int64_t elemId ) const { CSP_ASSERT( elemId >= 0 && elemId < m_size ); - return m_outputs[ elemId ]; + return m_outputs[elemId]; } size_t size() const { return m_size; } @@ -247,11 +263,10 @@ class OutputBasketInfo bool isDynamicBasket() const { return m_isDynamic; } protected: - - //owned - TimeSeriesProvider ** m_outputs; - INOUT_ELEMID_TYPE m_size; - bool m_isDynamic; + // owned + TimeSeriesProvider ** m_outputs; + INOUT_ELEMID_TYPE m_size; + bool m_isDynamic; }; class DynamicOutputBasketInfo : public OutputBasketInfo @@ -260,37 +275,37 @@ class DynamicOutputBasketInfo : public OutputBasketInfo DynamicOutputBasketInfo( CspTypePtr & type, Node * node ); ~DynamicOutputBasketInfo(); - DynamicOutputBasketInfo( const DynamicOutputBasketInfo & rhs ) = delete; + DynamicOutputBasketInfo( const DynamicOutputBasketInfo & rhs ) = delete; DynamicOutputBasketInfo & operator=( const DynamicOutputBasketInfo & rhs ) = delete; - //returns element id of added key, pushes shape tick + // returns element id of added key, pushes shape tick int64_t addDynamicKey( const DialectGenericType & key ); - //this is used exclusively by DynamicNode ( dynamic graphs ). See comment below on m_ownTs; + // this is used exclusively by DynamicNode ( dynamic graphs ). See comment below on m_ownTs; int64_t addDynamicTs( const DialectGenericType & key, const TimeSeriesProvider * ts ); - //when we remove elem N, we take elem at the end of the basket and move it into its slot - //returns index of ts moved from end to take this slot + // when we remove elem N, we take elem at the end of the basket and move it into its slot + // returns index of ts moved from end to take this slot int64_t removeDynamicKey( const DialectGenericType & key, int64_t elemId ); void linkInputBasket( Node * node, INOUT_ID_TYPE inputIdx ); private: - //called for every add + // called for every add bool addCapacity(); void propagateAddKey( const DialectGenericType & key, const TimeSeriesProvider * ts ); void addShapeChange( const DialectGenericType & key, bool added ); TimeSeriesProvider m_shapeTs; CspTypePtr m_elemType; - Node * m_parentNode; + Node * m_parentNode; INOUT_ELEMID_TYPE m_capacity; - //for dynamic graph basket outputs we take an existing ts and stick it in the dynamic output basket - //for efficiency ( alternative is to own a copy on the outpu basket and copy every tick, which we want to avoid ) - bool m_ownTs; + // for dynamic graph basket outputs we take an existing ts and stick it in the dynamic output basket + // for efficiency ( alternative is to own a copy on the outpu basket and copy every tick, which we want to avoid ) + bool m_ownTs; }; -} +} // namespace csp #endif diff --git a/cpp/csp/engine/ConstInputAdapter.h b/cpp/csp/engine/ConstInputAdapter.h index 691b55078..0b564f2ed 100644 --- a/cpp/csp/engine/ConstInputAdapter.h +++ b/cpp/csp/engine/ConstInputAdapter.h @@ -10,32 +10,31 @@ template class ConstInputAdapter final : public InputAdapter { public: - ConstInputAdapter( Engine * engine, CspTypePtr & type, const T & value, - TimeDelta delay ) : InputAdapter( engine, type, PushMode::LAST_VALUE ), m_delay( delay ), m_value( value ) + ConstInputAdapter( Engine * engine, CspTypePtr & type, const T & value, TimeDelta delay ) + : InputAdapter( engine, type, PushMode::LAST_VALUE ) + , m_delay( delay ) + , m_value( value ) { } void start( DateTime start, DateTime end ) override { - m_timerHandle = rootEngine() -> scheduleCallback( m_delay, - [this] - { - this -> outputTickTyped( rootEngine() -> now(), m_value ); - return nullptr; - } ); + m_timerHandle = rootEngine()->scheduleCallback( m_delay, + [this] + { + this->outputTickTyped( rootEngine()->now(), m_value ); + return nullptr; + } ); } - void stop() override - { - rootEngine() -> cancelCallback( m_timerHandle ); - } + void stop() override { rootEngine()->cancelCallback( m_timerHandle ); } private: Scheduler::Handle m_timerHandle; - TimeDelta m_delay; - T m_value; + TimeDelta m_delay; + T m_value; }; -}; +}; // namespace csp #endif diff --git a/cpp/csp/engine/Consumer.cpp b/cpp/csp/engine/Consumer.cpp index 4217bf0bf..79c6584f2 100644 --- a/cpp/csp/engine/Consumer.cpp +++ b/cpp/csp/engine/Consumer.cpp @@ -5,23 +5,18 @@ namespace csp { -Consumer::Consumer( Engine * engine ) : m_engine( engine ), - m_next( nullptr ), - m_rank( -1 ), - m_started( false ) +Consumer::Consumer( Engine * engine ) + : m_engine( engine ) + , m_next( nullptr ) + , m_rank( -1 ) + , m_started( false ) { } -Consumer::~Consumer() -{ -} +Consumer::~Consumer() {} -void Consumer::start() -{ -} +void Consumer::start() {} -void Consumer::stop() -{ -} +void Consumer::stop() {} -} +} // namespace csp diff --git a/cpp/csp/engine/Consumer.h b/cpp/csp/engine/Consumer.h index f9d2bc7c6..5125f7c9a 100644 --- a/cpp/csp/engine/Consumer.h +++ b/cpp/csp/engine/Consumer.h @@ -12,102 +12,102 @@ namespace csp class InputAdapter; -//Base of either regular Nodes or output adapters +// Base of either regular Nodes or output adapters class Consumer { public: Consumer( Engine * ); virtual ~Consumer(); - Consumer(const Consumer&) = delete; - Consumer(Consumer&&) = delete; - Consumer& operator=(const Consumer&) = delete; - Consumer& operator=(const Consumer&&) = delete; + Consumer( const Consumer & ) = delete; + Consumer( Consumer && ) = delete; + Consumer & operator=( const Consumer & ) = delete; + Consumer & operator=( const Consumer && ) = delete; virtual void start(); virtual void stop(); virtual const char * name() const = 0; - Engine * engine() const { return m_engine; } - RootEngine * rootEngine() const { return m_engine -> rootEngine(); } + Engine * engine() const { return m_engine; } + RootEngine * rootEngine() const { return m_engine->rootEngine(); } - DateTime now() const { return rootEngine() -> now(); } - uint64_t cycleCount() const { return rootEngine() -> cycleCount(); } + DateTime now() const { return rootEngine()->now(); } + uint64_t cycleCount() const { return rootEngine()->cycleCount(); } - void setStarted() { m_started = true; } - bool started() const { return m_started; } + void setStarted() { m_started = true; } + bool started() const { return m_started; } - //called when input timeseries has an event, schedules in - //step propagation. See if we can do better than virtual per tick... - virtual void handleEvent( InputId id ) - { - m_engine -> scheduleConsumer( this ); - } + // called when input timeseries has an event, schedules in + // step propagation. See if we can do better than virtual per tick... + virtual void handleEvent( InputId id ) { m_engine->scheduleConsumer( this ); } - void execute() - { - executeImpl(); - } + void execute() { executeImpl(); } - //actual logic + // actual logic virtual void executeImpl() = 0; - //internals + // internals - //graph / links / creation - int32_t rank() const { return m_rank; } + // graph / links / creation + int32_t rank() const { return m_rank; } void setRank( int32_t rank ) { m_rank = rank; } - - Consumer * next() { return m_next; } - void setNext( Consumer * next ) { m_next = next; } - //flattens out all inputs and basket inputs into one iteration of all input timeseries + Consumer * next() { return m_next; } + void setNext( Consumer * next ) { m_next = next; } + + // flattens out all inputs and basket inputs into one iteration of all input timeseries struct input_iterator { - using InputT = TaggedPointerUnion; + using InputT = TaggedPointerUnion; - //single input ( OutputAdapter ) case + // single input ( OutputAdapter ) case input_iterator( const TimeSeriesProvider * const * input ) { m_id = 0; - //safe cast since TaggedPointerUnion is a single ptr, and isSet will be false - m_inputiter = ( InputT * ) input; + // safe cast since TaggedPointerUnion is a single ptr, and isSet will be false + m_inputiter = (InputT *)input; m_inputend = m_inputiter + 1; } input_iterator( const InputT * inputs, size_t num_inputs ) { - m_id = 0; + m_id = 0; m_inputiter = inputs; m_inputend = inputs + num_inputs; - if( m_inputiter != m_inputend && m_inputiter -> isSet() ) - m_basketIter = m_inputiter -> get() -> begin_inputs( true ); + if( m_inputiter != m_inputend && m_inputiter->isSet() ) + m_basketIter = m_inputiter->get()->begin_inputs( true ); + } + + const TimeSeriesProvider * get() const + { + return m_basketIter ? m_basketIter.get() : m_inputiter->get(); } - - const TimeSeriesProvider * get() const { return m_basketIter ? m_basketIter.get() : m_inputiter -> get(); } - const TimeSeriesProvider * operator ->() const { return get(); } - const TimeSeriesProvider * ts() const { return get(); } + const TimeSeriesProvider * operator->() const { return get(); } + const TimeSeriesProvider * ts() const { return get(); } - InputId inputId() const { return InputId( m_id, ( m_basketIter ? m_basketIter.elemId() : InputId::ELEM_ID_NONE ) ); } + InputId inputId() const + { + return InputId( m_id, ( m_basketIter ? m_basketIter.elemId() : InputId::ELEM_ID_NONE ) ); + } - operator bool() const { return m_inputiter != m_inputend; } - input_iterator & operator++() + operator bool() const { return m_inputiter != m_inputend; } + input_iterator & operator++() { - //advance basket iteration if active + // advance basket iteration if active if( m_basketIter ) ++m_basketIter; - //if basket iterator is not active / no longer active go to next input + // if basket iterator is not active / no longer active go to next input if( !m_basketIter ) { ++m_inputiter; ++m_id; - if( m_inputiter != m_inputend && m_inputiter -> isSet() ) - m_basketIter = m_inputiter -> get() -> begin_inputs( true ); + if( m_inputiter != m_inputend && m_inputiter->isSet() ) + m_basketIter = m_inputiter->get()->begin_inputs( true ); } - return *this; + return *this; } private: @@ -118,20 +118,19 @@ class Consumer InputBasketInfo::input_iterator m_basketIter; }; - //this is only currently used on startup for ranking... shouldnt be called during runtime + // this is only currently used on startup for ranking... shouldnt be called during runtime virtual input_iterator inputs() const = 0; private: - Engine * m_engine; - //for intrusive linked list + // for intrusive linked list Consumer * m_next; - - int32_t m_rank; - bool m_started; -}; + int32_t m_rank; + bool m_started; }; +}; // namespace csp + #endif diff --git a/cpp/csp/engine/CppNode.h b/cpp/csp/engine/CppNode.h index f4d33f58e..0b781bb9b 100644 --- a/cpp/csp/engine/CppNode.h +++ b/cpp/csp/engine/CppNode.h @@ -11,12 +11,12 @@ namespace csp { -//CppNode is used specifically for C++ defined Nodes, and should only be used -//for definig c++ nodes using the macros defined at the end +// CppNode is used specifically for C++ defined Nodes, and should only be used +// for definig c++ nodes using the macros defined at the end class CppNode : public csp::Node { public: - using Shape = std::variant>; + using Shape = std::variant>; struct InOutDef { INOUT_ID_TYPE index; @@ -25,7 +25,7 @@ class CppNode : public csp::Node Shape shape; }; - using InOutDefs = std::unordered_map; + using InOutDefs = std::unordered_map; struct NodeDef { @@ -37,28 +37,29 @@ class CppNode : public csp::Node const InOutDef & tsinputDef( const char * inputName ) const { validateNodeDef(); - auto it = m_nodedef -> inputs.find( inputName ); - if( it == m_nodedef -> inputs.end() ) + auto it = m_nodedef->inputs.find( inputName ); + if( it == m_nodedef->inputs.end() ) CSP_THROW( ValueError, "CppNode failed to find input " << inputName << " on node " << name() ); - return it -> second; + return it->second; } const InOutDef & tsoutputDef( const char * outputName ) const { validateNodeDef(); - auto it = m_nodedef -> outputs.find( outputName ); - if( it == m_nodedef -> outputs.end() ) + auto it = m_nodedef->outputs.find( outputName ); + if( it == m_nodedef->outputs.end() ) CSP_THROW( ValueError, "CppNode failed to find output " << outputName << " on node " << name() ); - return it -> second; + return it->second; } const InOutDef & alarmDef( const char * alarmName ) const { auto & def = tsinputDef( alarmName ); if( !def.isAlarm ) - CSP_THROW( TypeError, "CppNode expected alarm " << alarmName << " but found it as an input on node " << name() ); + CSP_THROW( TypeError, + "CppNode expected alarm " << alarmName << " but found it as an input on node " << name() ); return def; } @@ -66,31 +67,34 @@ class CppNode : public csp::Node T scalarValue( const char * scalarName ) const { validateNodeDef(); - if( !m_nodedef -> scalars.exists( scalarName ) ) + if( !m_nodedef->scalars.exists( scalarName ) ) CSP_THROW( ValueError, "CppNode failed to find scalar " << scalarName << " on node " << name() ); - return m_nodedef -> scalars.get( scalarName ); + return m_nodedef->scalars.get( scalarName ); } void resetNodeDef() { m_nodedef = nullptr; } - DateTime now() const { return rootEngine() -> now(); } + DateTime now() const { return rootEngine()->now(); } - using Creator = std::function; + using Creator = std::function; protected: - CppNode( Engine * engine, - const NodeDef & nodedef ) : csp::Node( asCspNodeDef( nodedef ), engine ), - m_nodedef( &nodedef ) - {} + CppNode( Engine * engine, const NodeDef & nodedef ) + : csp::Node( asCspNodeDef( nodedef ), engine ) + , m_nodedef( &nodedef ) + { + } csp::NodeDef asCspNodeDef( const NodeDef & nodedef ) const { if( nodedef.inputs.size() > InputId::maxInputs() ) - CSP_THROW( ValueError, "number of inputs exceeds limit of " << InputId::maxInputs() << " on node " << name() ); + CSP_THROW( ValueError, + "number of inputs exceeds limit of " << InputId::maxInputs() << " on node " << name() ); if( nodedef.outputs.size() > OutputId::maxOutputs() ) - CSP_THROW( ValueError, "number of outputs exceeds limit of " << OutputId::maxOutputs() << " on node " << name() ); + CSP_THROW( ValueError, + "number of outputs exceeds limit of " << OutputId::maxOutputs() << " on node " << name() ); return csp::NodeDef{ INOUT_ID_TYPE( nodedef.inputs.size() ), INOUT_ID_TYPE( nodedef.outputs.size() ) }; } @@ -101,46 +105,56 @@ class CppNode : public csp::Node CSP_THROW( RuntimeException, "CppNode cpp nodedef information is only available during INIT" ); } - struct Generic {}; + struct Generic + { + }; class InputWrapper { public: - //for non-basket inputs only - InputWrapper( const char *name, const CppNode & node ) : m_node( node ), - m_id( 0 ) + // for non-basket inputs only + InputWrapper( const char * name, const CppNode & node ) + : m_node( node ) + , m_id( 0 ) { auto & def = m_node.tsinputDef( name ); if( def.isAlarm ) - CSP_THROW( TypeError, "CppNode expected input " << name << " but found it as an alarm on node " << m_node.name() ); + CSP_THROW( TypeError, + "CppNode expected input " << name << " but found it as an alarm on node " << m_node.name() ); m_id = InputId( def.index ); } - //for basket inputs - InputWrapper( const CppNode & node, InputId id ) : m_node( node ), - m_id( id ) + // for basket inputs + InputWrapper( const CppNode & node, InputId id ) + : m_node( node ) + , m_id( id ) { } - bool valid() const { return ts() -> valid(); } + bool valid() const { return ts()->valid(); } bool ticked() const { return m_node.inputTicked( m_id ); } - bool makeActive() const { return const_cast( m_node ).makeActive( m_id ); } - bool makePassive() const { return const_cast( m_node ).makePassive( m_id ); } + bool makeActive() const { return const_cast( m_node ).makeActive( m_id ); } + bool makePassive() const { return const_cast( m_node ).makePassive( m_id ); } - uint32_t count() const { return ts() -> count(); } + uint32_t count() const { return ts()->count(); } - const CspType * type() const { return ts() -> type(); } + const CspType * type() const { return ts()->type(); } - void setTickCountPolicy( int32_t tickCount ) { const_cast(ts()) -> setTickCountPolicy( tickCount ); } - void setTickTimeWindowPolicy( TimeDelta window ) { const_cast(ts()) -> setTickTimeWindowPolicy( window ); } + void setTickCountPolicy( int32_t tickCount ) + { + const_cast( ts() )->setTickCountPolicy( tickCount ); + } + void setTickTimeWindowPolicy( TimeDelta window ) + { + const_cast( ts() )->setTickTimeWindowPolicy( window ); + } protected: const TimeSeriesProvider * ts() const { return m_node.input( m_id ); } const CppNode & m_node; InputId m_id; - }; class GenericInputWrapper : public InputWrapper @@ -149,10 +163,16 @@ class CppNode : public csp::Node using InputWrapper::InputWrapper; template - const T & lastValue() const { return ts() -> lastValueTyped(); } + const T & lastValue() const + { + return ts()->lastValueTyped(); + } template - const T & valueAtIndex( int32_t index ) const { return ts() -> valueAtIndex( index ); } + const T & valueAtIndex( int32_t index ) const + { + return ts()->valueAtIndex( index ); + } }; template @@ -162,20 +182,21 @@ class CppNode : public csp::Node using InputWrapper::InputWrapper; operator const T &() { return lastValue(); } - const T & lastValue() const { return ts() -> template lastValueTyped(); } + const T & lastValue() const { return ts()->template lastValueTyped(); } - const T & valueAtIndex( int32_t index ) const { return ts() -> template valueAtIndex( index ); } + const T & valueAtIndex( int32_t index ) const { return ts()->template valueAtIndex( index ); } }; template class BasketInputWrapper { public: - BasketInputWrapper( const char * name, const CppNode & node ) : m_node( node ) + BasketInputWrapper( const char * name, const CppNode & node ) + : m_node( node ) { auto & def = m_node.tsinputDef( name ); - m_id = def.index; - m_type = def.type; + m_id = def.index; + m_type = def.type; } ElemWrapperT operator[]( INOUT_ELEMID_TYPE elemId ) const @@ -183,57 +204,49 @@ class CppNode : public csp::Node return ElemWrapperT( m_node, InputId( m_id, elemId ) ); } - bool makeActive() const - { - return const_cast( m_node ).makeBasketActive( m_id ); - } + bool makeActive() const { return const_cast( m_node ).makeBasketActive( m_id ); } - bool makePassive() const - { - return const_cast( m_node ).makeBasketPassive( m_id ); - } + bool makePassive() const { return const_cast( m_node ).makeBasketPassive( m_id ); } - bool valid() const { return basketInfo() -> allValid(); } - bool ticked() const { return basketInfo() -> ticked(); } + bool valid() const { return basketInfo()->allValid(); } + bool ticked() const { return basketInfo()->ticked(); } - InputBasketInfo::ticked_iterator tickedinputs() const { return basketInfo() -> begin_ticked(); } - InputBasketInfo::valid_iterator validinputs() const { return basketInfo() -> begin_valid(); } + InputBasketInfo::ticked_iterator tickedinputs() const { return basketInfo()->begin_ticked(); } + InputBasketInfo::valid_iterator validinputs() const { return basketInfo()->begin_valid(); } - size_t size() const { return basketInfo() -> size(); } + size_t size() const { return basketInfo()->size(); } /** - * @return The declared input types. NOTE: The declared input type. Note, we allow connecting slightly different types than declared - * types (derived class ts where baseclass ts is expected, native type to dialect generic ...) so care should be taking when using this type + * @return The declared input types. NOTE: The declared input type. Note, we allow connecting slightly different + * types than declared types (derived class ts where baseclass ts is expected, native type to dialect generic + * ...) so care should be taking when using this type */ - const CspTypePtr& type() const {return m_type;} + const CspTypePtr & type() const { return m_type; } protected: - const InputBasketInfo * basketInfo() const { return m_node.inputBasket( m_id ); } - void initBasket( size_t size ) - { - const_cast( this -> m_node ).initInputBasket( m_id, size, false ); - } + void initBasket( size_t size ) { const_cast( this->m_node ).initInputBasket( m_id, size, false ); } - const CppNode &m_node; - INOUT_ID_TYPE m_id; - CspTypePtr m_type; + const CppNode & m_node; + INOUT_ID_TYPE m_id; + CspTypePtr m_type; }; template class DictInputBasketWrapper : public BasketInputWrapper { public: - DictInputBasketWrapper( const char * name, const CppNode & node ) : BasketInputWrapper( name, node ) + DictInputBasketWrapper( const char * name, const CppNode & node ) + : BasketInputWrapper( name, node ) { - auto & def = node.tsinputDef( name ); - m_shape = std::get>( def.shape ); + auto & def = node.tsinputDef( name ); + m_shape = std::get>( def.shape ); INOUT_ELEMID_TYPE elemId = 0; for( auto & key : m_shape ) - m_keyMap[ key ] = elemId++; + m_keyMap[key] = elemId++; - //init input basket at this point - this -> initBasket( m_shape.size() ); + // init input basket at this point + this->initBasket( m_shape.size() ); } int64_t elemId( const std::string & key ) @@ -241,30 +254,32 @@ class CppNode : public csp::Node auto it = m_keyMap.find( key ); if( it == m_keyMap.end() ) return InputId::ELEM_ID_NONE; - return it -> second; + return it->second; } const std::vector & shape() const { return m_shape; } + private: - std::vector m_shape; - std::unordered_map m_keyMap; + std::vector m_shape; + std::unordered_map m_keyMap; }; template class ListInputBasketWrapper : public BasketInputWrapper { public: - ListInputBasketWrapper( const char * name, const CppNode & node ) : BasketInputWrapper( name, node ) + ListInputBasketWrapper( const char * name, const CppNode & node ) + : BasketInputWrapper( name, node ) { - auto & def = node.tsinputDef( name ); - auto shape = std::get( def.shape ); + auto & def = node.tsinputDef( name ); + auto shape = std::get( def.shape ); - //init input basket at this point - this -> initBasket( shape ); + // init input basket at this point + this->initBasket( shape ); } }; - //type selection helper for when requesting Generic input + // type selection helper for when requesting Generic input template struct InputTypeHelper { @@ -283,11 +298,11 @@ class CppNode : public csp::Node using type = DictInputBasketWrapper>; }; - //Generic alarm can only be used to schedule from generic inputs + // Generic alarm can only be used to schedule from generic inputs struct GenericAlarmWrapper : public GenericInputWrapper { - GenericAlarmWrapper( const char *name, const CppNode & node ) : GenericInputWrapper( node, - InputId( node.alarmDef( name ).index ) ) + GenericAlarmWrapper( const char * name, const CppNode & node ) + : GenericInputWrapper( node, InputId( node.alarmDef( name ).index ) ) { } @@ -299,66 +314,69 @@ class CppNode : public csp::Node Scheduler::Handle scheduleAlarm( DateTime time, const GenericInputWrapper & input ) { assert( input.type() == type() ); - return switchCspType( input.type(), [this,time,&input]( auto tag ) - { - using T = typename decltype(tag)::type; - auto * alarm = static_cast *>( const_cast( ts() ) ); - return alarm -> scheduleAlarm( time, input.lastValue() ); - } ); + return switchCspType( input.type(), + [this, time, &input]( auto tag ) + { + using T = typename decltype( tag )::type; + auto * alarm = static_cast *>( + const_cast( ts() ) ); + return alarm->scheduleAlarm( time, input.lastValue() ); + } ); } }; template struct TypedAlarmWrapper : public TypedInputWrapper { - TypedAlarmWrapper( const char *name, const CppNode & node ) : TypedInputWrapper( node, - InputId( node.alarmDef( name ).index ) ) + TypedAlarmWrapper( const char * name, const CppNode & node ) + : TypedInputWrapper( node, InputId( node.alarmDef( name ).index ) ) { } Scheduler::Handle scheduleAlarm( TimeDelta delta, const T & value ) { - auto * alarm = static_cast *>( const_cast( this -> ts() ) ); - return alarm -> scheduleAlarm( delta, value ); + auto * alarm = static_cast *>( const_cast( this->ts() ) ); + return alarm->scheduleAlarm( delta, value ); } Scheduler::Handle scheduleAlarm( DateTime time, const T & value ) { - auto * alarm = static_cast *>( const_cast( this -> ts() ) ); - return alarm -> scheduleAlarm( time, value ); + auto * alarm = static_cast *>( const_cast( this->ts() ) ); + return alarm->scheduleAlarm( time, value ); } void cancelAlarm( Scheduler::Handle handle ) { - auto * alarm = static_cast *>( const_cast( this -> ts() ) ); - alarm -> cancelAlarm( handle ); + auto * alarm = static_cast *>( const_cast( this->ts() ) ); + alarm->cancelAlarm( handle ); } }; - //type selection helper for when requesting Generic alarms + // type selection helper for when requesting Generic alarms template struct AlarmTypeHelper { using type = TypedAlarmWrapper; }; - class OutputWrapper { public: - //non-basket outputs - OutputWrapper( const char * name, const CppNode & node ) : m_node( node ), - m_id( m_node.tsoutputDef( name ).index ) + // non-basket outputs + OutputWrapper( const char * name, const CppNode & node ) + : m_node( node ) + , m_id( m_node.tsoutputDef( name ).index ) { } - //basket outputs - OutputWrapper( const CppNode & node, OutputId id ) : m_node( node ), - m_id( id ) + // basket outputs + OutputWrapper( const CppNode & node, OutputId id ) + : m_node( node ) + , m_id( id ) { } - const CspType * type() const { return ts() -> type(); } + const CspType * type() const { return ts()->type(); } protected: TimeSeriesProvider * ts() const { return m_node.output( m_id ); } @@ -373,19 +391,13 @@ class CppNode : public csp::Node public: using OutputWrapper::OutputWrapper; - void output( const T & value ) - { - ts() -> outputTickTyped( m_node.cycleCount(), m_node.now(), value ); - } + void output( const T & value ) { ts()->outputTickTyped( m_node.cycleCount(), m_node.now(), value ); } - T & reserveSpace() - { - return ts() -> template reserveTickTyped( m_node.cycleCount(), m_node.now() ); - } + T & reserveSpace() { return ts()->template reserveTickTyped( m_node.cycleCount(), m_node.now() ); } }; - //this is for nodes that dont actually inspect the data, they get 'T' in - //and return 'T' out + // this is for nodes that dont actually inspect the data, they get 'T' in + // and return 'T' out class GenericOutputWrapper : public OutputWrapper { public: @@ -394,30 +406,30 @@ class CppNode : public csp::Node void output( const GenericInputWrapper & input ) { assert( input.type() == type() ); - switchCspType( input.type(), [this,&input]( auto tag ) + switchCspType( input.type(), + [this, &input]( auto tag ) { - using T = typename decltype(tag)::type; - ts() -> outputTickTyped( m_node.cycleCount(), m_node.now(), - input.lastValue() ); + using T = typename decltype( tag )::type; + ts()->outputTickTyped( m_node.cycleCount(), m_node.now(), input.lastValue() ); } ); } void output( const GenericAlarmWrapper & alarm ) { - //force call non-template version - output( static_cast( alarm ) ); + // force call non-template version + output( static_cast( alarm ) ); } template void output( const T & value ) { - ts() -> outputTickTyped( m_node.cycleCount(), m_node.now(), value ); + ts()->outputTickTyped( m_node.cycleCount(), m_node.now(), value ); } template T & reserveSpace() { - return ts() -> reserveTickTyped( m_node.cycleCount(), m_node.now() ); + return ts()->reserveTickTyped( m_node.cycleCount(), m_node.now() ); } }; @@ -425,10 +437,11 @@ class CppNode : public csp::Node class BasketOutputWrapper { public: - BasketOutputWrapper( const char * name, const CppNode & node ) : m_node( node ) + BasketOutputWrapper( const char * name, const CppNode & node ) + : m_node( node ) { auto & def = m_node.tsoutputDef( name ); - m_id = def.index; + m_id = def.index; } ElemWrapperT operator[]( INOUT_ELEMID_TYPE elemId ) const @@ -445,13 +458,14 @@ class CppNode : public csp::Node class DictOutputBasketWrapper : public BasketOutputWrapper { public: - DictOutputBasketWrapper( const char * name, const CppNode & node ) : BasketOutputWrapper( name, node ) + DictOutputBasketWrapper( const char * name, const CppNode & node ) + : BasketOutputWrapper( name, node ) { - auto & def = node.tsoutputDef( name ); - auto & shape = std::get>( def.shape ); + auto & def = node.tsoutputDef( name ); + auto & shape = std::get>( def.shape ); INOUT_ELEMID_TYPE elemId = 0; for( auto & key : shape ) - m_keyMap[ key ] = elemId++; + m_keyMap[key] = elemId++; } int64_t elemId( const std::string & key ) @@ -459,25 +473,25 @@ class CppNode : public csp::Node auto it = m_keyMap.find( key ); if( it == m_keyMap.end() ) return InputId::ELEM_ID_NONE; - return it -> second; + return it->second; } private: - std::unordered_map m_keyMap; - + std::unordered_map m_keyMap; }; template class ListOutputBasketWrapper : public BasketOutputWrapper { public: - ListOutputBasketWrapper( const char * name, const CppNode & node ) : BasketOutputWrapper( name, node ) + ListOutputBasketWrapper( const char * name, const CppNode & node ) + : BasketOutputWrapper( name, node ) { // nothing needs to be done here } }; - //type selection helper for when requesting Generic outputs + // type selection helper for when requesting Generic outputs template struct OutputTypeHelper { @@ -499,10 +513,7 @@ class CppNode : public csp::Node template struct Scalar { - Scalar( const char * name, const CppNode & node ) - { - m_value = node.scalarValue( name ); - } + Scalar( const char * name, const CppNode & node ) { m_value = node.scalarValue( name ); } operator const T &() const { return m_value; } @@ -512,44 +523,48 @@ class CppNode : public csp::Node T m_value; }; - //for python-like csp namespace interface + // for python-like csp namespace interface class CSP { public: - //these two are templatized to take basket inputs as well + // these two are templatized to take basket inputs as well template - static bool make_passive( const InputWrapperT & input ) { return input.makePassive(); } + static bool make_passive( const InputWrapperT & input ) + { + return input.makePassive(); + } template - static bool make_active( const InputWrapperT & input ) { return input.makeActive(); } + static bool make_active( const InputWrapperT & input ) + { + return input.makeActive(); + } static bool ticked() { return false; } - template - static bool ticked( const T & input, const Args&... args ) + template + static bool ticked( const T & input, const Args &... args ) { return input.ticked() || ticked( args... ); } - static bool valid() { return true; } - template - static bool valid( const T & input, const Args&... args ) + template + static bool valid( const T & input, const Args &... args ) { return input.valid() && valid( args... ); } - static uint32_t count( const InputWrapper & input ) - { - return input.count(); - } + static uint32_t count( const InputWrapper & input ) { return input.count(); } - static Scheduler::Handle schedule_alarm( GenericAlarmWrapper & alarm, DateTime time, const GenericInputWrapper & input ) + static Scheduler::Handle schedule_alarm( GenericAlarmWrapper & alarm, DateTime time, + const GenericInputWrapper & input ) { return alarm.scheduleAlarm( time, input ); } - static Scheduler::Handle schedule_alarm( GenericAlarmWrapper & alarm, TimeDelta delta, const GenericInputWrapper & input ) + static Scheduler::Handle schedule_alarm( GenericAlarmWrapper & alarm, TimeDelta delta, + const GenericInputWrapper & input ) { return alarm.scheduleAlarm( delta, input ); } @@ -585,12 +600,11 @@ class CppNode : public csp::Node } }; - //this is set temporarily for constructing input and scalar wrappers - //easiuest way to get member-init to find the information. ptr is null-ed out after init + // this is set temporarily for constructing input and scalar wrappers + // easiuest way to get member-init to find the information. ptr is null-ed out after init const NodeDef * m_nodedef; }; - template<> struct csp::CppNode::InputTypeHelper { @@ -633,62 +647,130 @@ struct csp::CppNode::AlarmTypeHelper using type = GenericAlarmWrapper; }; - #define DECLARE_CPPNODE( Name ) class Name : public csp::CppNode -#define _STATIC_CREATE_METHOD( Class ) \ - static csp::CppNode * create( Engine * engine, const csp::CppNode::NodeDef & nodedef ) \ - { \ - auto * out = engine -> createOwnedObject( nodedef ); \ - out -> resetNodeDef(); \ - return out; \ +#define _STATIC_CREATE_METHOD( Class ) \ + static csp::CppNode * create( Engine * engine, const csp::CppNode::NodeDef & nodedef ) \ + { \ + auto * out = engine->createOwnedObject( nodedef ); \ + out->resetNodeDef(); \ + return out; \ } -#define INIT_CPPNODE_WITH_NAME( Class, Name ) \ - [[maybe_unused]] CSP csp; \ -public: \ - const char * name() const override { return #Name; } \ - _STATIC_CREATE_METHOD( Class ) \ - Class( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) : csp::CppNode( engine, nodedef ) - -#define INIT_CPPNODE( Class ) INIT_CPPNODE_WITH_NAME( Class, Class ) - -#define TS_INPUT( Type, Name ) typename InputTypeHelper::type Name{#Name,*this} -#define TS_INPUT_RENAMED( Type, DeclName, VarName ) typename InputTypeHelper::type VarName{#DeclName,*this} -#define TS_LISTBASKET_INPUT( Type, Name ) typename ListInputBasketTypeHelper::type Name{#Name,*this} -#define TS_DICTBASKET_INPUT( Type, Name ) typename DictInputBasketTypeHelper::type Name{#Name,*this} +#define INIT_CPPNODE_WITH_NAME( Class, Name ) \ + [[maybe_unused]] CSP csp; \ + \ +public: \ + const char * name() const override \ + { \ + return #Name; \ + } \ + _STATIC_CREATE_METHOD( Class ) \ + Class( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) \ + : csp::CppNode( engine, nodedef ) + +#define INIT_CPPNODE( Class ) INIT_CPPNODE_WITH_NAME( Class, Class ) + +#define TS_INPUT( Type, Name ) \ + typename InputTypeHelper::type Name \ + { \ + #Name, *this \ + } +#define TS_INPUT_RENAMED( Type, DeclName, VarName ) \ + typename InputTypeHelper::type VarName \ + { \ + #DeclName, *this \ + } +#define TS_LISTBASKET_INPUT( Type, Name ) \ + typename ListInputBasketTypeHelper::type Name \ + { \ + #Name, *this \ + } +#define TS_DICTBASKET_INPUT( Type, Name ) \ + typename DictInputBasketTypeHelper::type Name \ + { \ + #Name, *this \ + } -#define TS_OUTPUT( Type ) typename OutputTypeHelper::type __unnamed_output{"",*this}; auto & unnamed_output() { return __unnamed_output; } -#define TS_LISTBASKET_OUTPUT( Type ) typename ListOutputBasketTypeHelper::type __unnamed_output{"",*this}; auto & unnamed_output() { return __unnamed_output; } -#define TS_NAMED_LISTBASKET_OUTPUT( Type, Name ) typename ListOutputBasketTypeHelper::type Name{#Name,*this} -#define TS_DICTBASKET_OUTPUT( Type ) typename DictOutputBasketTypeHelper::type __unnamed_output{"",*this}; auto & unnamed_output() { return __unnamed_output; } +#define TS_OUTPUT( Type ) \ + typename OutputTypeHelper::type __unnamed_output{ "", *this }; \ + auto & unnamed_output() \ + { \ + return __unnamed_output; \ + } +#define TS_LISTBASKET_OUTPUT( Type ) \ + typename ListOutputBasketTypeHelper::type __unnamed_output{ "", *this }; \ + auto & unnamed_output() \ + { \ + return __unnamed_output; \ + } +#define TS_NAMED_LISTBASKET_OUTPUT( Type, Name ) \ + typename ListOutputBasketTypeHelper::type Name \ + { \ + #Name, *this \ + } +#define TS_DICTBASKET_OUTPUT( Type ) \ + typename DictOutputBasketTypeHelper::type __unnamed_output{ "", *this }; \ + auto & unnamed_output() \ + { \ + return __unnamed_output; \ + } -#define TS_NAMED_OUTPUT( Type, Name ) typename OutputTypeHelper::type Name{#Name,*this} -#define TS_NAMED_OUTPUT_RENAMED( Type, DeclName, VarName ) typename OutputTypeHelper::type VarName{#DeclName,*this} +#define TS_NAMED_OUTPUT( Type, Name ) \ + typename OutputTypeHelper::type Name \ + { \ + #Name, *this \ + } +#define TS_NAMED_OUTPUT_RENAMED( Type, DeclName, VarName ) \ + typename OutputTypeHelper::type VarName \ + { \ + #DeclName, *this \ + } -#define ALARM( Type, Name ) typename AlarmTypeHelper::type Name{#Name,*this} +#define ALARM( Type, Name ) \ + typename AlarmTypeHelper::type Name \ + { \ + #Name, *this \ + } -#define SCALAR_INPUT( Type, Name ) csp::CppNode::Scalar Name{#Name,*this}; -#define SCALAR_INPUT_RENAMED( Type, DeclName, VarName ) csp::CppNode::Scalar VarName{#DeclName,*this}; +#define SCALAR_INPUT( Type, Name ) csp::CppNode::Scalar Name{ #Name, *this }; +#define SCALAR_INPUT_RENAMED( Type, DeclName, VarName ) csp::CppNode::Scalar VarName{ #DeclName, *this }; #define STATE_VAR( Type, Name ) Type Name; #define START() NO_INLINE void start() override #define STOP() void stop() override -//wrap _executeImpl so we clear basket flags at the end of execute call +// wrap _executeImpl so we clear basket flags at the end of execute call #define INVOKE() void executeImpl() override -//only for unnamed outputs +// only for unnamed outputs #define CSP_OUTPUT( Value ) __unnamed_output.output( Value ) -#define RETURN( Value ) do{ CSP_OUTPUT( Value ); return; } while( 0 ); - -#define SINGLE_ARG(...) __VA_ARGS__ +#define RETURN( Value ) \ + do \ + { \ + CSP_OUTPUT( Value ); \ + return; \ + } while( 0 ); + +#define SINGLE_ARG( ... ) __VA_ARGS__ #define CPPNODE_CREATE_METHOD( Name ) Name##_create_method -#define CPPNODE_CREATE_FWD_DECL( Namespace, Name ) namespace Namespace { csp::CppNode * CPPNODE_CREATE_METHOD( Name )( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ); } -#define EXPORT_CPPNODE( Name ) csp::CppNode * CPPNODE_CREATE_METHOD( Name )( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) { return Name::create( engine, nodedef ); } -#define EXPORT_TEMPLATE_CPPNODE( Name, Typed ) csp::CppNode * CPPNODE_CREATE_METHOD( Name )( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) { return Typed::create( engine, nodedef ); } +#define CPPNODE_CREATE_FWD_DECL( Namespace, Name ) \ + namespace Namespace \ + { \ + csp::CppNode * CPPNODE_CREATE_METHOD( Name )( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ); \ + } +#define EXPORT_CPPNODE( Name ) \ + csp::CppNode * CPPNODE_CREATE_METHOD( Name )( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) \ + { \ + return Name::create( engine, nodedef ); \ + } +#define EXPORT_TEMPLATE_CPPNODE( Name, Typed ) \ + csp::CppNode * CPPNODE_CREATE_METHOD( Name )( csp::Engine * engine, const csp::CppNode::NodeDef & nodedef ) \ + { \ + return Typed::create( engine, nodedef ); \ + } -} +} // namespace csp #endif diff --git a/cpp/csp/engine/CspEnum.cpp b/cpp/csp/engine/CspEnum.cpp index 9bed122a8..63a9a6426 100644 --- a/cpp/csp/engine/CspEnum.cpp +++ b/cpp/csp/engine/CspEnum.cpp @@ -3,33 +3,32 @@ namespace csp { -static CspEnumInstance s_stubInstance( "", 0, new CspEnumMeta( "", CspEnumMeta::ValueDef{{ "", 0 }} ) ); +static CspEnumInstance s_stubInstance( "", 0, new CspEnumMeta( "", CspEnumMeta::ValueDef{ { "", 0 } } ) ); CspEnum::CspEnum() { m_instance = &s_stubInstance; } -CspEnumMeta::CspEnumMeta( const std::string & name, const ValueDef & def ) : m_name( name ) +CspEnumMeta::CspEnumMeta( const std::string & name, const ValueDef & def ) + : m_name( name ) { - for( auto [ key,value ] : def ) + for( auto [key, value] : def ) { auto [rit, inserted] = m_instanceMap.emplace( value, std::make_shared( key, value, this ) ); if( !inserted ) CSP_THROW( TypeError, "CspEnum type " << name << " defined with multiple entries for " << value ); - m_mapping[ rit -> second -> name().c_str() ] = rit; + m_mapping[rit->second->name().c_str()] = rit; } } -CspEnumMeta::~CspEnumMeta() -{ -} +CspEnumMeta::~CspEnumMeta() {} -std::ostream &operator<<( std::ostream &os, const CspEnum & rhs ) +std::ostream & operator<<( std::ostream & os, const CspEnum & rhs ) { os << rhs.name(); return os; }; -} +} // namespace csp diff --git a/cpp/csp/engine/CspEnum.h b/cpp/csp/engine/CspEnum.h index 93f083369..222912b9f 100644 --- a/cpp/csp/engine/CspEnum.h +++ b/cpp/csp/engine/CspEnum.h @@ -4,10 +4,10 @@ #include #include #include +#include #include -#include #include -#include +#include namespace csp { @@ -17,66 +17,79 @@ class CspEnumMeta; class CspEnumInstance { public: - CspEnumInstance( std::string name, int64_t value, csp::CspEnumMeta * meta ) : m_name( name ), m_value( value ), m_meta( meta ) {} - CspEnumInstance( CspEnumInstance && o ) : m_name( o.m_name ), m_value( o.m_value ), m_meta( o.m_meta ) {} - CspEnumInstance( const CspEnumInstance & o ) = delete; + CspEnumInstance( std::string name, int64_t value, csp::CspEnumMeta * meta ) + : m_name( name ) + , m_value( value ) + , m_meta( meta ) + { + } + CspEnumInstance( CspEnumInstance && o ) + : m_name( o.m_name ) + , m_value( o.m_value ) + , m_meta( o.m_meta ) + { + } + CspEnumInstance( const CspEnumInstance & o ) = delete; CspEnumInstance & operator=( CspEnumInstance o ) = delete; - int64_t value() const { return m_value; } + int64_t value() const { return m_value; } const std::string & name() const { return m_name; } const CspEnumMeta * meta() const { return m_meta; } private: - std::string m_name; - int64_t m_value; + std::string m_name; + int64_t m_value; CspEnumMeta * m_meta; }; -//As an optimization we do NOT attach meta or value to every instance of an enum. Instead, the enum -//holds only a pointer to a singleton CspEnumInstance, which holds the value, name, and meta pointer. +// As an optimization we do NOT attach meta or value to every instance of an enum. Instead, the enum +// holds only a pointer to a singleton CspEnumInstance, which holds the value, name, and meta pointer. class CspEnum { public: CspEnum(); CspEnum( const CspEnum & other ) { m_instance = other.m_instance; } - const int64_t value() const { return m_instance -> value(); } - const CspEnumMeta * meta() const { return m_instance -> meta(); } - const std::string & name() const { return m_instance -> name(); } + const int64_t value() const { return m_instance->value(); } + const CspEnumMeta * meta() const { return m_instance->meta(); } + const std::string & name() const { return m_instance->name(); } // check instance to ensure value and meta are the same bool operator==( const CspEnum & rhs ) const { return m_instance == rhs.m_instance; } bool operator!=( const CspEnum & rhs ) const { return m_instance != rhs.m_instance; } protected: - explicit CspEnum( const CspEnumInstance * instance ) : m_instance( instance ) {} + explicit CspEnum( const CspEnumInstance * instance ) + : m_instance( instance ) + { + } const CspEnumInstance * m_instance; friend class CspEnumMeta; }; -std::ostream &operator<<( std::ostream &os, const CspEnum & rhs ); +std::ostream & operator<<( std::ostream & os, const CspEnum & rhs ); class CspEnumMeta { public: - using ValueDef = std::unordered_map; - using Ptr = std::shared_ptr; + using ValueDef = std::unordered_map; + using Ptr = std::shared_ptr; CspEnumMeta( const std::string & name, const ValueDef & def ); virtual ~CspEnumMeta(); const std::string & name() const { return m_name; } - size_t size() const { return m_mapping.size(); } + size_t size() const { return m_mapping.size(); } - //note this will throw on invalid values + // note this will throw on invalid values CspEnum fromString( const char * key ) const { auto it = m_mapping.find( key ); if( it == m_mapping.end() ) CSP_THROW( ValueError, "Unrecognized enum name " << key << " for enum " << m_name ); - return CspEnum( it -> second -> second.get() ); + return CspEnum( it->second->second.get() ); } CspEnum create( int64_t value ) const @@ -84,20 +97,20 @@ class CspEnumMeta auto found = m_instanceMap.find( value ); if( found == m_instanceMap.end() ) CSP_THROW( RuntimeException, "Unrecognized value " << value << " for enum " << m_name ); - return CspEnum( found -> second.get() ); + return CspEnum( found->second.get() ); } private: using InstanceMapping = std::unordered_map>; - using Mapping = std::unordered_map; + using Mapping = std::unordered_map; - std::string m_name; - Mapping m_mapping; + std::string m_name; + Mapping m_mapping; InstanceMapping m_instanceMap; }; -} +} // namespace csp namespace std { @@ -105,12 +118,9 @@ namespace std template<> struct hash { - size_t operator()( csp::CspEnum e ) const - { - return std::hash()( e.value() ); - } + size_t operator()( csp::CspEnum e ) const { return std::hash()( e.value() ); } }; -} +} // namespace std #endif diff --git a/cpp/csp/engine/CspType.cpp b/cpp/csp/engine/CspType.cpp index 884fc17cc..a16f03c3f 100644 --- a/cpp/csp/engine/CspType.cpp +++ b/cpp/csp/engine/CspType.cpp @@ -4,28 +4,9 @@ namespace csp { -INIT_CSP_ENUM( CspType::Type, - "UNKNOWN", - "BOOL", - "INT8", - "UINT8", - "INT16", - "UINT16", - "INT32", - "UINT32", - "INT64", - "UINT64", - "DOUBLE", - "DATETIME", - "TIMEDELTA", - "DATE", - "TIME", - "ENUM", - "STRING", - "STRUCT", - "ARRAY", - "DIALECT_GENERIC" - ); +INIT_CSP_ENUM( CspType::Type, "UNKNOWN", "BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", + "UINT64", "DOUBLE", "DATETIME", "TIMEDELTA", "DATE", "TIME", "ENUM", "STRING", "STRUCT", "ARRAY", + "DIALECT_GENERIC" ); CspTypePtr & CspArrayType::create( const CspTypePtr & elemType, bool isPyStructFastList ) { @@ -37,10 +18,10 @@ CspTypePtr & CspArrayType::create( const CspTypePtr & elemType, bool isPyStructF auto & cache = isPyStructFastList ? s_pyStructFastListCache : s_cache; std::lock_guard guard( s_mutex ); - auto rv = cache.emplace( elemType.get(), nullptr ); + auto rv = cache.emplace( elemType.get(), nullptr ); if( rv.second ) - rv.first -> second = std::make_shared( elemType, isPyStructFastList ); - return rv.first -> second; + rv.first->second = std::make_shared( elemType, isPyStructFastList ); + return rv.first->second; } -} +} // namespace csp diff --git a/cpp/csp/engine/CspType.h b/cpp/csp/engine/CspType.h index b85ab4148..a5be71e77 100644 --- a/cpp/csp/engine/CspType.h +++ b/cpp/csp/engine/CspType.h @@ -17,11 +17,9 @@ class CspStringType; class CspType { public: - struct TypeTraits { public: - enum _enum : uint8_t { UNKNOWN, @@ -41,35 +39,35 @@ class CspType TIME, ENUM, - //Native implied the data is memcpy-able + // Native implied the data is memcpy-able MAX_NATIVE_TYPE = ENUM, STRING, STRUCT, ARRAY, - //These types are currently all dialect specific, no native primitives + // These types are currently all dialect specific, no native primitives DIALECT_GENERIC, NUM_TYPES }; - template< typename T > + template struct fromCType; - template< uint8_t T > + template struct toCType; - //We store bool vectors as vector to account for oddities with vector across build environments - //toCArrayElemType means from the array's storage type to csp element type ( ie uint8_t -> bool ), otherwise its just Storage. - //toCArrayStorageType is the inverse - template< typename StorageT > + // We store bool vectors as vector to account for oddities with vector across build environments + // toCArrayElemType means from the array's storage type to csp element type ( ie uint8_t -> bool ), otherwise + // its just Storage. toCArrayStorageType is the inverse + template struct toCArrayElemType; - template< typename ElemT > + template struct toCArrayStorageType; - template< typename ElemT > + template struct toCArrayType; protected: @@ -78,29 +76,92 @@ class CspType using Type = Enum; - CspType( Type t ) : m_type( t ) {} + CspType( Type t ) + : m_type( t ) + { + } Type type() const { return m_type; } using Ptr = std::shared_ptr; - static Ptr & BOOL() { static auto s_type = std::make_shared( Type::BOOL ); return s_type; } - static Ptr & INT8() { static auto s_type = std::make_shared( Type::INT8 ); return s_type; } - static Ptr & UINT8() { static auto s_type = std::make_shared( Type::UINT8 ); return s_type; } - static Ptr & INT16() { static auto s_type = std::make_shared( Type::INT16 ); return s_type; } - static Ptr & UINT16() { static auto s_type = std::make_shared( Type::UINT16 ); return s_type; } - static Ptr & INT32() { static auto s_type = std::make_shared( Type::INT32 ); return s_type; } - static Ptr & UINT32() { static auto s_type = std::make_shared( Type::UINT32 ); return s_type; } - static Ptr & INT64() { static auto s_type = std::make_shared( Type::INT64 ); return s_type; } - static Ptr & UINT64() { static auto s_type = std::make_shared( Type::UINT64 ); return s_type; } - static Ptr & DOUBLE() { static auto s_type = std::make_shared( Type::DOUBLE ); return s_type; } - static Ptr & DATETIME() { static auto s_type = std::make_shared( Type::DATETIME ); return s_type; } - static Ptr & TIMEDELTA() { static auto s_type = std::make_shared( Type::TIMEDELTA ); return s_type; } - static Ptr & DATE() { static auto s_type = std::make_shared( Type::DATE ); return s_type; } - static Ptr & TIME() { static auto s_type = std::make_shared( Type::TIME ); return s_type; } + static Ptr & BOOL() + { + static auto s_type = std::make_shared( Type::BOOL ); + return s_type; + } + static Ptr & INT8() + { + static auto s_type = std::make_shared( Type::INT8 ); + return s_type; + } + static Ptr & UINT8() + { + static auto s_type = std::make_shared( Type::UINT8 ); + return s_type; + } + static Ptr & INT16() + { + static auto s_type = std::make_shared( Type::INT16 ); + return s_type; + } + static Ptr & UINT16() + { + static auto s_type = std::make_shared( Type::UINT16 ); + return s_type; + } + static Ptr & INT32() + { + static auto s_type = std::make_shared( Type::INT32 ); + return s_type; + } + static Ptr & UINT32() + { + static auto s_type = std::make_shared( Type::UINT32 ); + return s_type; + } + static Ptr & INT64() + { + static auto s_type = std::make_shared( Type::INT64 ); + return s_type; + } + static Ptr & UINT64() + { + static auto s_type = std::make_shared( Type::UINT64 ); + return s_type; + } + static Ptr & DOUBLE() + { + static auto s_type = std::make_shared( Type::DOUBLE ); + return s_type; + } + static Ptr & DATETIME() + { + static auto s_type = std::make_shared( Type::DATETIME ); + return s_type; + } + static Ptr & TIMEDELTA() + { + static auto s_type = std::make_shared( Type::TIMEDELTA ); + return s_type; + } + static Ptr & DATE() + { + static auto s_type = std::make_shared( Type::DATE ); + return s_type; + } + static Ptr & TIME() + { + static auto s_type = std::make_shared( Type::TIME ); + return s_type; + } static Ptr & STRING(); static Ptr & BYTES(); - static Ptr & DIALECT_GENERIC() { static auto s_type = std::make_shared( Type::DIALECT_GENERIC ); return s_type; } + static Ptr & DIALECT_GENERIC() + { + static auto s_type = std::make_shared( Type::DIALECT_GENERIC ); + return s_type; + } static constexpr bool isNative( TypeTraits::_enum t ) { return t <= TypeTraits::MAX_NATIVE_TYPE; } @@ -118,15 +179,27 @@ class CspType class CspStringType : public CspType { public: - CspStringType(bool isBytes) - : CspType(CspType::Type::STRING), m_isBytes(isBytes) {} - inline bool isBytes() const {return m_isBytes;} + CspStringType( bool isBytes ) + : CspType( CspType::Type::STRING ) + , m_isBytes( isBytes ) + { + } + inline bool isBytes() const { return m_isBytes; } + private: const bool m_isBytes; }; -inline CspType::Ptr & CspType::STRING() { static CspType::Ptr s_type = std::make_shared( false ); return s_type; } -inline CspType::Ptr & CspType::BYTES() { static CspType::Ptr s_type = std::make_shared( true ); return s_type; } +inline CspType::Ptr & CspType::STRING() +{ + static CspType::Ptr s_type = std::make_shared( false ); + return s_type; +} +inline CspType::Ptr & CspType::BYTES() +{ + static CspType::Ptr s_type = std::make_shared( true ); + return s_type; +} using CspTypePtr = CspType::Ptr; @@ -137,9 +210,11 @@ class CspEnumMeta; class CspEnumType : public CspType { public: - CspEnumType( std::shared_ptr & meta ) : CspType( CspType::Type::ENUM ), - m_meta( meta ) - {} + CspEnumType( std::shared_ptr & meta ) + : CspType( CspType::Type::ENUM ) + , m_meta( meta ) + { + } const std::shared_ptr & meta() const { return m_meta; } @@ -157,10 +232,12 @@ class StructMeta; class CspStructType : public CspType { public: - CspStructType( const std::shared_ptr & meta ) : CspType( CspType::Type::STRUCT ), - m_meta( meta ) - {} - + CspStructType( const std::shared_ptr & meta ) + : CspType( CspType::Type::STRUCT ) + , m_meta( meta ) + { + } + const std::shared_ptr & meta() const { return m_meta; } private: @@ -170,16 +247,19 @@ class CspStructType : public CspType class CspArrayType : public CspType { public: - CspArrayType( CspTypePtr elemType, bool isPyStructFastList = false ) : - CspType( CspType::Type::ARRAY ), m_elemType( elemType ), m_isPyStructFastList( isPyStructFastList ) - {} + CspArrayType( CspTypePtr elemType, bool isPyStructFastList = false ) + : CspType( CspType::Type::ARRAY ) + , m_elemType( elemType ) + , m_isPyStructFastList( isPyStructFastList ) + { + } ~CspArrayType() {} const CspTypePtr & elemType() const { return m_elemType; } - bool isPyStructFastList() const { return m_isPyStructFastList; } + bool isPyStructFastList() const { return m_isPyStructFastList; } - //Used by BURST mode to avoid creating more instances of CspArrayTypes than needed - //returns CspArrayType with the given elemType + // Used by BURST mode to avoid creating more instances of CspArrayTypes than needed + // returns CspArrayType with the given elemType static CspTypePtr & create( const CspTypePtr & elemType, bool isPyStructFastList = false ); private: @@ -187,78 +267,308 @@ class CspArrayType : public CspType bool m_isPyStructFastList; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::BOOL; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::INT8; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::UINT8; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::INT16; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::UINT16; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::INT32; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::UINT32; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::INT64; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::UINT64; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::DOUBLE; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::DATETIME; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::TIMEDELTA; }; -template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::DATE; }; -template<> struct CspType::TypeTraits::fromCType