From 824d70a86319b503980ed48523daa37f87993466 Mon Sep 17 00:00:00 2001 From: chirag Date: Sun, 30 Oct 2016 01:41:33 -0400 Subject: [PATCH 1/3] Eigen Implementatin (PCL+Eigen) --- include/compute_flow/compute_flow.h | 43 +++-- include/dvs_flow/event_loader.h | 12 +- src/compute_flow.cpp | 256 +++++++++++++++------------- src/event_loader.cpp | 51 +++--- 4 files changed, 197 insertions(+), 165 deletions(-) diff --git a/include/compute_flow/compute_flow.h b/include/compute_flow/compute_flow.h index da6d131..1242e90 100644 --- a/include/compute_flow/compute_flow.h +++ b/include/compute_flow/compute_flow.h @@ -5,27 +5,40 @@ #include #include #include "ros/ros.h" - -#include #include #include -// Headers for extractNormals -#include - -// Headers for fit plane -#include -#include -#include +#include namespace compute_flow { - typedef pcl::PointXYZI PointT; - void computeFlow(pcl::PointCloud &cloud); - void pcaModified(); - void robustSVD(); - void fitPlane(pcl::PointCloud &cloud); - void extractNormals(pcl::PointCloud cloud); + const int NROWS = 340, NCOLS = 340; //Max and Min Value of X and Y (Event Positions) + const int N = 5; // TimeStamp Patch Mask Dimension around the Events + const double TIME_THRES = 0.2; + const double TH2 = 0.1; + typedef pcl::PointXYZI PointT; + typedef Eigen::MatrixXd Mat; + + //Eigen::MatrixXd pclToMatrix(pcl::PointCloud::Ptr cloud); + void computeFlow(pcl::PointCloud::Ptr cloud); + void fitPlane(Mat &m, const double TH2); + void pca(Mat &X); + + void find(Mat &m, Mat &X); + Mat getPatch(Mat &m,int x, int y, int N); + Mat getBlock(Mat &m,int x,int y,int N); + + + struct CWiseThres { + CWiseThres(const double val,const double thres) : val_(val),thres_(thres){} + const double operator()(const double& x) const { + if(abs(x - val_)/val_ > thres_) + return 0.0; + else + return x; + }; + double val_, thres_; + }; } #endif //_compute_flow_H_ \ No newline at end of file diff --git a/include/dvs_flow/event_loader.h b/include/dvs_flow/event_loader.h index a91d04d..ebdd794 100644 --- a/include/dvs_flow/event_loader.h +++ b/include/dvs_flow/event_loader.h @@ -30,8 +30,8 @@ class EventLoader { virtual ~EventLoader(); private: - pcl::PointCloud events_pcl_; - //pcl::PointCloud::Ptr events_pcl_ptr_; + //pcl::PointCloud events_pcl_; + pcl::PointCloud::Ptr events_pcl_ptr_; dvs_events events_; std::vector events_array_; @@ -41,12 +41,16 @@ class EventLoader { ros::NodeHandle nh_; float t_first_ = -1; ros::Subscriber events_sub_; - ros::Publisher events_pub_; + + + image_transport::Subscriber image_sub_; void eventListenerCallback(const dvs_msgs::EventArray::ConstPtr& msg); void imageListenerCallback(const sensor_msgs::Image::ConstPtr& msg); - void displayEvents(std::vector &e_array); + + ros::Publisher events_pub_; + void displayEvents(); }; } // namespace diff --git a/src/compute_flow.cpp b/src/compute_flow.cpp index 57fdbc5..9a2fbae 100644 --- a/src/compute_flow.cpp +++ b/src/compute_flow.cpp @@ -3,11 +3,66 @@ namespace compute_flow { //[vx, vy, It] = computeFlow(x, y, t, pol, N, TH1, TH2, NCOLS, NROWS) - void computeFlow(pcl::PointCloud &cloud) + void computeFlow(pcl::PointCloud::Ptr events_ptr) { + + ROS_INFO_STREAM("compute_flow()"); + + // Mat m = Mat::Random(5,6); + // getPatch(m,3,3,2); + + /** CONVERT THIS FUNCTION + **/ + + Mat img_ts_pos = Mat::Zero(NROWS, NROWS); //Matrix of timestamps of last events with positive polarity + Mat img_ts_neg = Mat::Zero(NROWS, NCOLS); //Matrix of timestamps of last events with negative polarity + Mat vx= Mat::Zero(NROWS,NCOLS); + Mat vy= Mat::Zero(NROWS,NCOLS); + Mat m; + + + std::cout <<"mask timestampe pos" <size(); ++i) + { + int x = events_ptr->points[i].x; + int y = events_ptr->points[i].y; + double ts = events_ptr->points[i].z; + + if(events_ptr->points[i].intensity == 255) + { + std::cout<<"Patch from Positive Polarity"<points[i].intensity == 0) + { + std::cout<<"Patch from Negative Polarity"< &cloud_) + void find(Mat &m, Mat &X) { - const pcl::PointCloud::Ptr cloud(&cloud_); - pcl::IndicesPtr indices(new std::vector); - float distanceThreshold = 0.01; - int maxIterations = 1000; - // Extract plane - pcl::ModelCoefficients::Ptr coefficients (new pcl::ModelCoefficients); - pcl::PointIndices::Ptr inliers (new pcl::PointIndices); - // Create the segmentation object - pcl::SACSegmentation seg; - // Optional - seg.setOptimizeCoefficients (true); - seg.setMaxIterations (maxIterations); - // Mandatory - seg.setModelType (pcl::SACMODEL_PLANE); - seg.setMethodType (pcl::SAC_RANSAC); - seg.setDistanceThreshold (distanceThreshold); - - seg.setInputCloud (cloud); - if(indices->size()) + int cnt = 0; + for(int i = 0; i < m.rows(); ++i) { - seg.setIndices(indices); + for(int j = 0; j < m.cols(); ++j) + { + if(m(i,j)!=0) + { + Eigen::Vector3d vec(j,i,m(i,j)); //Check for X and Y Values + //std::cout<<" Vector = "<m) + double c_val = m(N,N); + // std::cout<<"central val = "< 0) { - X=X(:,randperm(n,m)); - n=m; + Mat X(0,3); + find(m,X); + ROS_INFO_STREAM("fit plane input \n"<n ) - { - [~,SS,V]= robustSvd(X'*X); - **/ - robustSVD(); - /** - vars=diag(SS); - U = X * V * diag(1./sqrt(vars)); - } - else - { - [~,SS,U]=robustSvd(X*X'); - **/ - robustSVD(); - /** - vars=diag(SS); - } - K=vars>1e-30; - vars=vars(K); - U=U(:,K); - */ } - - //function [U,S,V] = robustSvd( X, trials ) - void robustSVD() + //function [U,mu,vars] = pca( X ) + void pca(Mat &X) { - ROS_INFO_STREAM("robust_svd()"); - /** - if(nargin<2) + if(X.rows() != 1) { - - trials=100; + X = (X.rowwise() - X.colwise().mean())/std::sqrt(X.rows()-1); + std::cout<<"X_centered"< svd(X, Eigen::ComputeFullU | Eigen::ComputeFullV); + std::cout << "U: " << svd.matrixU().size() << std::endl; + std::cout << "V: " << svd.matrixV() << std::endl; + std::cout<<"Press any key to continue"; + std::cin.ignore(); + ROS_INFO_STREAM("pca_modified()"); } + } - try - { - [U,S,V] = svd(X) - } - catch - { - if(trials<=0) - { - error('svd did not converge'); - } - n=numel(X); - j=randi(n); - X(j)=X(j)+eps; - [U,S,V]=robustSvd(X,trials-1); - } - **/ + Mat getPatch(Mat &m,int x, int y, int N) + { + //std::cout<= m.rows() || e_y >= m.cols()) + { + return patch; //return patch of zero size + } + + patch = m.block(x-N,y-N,L,L); + + ROS_INFO_STREAM("\npatch = \n"<> ("events", 10); - events_pcl_.header.frame_id = "odom"; - events_pcl_.width = 0; - events_pcl_.height = 1; + pcl::PointCloud::Ptr temp_ptr (new pcl::PointCloud); + events_pcl_ptr_ = temp_ptr; + events_pcl_ptr_->header.frame_id = "odom"; + events_pcl_ptr_->width = 0; + events_pcl_ptr_->height = 1; image_transport::ImageTransport it_(nh_); @@ -34,12 +36,12 @@ namespace event_loader{ { ROS_INFO_STREAM("Fresh Events Received: "<events.size()); //ROS_INFO_STREAM("events_array_ size"<size()events.begin(),msg->events.end()); - // events_pcl_.width += msg->events.size(); - // events_pcl_.data.insert(events_array_[0].end(),msg->events.begin(),msg->events.end()); - for(int i = 0; i < msg->events.size();++i) + // events_pcl_ptr_->width += msg->events.size(); + // events_pcl_ptr_->data.insert(events_array_[0].end(),msg->events.begin(),msg->events.end()); + for(auto i = 0; i < msg->events.size();++i) { PointT f_event; f_event.x = msg->events[i].x/10; @@ -47,24 +49,21 @@ namespace event_loader{ f_event.z = (msg->events[i].ts.toSec() - msg->events[0].ts.toSec())*100; //ROS_INFO_STREAM("Z = "<events[i].polarity?0:1) * 255; - events_pcl_.push_back(f_event); + events_pcl_ptr_->push_back(f_event); } - ROS_INFO_STREAM("Events Recorded in PCL"<size()); } - if(events_pcl_.size()>=BATCH_SIZE) + if(events_pcl_ptr_->size()>=BATCH_SIZE) { - ROS_INFO_STREAM(" --- Events Recorded in PCL"<size()); + displayEvents(); + compute_flow::computeFlow(events_pcl_ptr_); + + events_pcl_ptr_->clear(); + events_pcl_ptr_->height = 1; + ROS_INFO_STREAM(" -- Events Erased"<< events_pcl_ptr_->size()); } - - //displayEvents(events_array_); } void EventLoader::imageListenerCallback(const sensor_msgs::Image::ConstPtr& msg) @@ -96,14 +95,14 @@ namespace event_loader{ used_last_image_ = false;*/ } - void EventLoader::displayEvents(std::vector &e) + void EventLoader::displayEvents() { - for ( int i = 0; i < e.size(); ++i ) + if(events_pub_.getNumSubscribers()>0) { - for (int j = 0; j < e[i].size() ; ++j) - { - ROS_INFO_STREAM(" x ["<header.stamp = ros::Time::now().toNSec(); + events_pub_.publish(*events_pcl_ptr_); + ROS_INFO_STREAM("PUBLISHED POINT_CLOUD "); } } + } \ No newline at end of file From a3af0a80394378f920d29c0b6c51a6a8f7f69859 Mon Sep 17 00:00:00 2001 From: chirag Date: Sun, 30 Oct 2016 04:09:32 -0400 Subject: [PATCH 2/3] compute flow done -- bug fix required --- include/compute_flow/compute_flow.h | 4 ++-- src/compute_flow.cpp | 36 ++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/include/compute_flow/compute_flow.h b/include/compute_flow/compute_flow.h index 1242e90..58d4b8f 100644 --- a/include/compute_flow/compute_flow.h +++ b/include/compute_flow/compute_flow.h @@ -21,8 +21,8 @@ namespace compute_flow //Eigen::MatrixXd pclToMatrix(pcl::PointCloud::Ptr cloud); void computeFlow(pcl::PointCloud::Ptr cloud); - void fitPlane(Mat &m, const double TH2); - void pca(Mat &X); + bool fitPlane(Mat &m,Eigen::Vector3d &normal, const double TH2); + void pca(Mat &X,Mat &U); void find(Mat &m, Mat &X); Mat getPatch(Mat &m,int x, int y, int N); diff --git a/src/compute_flow.cpp b/src/compute_flow.cpp index 9a2fbae..90638d9 100644 --- a/src/compute_flow.cpp +++ b/src/compute_flow.cpp @@ -52,8 +52,16 @@ namespace compute_flow if(m.size()!=0) { //Select Events that are close in the time_stamp - fitPlane(m,TH2); - + Eigen::Vector3d normal; + if(fitPlane(m,normal,TH2)) + { + std::cout<<"Normal computed "<= 3) + { + pca(X,U); + normal= U.col(2); + std::cout<<"computed Normal"<< normal<= 1) //Checking if there is sufficient data to get coeff > 3 { X = (X.rowwise() - X.colwise().mean())/std::sqrt(X.rows()-1); std::cout<<"X_centered"< svd(X, Eigen::ComputeFullU | Eigen::ComputeFullV); std::cout << "U: " << svd.matrixU().size() << std::endl; + Mat S = svd.singularValues().asDiagonal(); + std::cout<<"S: "<< S < Date: Sun, 30 Oct 2016 04:16:58 -0400 Subject: [PATCH 3/3] compute flow -- bug fix required --- include/compute_flow/compute_flow.h | 2 +- src/compute_flow.cpp | 53 +---------------------------- 2 files changed, 2 insertions(+), 53 deletions(-) diff --git a/include/compute_flow/compute_flow.h b/include/compute_flow/compute_flow.h index 58d4b8f..2965da8 100644 --- a/include/compute_flow/compute_flow.h +++ b/include/compute_flow/compute_flow.h @@ -14,7 +14,7 @@ namespace compute_flow { const int NROWS = 340, NCOLS = 340; //Max and Min Value of X and Y (Event Positions) const int N = 5; // TimeStamp Patch Mask Dimension around the Events - const double TIME_THRES = 0.2; + const double TIME_THRES = 0.99; const double TH2 = 0.1; typedef pcl::PointXYZI PointT; typedef Eigen::MatrixXd Mat; diff --git a/src/compute_flow.cpp b/src/compute_flow.cpp index 90638d9..b42dfe6 100644 --- a/src/compute_flow.cpp +++ b/src/compute_flow.cpp @@ -70,58 +70,6 @@ namespace compute_flow //Mat m = Mat::Zero(2*N+1,2*N+1); //Time Stamp patch } - /** - It_pos=zeros(NROWS,NCOLS); - It_neg=zeros(NROWS,NCOLS); - vx=zeros(NROWS,NCOLS); vy=zeros(NROWS,NCOLS); - for ii=1:1:length(t) - { - ptx=x(ii)+1; - pty=y(ii)+1; - - if pol(ii)==1 - { - It_pos(pty,ptx)=t(ii); - m=It_pos(max(pty-N,1):min(pty+N, NROWS),max(ptx-N,1):min(ptx+N, NCOLS)); - } - else - { - It_neg(pty,ptx)=t(ii); - m=It_neg(max(pty-N,1):min(pty+N, NROWS),max(ptx-N,1):min(ptx+N, NCOLS)); - } - - if numel(m) == (2*N+1)*(2*N+1) - { - m(abs(m(N+1,N+1)-m)/m(N+1,N+1)>TH1)=0; - if (sum(m(:)>0)) - { - - [vvx,vvy]=fitplane(m, TH2); - **/ - - //estimateNormals(cloud); - /** - if(isnan(vvx) || isinf(vvx)) - { - vvx = 0; - } - - if(isnan(vvy) || isinf(vvy)) - { - vvy = 0; - } - - if (norm([vvx,vvy])>0) - { - aa=[vvx vvy]; - vy(pty,ptx)=aa(1); - vx(pty,ptx)=aa(2); - } - } - } - } - It = max(cat(3, It_pos, It_neg), [], 3); - **/ } void find(Mat &m, Mat &X) @@ -184,6 +132,7 @@ namespace compute_flow std::cout << "V: " << svd.matrixV() << std::endl; //U = X * svd.matrixV() * S.inverse(); //check if diagnol(1/S) works U = svd.matrixV(); + std::cout<<"Checking orthonormality"<