Fix issue with BG substraction reference image when the size is different from the original

README.md

@@ -72,4 +72,4 @@ On the main simulation window:
 - `space` Start/stop the sequencer
 - `m` Mutates the current preset
 - `Ctrl+s` Save the preset on the current slot
-- `Ctrl+c` Clear the current preset slot
+- `Ctrl+c` Clear the current preset slot

bin/data/support/gui-styles.json

@@ -2,6 +2,9 @@
   "default": {
     "base": {
       "border-radius": 4,
+      "border-color": "#444444",
+      "fill-color": "#aaaaaa",
+      "border-width": 1,
       "text-color": "#ffffef"
     },
 

esencia.sln

@@ -1,9 +1,17 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
+# Visual Studio Version 17
+VisualStudioVersion = 17.12.35527.113 d17.12
+MinimumVisualStudioVersion = 10.0.40219.1
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "esencia", "esencia.vcxproj", "{7FD42DF7-442E-479A-BA76-D0022F99702A}"
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "openframeworksLib", "..\..\..\libs\openFrameworksCompiled\project\vs\openframeworksLib.vcxproj", "{5837595D-ACA9-485C-8E76-729040CE4B0B}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{E2104F86-B40A-465C-8AE0-704E30EB6E5C}"
+	ProjectSection(SolutionItems) = preProject
+		LICENSE = LICENSE
+		README.md = README.md
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64

esencia.vcxproj

@@ -163,7 +163,7 @@
     <ClInclude Include="src\gui\panels\SequencePanel.h" />
     <ClInclude Include="src\gui\panels\SimulationPanel.h" />
     <ClInclude Include="src\gui\panels\SystemstatsPanel.h" />
-    <ClInclude Include="src\gui\panels\VideoOriginPanel.h" />
+    <ClInclude Include="src\gui\panels\VideoSourcePanel.h" />
     <ClInclude Include="src\gui\panels\VideoProcessingPanel.h" />
     <ClInclude Include="src\gui\parameters\EsenciaParameters.h" />
     <ClInclude Include="src\render\RenderApp.h" />

esencia.vcxproj.filters

@@ -530,7 +530,7 @@
     <ClInclude Include="src\gui\panels\SystemstatsPanel.h">
       <Filter>src\gui\panels</Filter>
     </ClInclude>
-    <ClInclude Include="src\gui\panels\VideoOriginPanel.h">
+    <ClInclude Include="src\gui\panels\VideoSourcePanel.h">
       <Filter>src\gui\panels</Filter>
     </ClInclude>
     <ClInclude Include="src\gui\panels\VideoProcessingPanel.h">

src/camera/Camera.cpp

@@ -36,8 +36,6 @@ void Camera::setup(CameraParameters* params) {
     else {
         changeSource(VideoSources::VIDEOSOURCE_VIDEOFILE);
     }
-
-    restoreBackgroundReference(backgroundReference);
 }
 
 /// <summary>
@@ -58,6 +56,8 @@ void Camera::onGUIChangeSource(bool& _) {
     } else {
         stopCurrentSource();
     }
+
+    restoreBackgroundReference(backgroundReference);
 }
 
 
@@ -67,6 +67,8 @@ void Camera::onGUIChangeSource(bool& _) {
 /// </summary>
 /// <param name="newSource">From the list of VideoSources</param>
 void Camera::changeSource(VideoSources newSource) {
+    ofLogNotice(__FUNCTION__) << "Attempting to change source to " << (int)newSource;
+
     stopCurrentSource();
 
     currentVideosource = VideoSources::VIDEOSOURCE_NONE; // in the new setup fail, stay at None
@@ -80,6 +82,9 @@ void Camera::changeSource(VideoSources newSource) {
     else if (newSource == VideoSources::VIDEOSOURCE_WEBCAM) {
         setupWebcam();
     }
+    else {
+        return;
+    }
 
     // to-do: need to call setFrameSize with the updated frame size from the new source
     setFrameSize(cameraResolutions[selectedOrbbecResolution].x, cameraResolutions[selectedOrbbecResolution].y);
@@ -107,9 +112,15 @@ void Camera::changeSource(VideoSources newSource) {
 /// <param name="width"></param>
 /// <param name="height"></param>
 void Camera::setFrameSize(int width, int height) {
+    ofLogNotice(__FUNCTION__) << "Resizing every image and fb to " << width << ", " << height;
+
+    if (currentVideosource == VideoSources::VIDEOSOURCE_ORBBEC) {
     if (orbbecSettings.rotation == OB_ROTATE_DEGREE_90 || orbbecSettings.rotation == OB_ROTATE_DEGREE_270) {
+            ofLogNotice(__FUNCTION__) << "Rotating orbbec camera also rotates dimentions (rotation " << orbbecSettings.rotation << " degrees)";
         std::swap(width, height);
     }
+    }
+
     IMG_WIDTH = width;
     IMG_HEIGHT = height;
 
@@ -154,18 +165,24 @@ void Camera::onGUIStartBackgroundReference(bool &value) {
 /// Loads a video file to use it as the video source
 /// </summary>
 void Camera::loadVideoFile() {
-    ofLogNotice("Camera::loadVideoFile()") << "Switching source to video file";
+    ofLogNotice(__FUNCTION__) << "Switching source to video file";
 
     // to-do: let the user select the file (ofFileDialogResult from ofSystemLoadDialog)
-    ofLogNotice("Camera::loadVideoFile()") << "Attempting to load a video file";
+    ofLogNotice(__FUNCTION__) << "Attempting to load a video file";
     prerecordedVideo.load("video_mocks/movement_nfov_h264.mp4");
     // to-do: throw error when file does not exist, or cant be loaded
 
     prerecordedVideo.setLoopState(OF_LOOP_NORMAL);
     prerecordedVideo.play();
 
     currentVideosource = VideoSources::VIDEOSOURCE_VIDEOFILE;
-    ofLogNotice("Camera::loadVideoFile()") << "Camera::loadVideoFile Video file loaded";
+    ofLogNotice(__FUNCTION__) << "Video file loaded";
+
+    // copy the the associated precaptured background into the right folder
+    ofFile::removeFile(BG_REFERENCE_FILENAME);
+    ofFile::copyFromTo("video_mocks\\" + BG_REFERENCE_FILENAME, BG_REFERENCE_FILENAME);
+    restoreBackgroundReference(backgroundReference);
+    ofLogNotice(__FUNCTION__) << "Load precaptured background reference from video file";
 }
 
 /// <summary>
@@ -188,6 +205,7 @@ void Camera::setupOrbbecCamera() {
     orbbecSettings.depthFrameSize.requestWidth = orbbecRequestedWidth;
     orbbecSettings.rotation = OB_ROTATE_DEGREE_90;
     orbbecCam.open(orbbecSettings);
+
     currentVideosource = VideoSources::VIDEOSOURCE_ORBBEC;
 }
 
@@ -215,10 +233,11 @@ void Camera::stopCurrentSource() {
         ofLogNotice("Camera::stopCurrentSource()") << "Closing the orbbec camera";
         orbbecCam.close();
     }
-    else if (currentVideosource == VideoSources::VIDEOSOURCE_VIDEOFILE) {
+    if (currentVideosource == VideoSources::VIDEOSOURCE_VIDEOFILE) {
         ofLogNotice("Camera::stopCurrentSource()") << "Stopping the video file playback";
         prerecordedVideo.stop();
     }
+    currentVideosource = VideoSources::VIDEOSOURCE_NONE;
     // webcam.stop
 }
 
@@ -280,23 +299,28 @@ void Camera::draw() {
 void Camera::update() {
 
     // acquire frame
-    if (currentVideosource == VideoSources::VIDEOSOURCE_VIDEOFILE) {
+    if (currentVideosource == VideoSources::VIDEOSOURCE_ORBBEC) {
+        orbbecCam.update();
+
+        if (orbbecCam.isFrameNewDepth()) {
+            source.setFromPixels(orbbecCam.getDepthPixels());
+        }
+    }
+    else if (currentVideosource == VideoSources::VIDEOSOURCE_VIDEOFILE) {
         prerecordedVideo.update();
 
         if (prerecordedVideo.isFrameNew()) {
             colorFrame.setFromPixels(prerecordedVideo.getPixels());
             source = colorFrame;
         }
     }
-
-    if (currentVideosource == VideoSources::VIDEOSOURCE_ORBBEC) {
-        orbbecCam.update();
-
-        if (orbbecCam.isFrameNewDepth()) {
-            source.setFromPixels(orbbecCam.getDepthPixels());
-        }
+    else {
+        // no source, no update
+        return;
     }
 
+    parameters->previewSource.setFromPixels(source.getPixels());
+
     // to-do: make backgroundref an object with state?
     if (isTakingBackgroundReference) {
         addSampleToBackgroundReference(source, backgroundReference, BG_SAMPLE_FRAMES);
@@ -306,10 +330,10 @@ void Camera::update() {
     // all the processing from source to extract the final segment
     processCameraFrame(source, backgroundReference);
 
-
     // update the preview images on the shared parameters data structure for the GUI
-    parameters->previewSource.setFromPixels(source.getPixels());
     convertToTransparent(segment, parameters->previewSegment); // to-do: should be called only when accessed 
+
+    saveDebugImage(segment, "segment", "final");
     // parameters->previewBackground.setFromPixels(backgroundReference.getPixels()); // this does not need to run every update, so its placed when updating backgroundReference data
 }
 
@@ -364,9 +388,11 @@ void Camera::processCameraFrame(ofxCvGrayscaleImage &frame, ofxCvGrayscaleImage
     // ---------
     // remove the background from the frame (save the output in maskImage object)
     if (backgroundReferenceTaken) {
+        if (backgroundNewFrame.width != processedImage.width) return;
+
         cvAbsDiff(processedImage.getCvImage(), backgroundNewFrame.getCvImage(), segment.getCvImage()); // this works great for a single background frame of reference!
 
-        saveDebugImage(processedImage, "processedImage", "removed background absdiff");
+        saveDebugImage(segment, "segment", "removed background absdiff");
     }
     else {
         segment = processedImage;
@@ -422,6 +448,8 @@ void Camera::processCameraFrame(ofxCvGrayscaleImage &frame, ofxCvGrayscaleImage
         getContourPolygonsFromImage(processedImage, &polygons);
     }
 
+    saveDebugImage(processedImage, "processedImage", "pre-gpuBlur");
+
     // add gaussian blur to the silouetes
     // (this step was originaly performed by the simulation on the sorounds of each particle, its here now to test if the performance is better)
     gpuBlur(processedImage, parameters->gaussianBlur);
@@ -582,18 +610,25 @@ void Camera::startBackgroundReferenceSampling() {
 }
 
 void Camera::startBackgroundReferenceSampling(int samples) {
-    ofLogNotice("Camera::startBackgroundReferenceSampling()") << "Starting background";
+    ofLogNotice(__FUNCTION__) << "Starting background sampling for futher substraction";
     isTakingBackgroundReference = true;
     backgroundReferenceTaken = false;
     clearBackgroundReference();
     backgroundReference = source; // Important step to take an initial sample, so any strategy of adding/weighting/... dont start from a black image
     backgroundReferenceLeftFrames = samples;
 }
 
+/// <summary>
+/// Clear the background reference image
+/// will remove the previous bg reference file
+/// wil set the Taken flag to false
+/// </summary>
 void Camera::clearBackgroundReference() {
-    ofLogNotice("Camera::clearBackgroundReference()") << "Clearing background";
+    ofLogNotice("Camera::clearBackgroundReference()") << "Cleaning background";
     backgroundReference.set(0);
     parameters->previewBackground.clear();
+	ofFile::removeFile(BG_REFERENCE_FILENAME);
+	backgroundReferenceTaken = false;
 }
 
 
@@ -612,20 +647,27 @@ void Camera::saveBackgroundReference(ofxCvGrayscaleImage image) {
 
 
 /// <summary>
-/// Reloads a previous reference from the disk
+/// Reloads a previous background reference from the disk
 /// </summary>
 /// <param name="imageObject"></param>
 /// <returns>True if loads the reference successfully</returns>
 bool Camera::restoreBackgroundReference(ofxCvGrayscaleImage & outputImage) {
-    ofLogNotice("Camera::restoreBackgroundReference") << "Attempting to load a background reference at data/" + BG_REFERENCE_FILENAME;
+    ofLogNotice(__FUNCTION__) << "Attempting to load a background reference at data/" + BG_REFERENCE_FILENAME;
+
     ofPixels pixels;
     bool loaded = ofLoadImage(pixels, BG_REFERENCE_FILENAME);
+
     if (loaded) {
-        // to-do: validate bg ref and frame have the same size resolution
+		// validate if the bg ref and the camera frame have the same size to continue, otherwise delete the bg ref for sanity
+		if (pixels.getWidth() != IMG_WIDTH || pixels.getHeight() != IMG_HEIGHT) {
+			ofLogError("Camera::restoreBackgroundReference") << "The background reference image has a different resolution from the current frame and will be deleted";
+			ofFile::removeFile(BG_REFERENCE_FILENAME);
+			return false;
+		}
         outputImage.allocate(pixels.getWidth(), pixels.getHeight());
         outputImage.setFromPixels(pixels);
         backgroundReferenceTaken = true;
-        parameters->previewBackground.setFromPixels(backgroundReference.getPixels()); // updates the gui preview // not perfect code, since this function should be agnostic and saving value to a pointer, but this param access is used directly
+        parameters->previewBackground.setFromPixels(backgroundReference.getPixels()); // updates the gui preview // not perfect code, since this function should be agnostic and saving value to a pointer, but using a param is convenient and params are already glueing the modules
         ofLogNotice("Camera::restoreBackgroundReference") << "Load successfull";
     }
     else {
@@ -671,12 +713,17 @@ void convertToTransparent(ofxCvGrayscaleImage &grayImage, ofImage &rgbaImage) {
 /// <param name="name">object name id (background, camera, cleared..)</param>
 /// <param name="step">additional id, i.e. sequence step</param>
 void Camera::saveDebugImage(ofxCvGrayscaleImage img, string name, string step) {
-#ifdef DEBUG_IMAGES
     if (parameters->saveDebugImages) {
-        const string& filename = ofGetTimestampString() + "_" + name + "_" + step + ".png";
+        const string& filename = "raw_recording\\" + ofGetTimestampString() + "_" + name + "_" + step + ".png";
         ofSaveImage(img.getPixels(), filename);
     }
-#endif
+}
+
+void Camera::saveDebugImage(ofPixels img, string name, string step) {
+    if (parameters->saveDebugImages) {
+        const string& filename = "raw_recording\\" + ofGetTimestampString() + "_" + name + "_" + step + ".png";
+        ofSaveImage(img, filename);
+    }
 }
 
 void Camera::recordTestingFrames(ofxCvGrayscaleImage img) {

src/camera/Camera.h

@@ -9,10 +9,10 @@
 class Camera
 {
     enum class VideoSources : int {
-        VIDEOSOURCE_ORBBEC,
-        VIDEOSOURCE_VIDEOFILE,
-        VIDEOSOURCE_WEBCAM,
-        VIDEOSOURCE_NONE
+        VIDEOSOURCE_ORBBEC = 1,
+        VIDEOSOURCE_VIDEOFILE = 2,
+        VIDEOSOURCE_WEBCAM = 3,
+        VIDEOSOURCE_NONE = 0
     };
 
     public:
@@ -34,6 +34,7 @@ class Camera
         void getContourPolygonsFromImage(ofxCvGrayscaleImage image, vector<ofPolyline>* output);
 
         void saveDebugImage(ofxCvGrayscaleImage img, string name, string step);
+        void saveDebugImage(ofPixels img, string name, string step);
         void recordTestingFrames(ofxCvGrayscaleImage frame);
         uint64 recordTestingFramesCounter = 0;
         void saveMeshFrame();

src/gui/EsenciaPanelBase.h

@@ -9,6 +9,10 @@ const int PANELS_CIRCLE_RESOLUTION = 8;
 const int PANELS_BEZIER_PADDING = 40;
 const int PANELS_BEZIER_RESOLUTION = 10;
 const int PANELS_TRIANGLE_SIZE = 4;
+const ofColor LINE_COLOR1 = ofColor::darkSlateGrey; //paleGoldenRod;
+const ofColor LINE_COLOR2 = ofColor::darkSlateGrey; //paleTurquoise;
+const ofColor LINE_COLOR3 = ofColor::darkSlateGrey; // khaki;
+const ofColor FLOW_COLOR  = ofColor::darkSlateGrey; //paleGoldenRod;
 
 class EsenciaPanelBase {
 
@@ -58,16 +62,16 @@ class EsenciaPanelBase {
         ofPushMatrix();
 
         // origin glyph
-        ofSetColor(ofColor::paleGoldenRod, 200);
+        ofSetColor(LINE_COLOR2, 200);
 		ofSetCircleResolution(PANELS_CIRCLE_RESOLUTION);
         ofFill();
         ofDrawCircle(originCircleX, originCircleY, PANELS_CIRCLE_RADIUS);
 
         // destination glyph
-        ofSetColor(ofColor::paleTurquoise, 200);
+        ofSetColor(LINE_COLOR1, 200);
 		ofDrawArrow(destinationArrowStart, destinationArrowEnd, PANELS_TRIANGLE_SIZE);
 
-        ofSetColor(ofColor::khaki, 200);
+        ofSetColor(LINE_COLOR3, 200);
         ofPolyline l;
         l.addVertex(originCircleX, originCircleY);
         l.bezierTo(bezierCX1, bezierCY1,
@@ -79,7 +83,7 @@ class EsenciaPanelBase {
         // Draw a 3x3 rectangle moving along the Bezier line
         float percent = fmod((oy + ofGetElapsedTimef()) / 5, 1.0f); // Get a percentage value that loops from 0 to 1
         ofVec3f rectPos = l.getPointAtPercent(percent);
-        ofSetColor(ofColor::paleTurquoise, 180);
+        ofSetColor(FLOW_COLOR, 220);
         ofDrawRectangle(rectPos.x - 1.5f, rectPos.y - 1.5f, 3, 3);
 
         ofPopMatrix();