From 1b8f1e8fc87bfe9082ce03a51b4441f61f408190 Mon Sep 17 00:00:00 2001 From: luis toledo Date: Thu, 6 Mar 2025 11:44:11 +0100 Subject: [PATCH 01/11] Fix BG-substraction reference crashes when new source has different resolution --- src/camera/Camera.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/camera/Camera.cpp b/src/camera/Camera.cpp index 36a070b..cf9d021 100644 --- a/src/camera/Camera.cpp +++ b/src/camera/Camera.cpp @@ -612,20 +612,25 @@ void Camera::saveBackgroundReference(ofxCvGrayscaleImage image) { /// -/// Reloads a previous reference from the disk +/// Reloads a previous background reference from the disk /// /// /// True if loads the reference successfully bool Camera::restoreBackgroundReference(ofxCvGrayscaleImage & outputImage) { - ofLogNotice("Camera::restoreBackgroundReference") << "Attempting to load a background reference at data/" + BG_REFERENCE_FILENAME; + ofLogNotice(__FUNCTION__) << "Attempting to load a background reference at data/" + BG_REFERENCE_FILENAME; ofPixels pixels; bool loaded = ofLoadImage(pixels, BG_REFERENCE_FILENAME); if (loaded) { - // to-do: validate bg ref and frame have the same size resolution + // validate if the bg ref and the camera frame have the same size to continue, otherwise delete the bg ref for sanity + if (pixels.getWidth() != IMG_WIDTH || pixels.getHeight() != IMG_HEIGHT) { + ofLogError("Camera::restoreBackgroundReference") << "The background reference image has a different resolution from the current frame and will be deleted"; + ofFile::removeFile(BG_REFERENCE_FILENAME); + return false; + } outputImage.allocate(pixels.getWidth(), pixels.getHeight()); outputImage.setFromPixels(pixels); backgroundReferenceTaken = true; - parameters->previewBackground.setFromPixels(backgroundReference.getPixels()); // updates the gui preview // not perfect code, since this function should be agnostic and saving value to a pointer, but this param access is used directly + parameters->previewBackground.setFromPixels(backgroundReference.getPixels()); // updates the gui preview // not perfect code, since this function should be agnostic and saving value to a pointer, but using a param is convenient and params are already glueing the modules ofLogNotice("Camera::restoreBackgroundReference") << "Load successfull"; } else { From 61b9ecf6948a085b380027c8adc0a96b95355a27 Mon Sep 17 00:00:00 2001 From: luis toledo Date: Fri, 7 Mar 2025 13:22:26 +0100 Subject: [PATCH 02/11] Update video source class/file name --- esencia.sln | 10 +++++++++- esencia.vcxproj | 2 +- esencia.vcxproj.filters | 2 +- src/gui/GuiApp.h | 4 ++-- .../panels/{VideoOriginPanel.h => VideoSourcePanel.h} | 3 +-- src/main.cpp | 4 ++-- 6 files changed, 16 insertions(+), 9 deletions(-) rename src/gui/panels/{VideoOriginPanel.h => VideoSourcePanel.h} (96%) diff --git a/esencia.sln b/esencia.sln index 42a863d..095dde9 100644 --- a/esencia.sln +++ b/esencia.sln @@ -1,9 +1,17 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35527.113 d17.12 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "esencia", "esencia.vcxproj", "{7FD42DF7-442E-479A-BA76-D0022F99702A}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "openframeworksLib", "..\..\..\libs\openFrameworksCompiled\project\vs\openframeworksLib.vcxproj", "{5837595D-ACA9-485C-8E76-729040CE4B0B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{E2104F86-B40A-465C-8AE0-704E30EB6E5C}" + ProjectSection(SolutionItems) = preProject + LICENSE = LICENSE + README.md = README.md + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 diff --git a/esencia.vcxproj b/esencia.vcxproj index a5da00b..4c4bcd6 100644 --- a/esencia.vcxproj +++ b/esencia.vcxproj @@ -163,7 +163,7 @@ - + diff --git a/esencia.vcxproj.filters b/esencia.vcxproj.filters index e22990d..4a93195 100644 --- a/esencia.vcxproj.filters +++ b/esencia.vcxproj.filters @@ -530,7 +530,7 @@ src\gui\panels - + src\gui\panels diff --git a/src/gui/GuiApp.h b/src/gui/GuiApp.h index a701f7d..c3ee2fd 100644 --- a/src/gui/GuiApp.h +++ b/src/gui/GuiApp.h @@ -11,7 +11,7 @@ #include "panels/SystemstatsPanel.h" #include "panels/SimulationPanel.h" #include "panels/RenderPanel.h" -#include "panels/VideoOriginPanel.h" +#include "panels/VideoSourcePanel.h" #include "panels/VideoProcessingPanel.h" #include "panels/PresetsPanel.h" #include "panels/SequencePanel.h" @@ -57,7 +57,7 @@ class GuiApp VideoProcessingPanel videoProcessingPanel; - VideoOriginPanel videoOriginPanel; + VideoSourcePanel videoOriginPanel; RenderPanel renderPanel; diff --git a/src/gui/panels/VideoOriginPanel.h b/src/gui/panels/VideoSourcePanel.h similarity index 96% rename from src/gui/panels/VideoOriginPanel.h rename to src/gui/panels/VideoSourcePanel.h index 9836da3..7dadf7b 100644 --- a/src/gui/panels/VideoOriginPanel.h +++ b/src/gui/panels/VideoSourcePanel.h @@ -2,7 +2,7 @@ #include "EsenciaPanelBase.h" -class VideoOriginPanel : public EsenciaPanelBase { +class VideoSourcePanel : public EsenciaPanelBase { const bool SOURCE_ORBBEC = { false }; const bool SOURCE_FILE = { false }; @@ -27,7 +27,6 @@ class VideoOriginPanel : public EsenciaPanelBase { cameraSourcePanel->add(params._sourceOrbbec.set("orbbec camera", SOURCE_ORBBEC)); cameraSourcePanel->add(params._sourceVideofile.set("video file", SOURCE_FILE)); - //cameraSourcePanel->setWidth(w * 30); cameraSourcePanel->minimize(); // DEPTH CLIPPING diff --git a/src/main.cpp b/src/main.cpp index 4517596..0cbc87c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -20,8 +20,8 @@ int main() // gui window - settings.setSize(34*30, 33*30); - settings.setPosition(glm::vec2(0,50)); + settings.setSize(34*30, 34*30); + settings.setPosition(glm::vec2(0,40)); settings.resizable = true; auto mainWindow = ofCreateWindow(settings); shared_ptr mainApp(new ofApp); From 8a1a033f54c29a28a9df26b0b14407dfb9380bd7 Mon Sep 17 00:00:00 2001 From: luis toledo Date: Fri, 7 Mar 2025 13:32:09 +0100 Subject: [PATCH 03/11] Update Readme with instructions and shortcuts Add License --- LICENSE | 21 +++++++++++++++++++++ README.md | 28 ++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..aa758de --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Intangible Realities Lab (https://www.intangiblerealitieslab.org), University of Santiago de Compostela and other contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 9cc2cef..a6829fa 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Currently working on the [Orbbec Femto Bolt](https://www.orbbec.com/products/tof ### Developing environment 1. [Install and setup openFrameworks 0.12](https://openframeworks.cc/download/) - - [Windows visual studio 2019](https://openframeworks.cc/setup/vs/) + - [Windows visual studio 2019/2022](https://openframeworks.cc/setup/vs/) Recommended - [Linux visual studio code](https://openframeworks.cc/setup/vscode/) Not tested - [OSX/macOS via Xcode](https://openframeworks.cc/setup/xcode/) Orbbec Femto Bolt camera is not currently supported on macOS @@ -40,7 +40,8 @@ Currently working on the [Orbbec Femto Bolt](https://www.orbbec.com/products/tof 1. Connect the camera - If ther camera is not connected or detected, it will automatically play a prerecorded video instead (file included). - +- Place the camera and turn it 90 degrees to the right. + 2. Run the project: - For Windows Visual Studio 2019 open the solution `esencia.sln` @@ -49,6 +50,25 @@ Currently working on the [Orbbec Femto Bolt](https://www.orbbec.com/products/tof - Or just run `make` from the terminal **Release mode gives better performance.** -3. Play with the gui parameters [See the wiki](https://github.com/IRL2/esencia/wiki). - - Currently the simulation runs on CPU so its limited to 200 particles. +3. Save a background reference image (for the background subtraction) + +- Once its running, expand the video source panel source and background groups +- Ensure the camera is pointing to the right space +- Clear the space +- Click on the `Save Background` button or press `Ctrl+b` on the keyboard + +4. Play with the gui parameters [See the wiki](https://github.com/IRL2/esencia/wiki). + +### Shortcuts + +On the GUI window: +- `Ctrl+b` Save background +On the main simulation window: +- `f` Toggle fullscreen +- `1-9` Change the presets (combine with shift for 10-19) +- `Shift+` `1-9` Save the current preset +- `space` Start/stop the sequencer +- `m` Mutates the current preset +- `Ctrl+s` Save the preset on the current slot +- `Ctrl+c` Clear the current preset slot \ No newline at end of file From 2761670563ad59aff0ab95b2d7d3111633b0c842 Mon Sep 17 00:00:00 2001 From: luis toledo Date: Wed, 30 Apr 2025 16:05:13 +0200 Subject: [PATCH 04/11] Add screenshot to render Fix trail noise using gl_rgba fbos --- src/render/RenderApp.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/render/RenderApp.cpp b/src/render/RenderApp.cpp index 61bf081..73e838b 100644 --- a/src/render/RenderApp.cpp +++ b/src/render/RenderApp.cpp @@ -74,6 +74,8 @@ void RenderApp::draw() shader.setUniform1f("blurAmnt", 5); shader.setUniform1f("texwidth", ofGetWidth()); shader.setUniform1f("texheight", ofGetHeight()); + ofBackground(0); + fbo.draw(0,0); shader.end(); fboS.end(); @@ -103,8 +105,14 @@ void RenderApp::keyReleased(ofKeyEventArgs& e) case 'F': { ofToggleFullscreen(); - fbo.allocate(ofGetWidth(), ofGetHeight()); - fboS.allocate(ofGetWidth(), ofGetHeight()); + fbo.allocate(ofGetWidth(), ofGetHeight(), GL_RGBA); + fboS.allocate(ofGetWidth(), ofGetHeight(), GL_RGBA); + break; + } + + case 'S': + { + ofSaveFrame(); break; } default: break; @@ -125,8 +133,8 @@ void RenderApp::mouseMoved(int x, int y ){ void RenderApp::windowResized(int _width, int _height) { ofLogNotice("RenderApp::windowResized()") << "window resized to: " << _width << "," << _height; - fbo.allocate(_width, _height); - fboS.allocate(_width, _height); + fbo.allocate(_width, _height, GL_RGBA); + fboS.allocate(_width, _height, GL_RGBA); glm::vec2 newSize = glm::vec2(_width, _height); parameters->windowSize.set(glm::vec2(_width, _height)); From ce2127aa2e0cc4e4149caef97a8b4b66a4e7a471 Mon Sep 17 00:00:00 2001 From: luis toledo Date: Wed, 30 Apr 2025 16:23:43 +0200 Subject: [PATCH 05/11] Add screenshot and trailblur on gui --- src/gui/GuiApp.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/gui/GuiApp.cpp b/src/gui/GuiApp.cpp index 6fc28c7..ed7c9df 100644 --- a/src/gui/GuiApp.cpp +++ b/src/gui/GuiApp.cpp @@ -5,7 +5,7 @@ void GuiApp::setup() { ofBackground(0); - fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight()); + fbo.allocate(ofGetWindowWidth(), ofGetWindowHeight(), GL_RGBA32F_ARB); ofEnableSmoothing(); gui.setupFlexBoxLayout(); @@ -38,7 +38,7 @@ void GuiApp::update() { presetsPanel.update(); - float colorProgress = ofxeasing::map_clamp(bgChangeFrequency++, 0, bgChangeDuration, 0, 1, &ofxeasing::linear::easeInOut); + float colorProgress = ofxeasing::map_clamp(bgChangeFrequency++, 0, bgChangeDuration, 0, 1, &ofxeasing::linear::easeInOut); if (bgChangeFrequency == bgChangeDuration) bgChangeFrequency = 0; if (colorProgress == 0) { @@ -59,8 +59,11 @@ void GuiApp::update() void GuiApp::draw() { - fbo.begin(); ofBackgroundGradient(bgColor2, bgColor1, OF_GRADIENT_LINEAR); + fbo.begin(); + + ofSetColor(255, 150); + ofDrawRectangle(0, 0, ofGetWidth(), ofGetHeight()); // draw lines EsenciaPanelBase::drawLineBetween(videoOriginPanel, videoProcessingPanel); @@ -75,10 +78,13 @@ void GuiApp::draw() - void GuiApp::keyReleased(ofKeyEventArgs& e) { presetsPanel.keyReleased(e); sequencePanel.keyReleased(e); + + if (e.keycode == 'S' && e.hasModifier(OF_KEY_SHIFT)) { + ofSaveFrame(true); + } } From 09d3fd401c47bb2250bedef274d31aba9276c292 Mon Sep 17 00:00:00 2001 From: luis toledo Date: Wed, 30 Apr 2025 16:24:21 +0200 Subject: [PATCH 06/11] small refactor on base panel colors --- src/gui/EsenciaPanelBase.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/gui/EsenciaPanelBase.h b/src/gui/EsenciaPanelBase.h index 09bedb2..e9a8658 100644 --- a/src/gui/EsenciaPanelBase.h +++ b/src/gui/EsenciaPanelBase.h @@ -9,6 +9,10 @@ const int PANELS_CIRCLE_RESOLUTION = 8; const int PANELS_BEZIER_PADDING = 40; const int PANELS_BEZIER_RESOLUTION = 10; const int PANELS_TRIANGLE_SIZE = 4; +const ofColor LINE_COLOR1 = ofColor::darkSlateGrey; //paleGoldenRod; +const ofColor LINE_COLOR2 = ofColor::darkSlateGrey; //paleTurquoise; +const ofColor LINE_COLOR3 = ofColor::darkSlateGrey; // khaki; +const ofColor FLOW_COLOR = ofColor::darkSlateGrey; //paleGoldenRod; class EsenciaPanelBase { @@ -58,16 +62,16 @@ class EsenciaPanelBase { ofPushMatrix(); // origin glyph - ofSetColor(ofColor::paleGoldenRod, 200); + ofSetColor(LINE_COLOR2, 200); ofSetCircleResolution(PANELS_CIRCLE_RESOLUTION); ofFill(); ofDrawCircle(originCircleX, originCircleY, PANELS_CIRCLE_RADIUS); // destination glyph - ofSetColor(ofColor::paleTurquoise, 200); + ofSetColor(LINE_COLOR1, 200); ofDrawArrow(destinationArrowStart, destinationArrowEnd, PANELS_TRIANGLE_SIZE); - ofSetColor(ofColor::khaki, 200); + ofSetColor(LINE_COLOR3, 200); ofPolyline l; l.addVertex(originCircleX, originCircleY); l.bezierTo(bezierCX1, bezierCY1, @@ -79,7 +83,7 @@ class EsenciaPanelBase { // Draw a 3x3 rectangle moving along the Bezier line float percent = fmod((oy + ofGetElapsedTimef()) / 5, 1.0f); // Get a percentage value that loops from 0 to 1 ofVec3f rectPos = l.getPointAtPercent(percent); - ofSetColor(ofColor::paleTurquoise, 180); + ofSetColor(FLOW_COLOR, 220); ofDrawRectangle(rectPos.x - 1.5f, rectPos.y - 1.5f, 3, 3); ofPopMatrix(); From 5fe8ddef411d8c74873be3297e0bf907488c22b1 Mon Sep 17 00:00:00 2001 From: luis toledo Date: Wed, 30 Apr 2025 16:24:58 +0200 Subject: [PATCH 07/11] Add gui node borders --- bin/data/support/gui-styles.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/data/support/gui-styles.json b/bin/data/support/gui-styles.json index f475c7a..240127e 100644 --- a/bin/data/support/gui-styles.json +++ b/bin/data/support/gui-styles.json @@ -2,6 +2,9 @@ "default": { "base": { "border-radius": 4, + "border-color": "#444444", + "fill-color": "#aaaaaa", + "border-width": 1, "text-color": "#ffffef" }, From a4d5d82f6825ff073ae499f2aad70cff79993446 Mon Sep 17 00:00:00 2001 From: luis toledo Date: Wed, 30 Apr 2025 16:25:22 +0200 Subject: [PATCH 08/11] Update sequencer labels, size and position --- src/gui/panels/SequencePanel.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gui/panels/SequencePanel.h b/src/gui/panels/SequencePanel.h index cc26b4c..78bbc18 100644 --- a/src/gui/panels/SequencePanel.h +++ b/src/gui/panels/SequencePanel.h @@ -5,7 +5,7 @@ class SequencePanel : public EsenciaPanelBase { - const ofRectangle PANEL_RECT = ofRectangle(11, 25, 12, 0); + const ofRectangle PANEL_RECT = ofRectangle(13, 26, 8, 0); const ofColor &BG_COLOR = ofColor(100, 100, 100, 100); const float DEFAULT_TRANSITION_DURATION_INIT = 5.0; @@ -27,7 +27,7 @@ class SequencePanel : public EsenciaPanelBase { ofParameter curPreset; void setup(ofxGui& gui, PresetsParameters *params, ofxPresets&presetMan) { - panel = gui.addPanel("sequence"); + panel = gui.addPanel("sequencer"); presetParams = params; presetManager = &presetMan; @@ -98,7 +98,7 @@ class SequencePanel : public EsenciaPanelBase { playButton->setNeedsRedraw(); } void updatePlaybuttonToStopped() { - playButton->setLabel("stopped"); + playButton->setLabel("play"); playButton->setBackgroundColor(ofColor(ofColor::orangeRed, 200)); playButton->setNeedsRedraw(); } From f08a7e10deb94067b7dd6fcab461a62f9daf9e3d Mon Sep 17 00:00:00 2001 From: luis toledo Date: Fri, 2 May 2025 12:07:39 +0200 Subject: [PATCH 09/11] Add debug option on videosource to record the video processing steps + On video processing, improve log messages and update image debugging during the process --- src/camera/Camera.cpp | 44 +++++++++++++++++++++++-------- src/camera/Camera.h | 9 ++++--- src/gui/GuiApp.h | 1 + src/gui/panels/VideoSourcePanel.h | 12 ++++++--- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/camera/Camera.cpp b/src/camera/Camera.cpp index cf9d021..20f92db 100644 --- a/src/camera/Camera.cpp +++ b/src/camera/Camera.cpp @@ -36,8 +36,6 @@ void Camera::setup(CameraParameters* params) { else { changeSource(VideoSources::VIDEOSOURCE_VIDEOFILE); } - - restoreBackgroundReference(backgroundReference); } /// @@ -67,6 +65,8 @@ void Camera::onGUIChangeSource(bool& _) { /// /// From the list of VideoSources void Camera::changeSource(VideoSources newSource) { + ofLogNotice(__FUNCTION__) << "Attempting to change source to " << (int)newSource; + stopCurrentSource(); currentVideosource = VideoSources::VIDEOSOURCE_NONE; // in the new setup fail, stay at None @@ -80,6 +80,9 @@ void Camera::changeSource(VideoSources newSource) { else if (newSource == VideoSources::VIDEOSOURCE_WEBCAM) { setupWebcam(); } + else { + return; + } // to-do: need to call setFrameSize with the updated frame size from the new source setFrameSize(cameraResolutions[selectedOrbbecResolution].x, cameraResolutions[selectedOrbbecResolution].y); @@ -107,9 +110,15 @@ void Camera::changeSource(VideoSources newSource) { /// /// void Camera::setFrameSize(int width, int height) { + ofLogNotice(__FUNCTION__) << "Resizing every image and fb to " << width << ", " << height; + + if (currentVideosource == VideoSources::VIDEOSOURCE_ORBBEC) { if (orbbecSettings.rotation == OB_ROTATE_DEGREE_90 || orbbecSettings.rotation == OB_ROTATE_DEGREE_270) { + ofLogNotice(__FUNCTION__) << "Rotating orbbec camera also rotates dimentions (rotation " << orbbecSettings.rotation << " degrees)"; std::swap(width, height); } + } + IMG_WIDTH = width; IMG_HEIGHT = height; @@ -154,10 +163,10 @@ void Camera::onGUIStartBackgroundReference(bool &value) { /// Loads a video file to use it as the video source /// void Camera::loadVideoFile() { - ofLogNotice("Camera::loadVideoFile()") << "Switching source to video file"; + ofLogNotice(__FUNCTION__) << "Switching source to video file"; // to-do: let the user select the file (ofFileDialogResult from ofSystemLoadDialog) - ofLogNotice("Camera::loadVideoFile()") << "Attempting to load a video file"; + ofLogNotice(__FUNCTION__) << "Attempting to load a video file"; prerecordedVideo.load("video_mocks/movement_nfov_h264.mp4"); // to-do: throw error when file does not exist, or cant be loaded @@ -165,7 +174,7 @@ void Camera::loadVideoFile() { prerecordedVideo.play(); currentVideosource = VideoSources::VIDEOSOURCE_VIDEOFILE; - ofLogNotice("Camera::loadVideoFile()") << "Camera::loadVideoFile Video file loaded"; + ofLogNotice(__FUNCTION__) << "Video file loaded"; } /// @@ -215,10 +224,11 @@ void Camera::stopCurrentSource() { ofLogNotice("Camera::stopCurrentSource()") << "Closing the orbbec camera"; orbbecCam.close(); } - else if (currentVideosource == VideoSources::VIDEOSOURCE_VIDEOFILE) { + if (currentVideosource == VideoSources::VIDEOSOURCE_VIDEOFILE) { ofLogNotice("Camera::stopCurrentSource()") << "Stopping the video file playback"; prerecordedVideo.stop(); } + currentVideosource = VideoSources::VIDEOSOURCE_NONE; // webcam.stop } @@ -310,6 +320,8 @@ void Camera::update() { // update the preview images on the shared parameters data structure for the GUI parameters->previewSource.setFromPixels(source.getPixels()); convertToTransparent(segment, parameters->previewSegment); // to-do: should be called only when accessed + + saveDebugImage(segment, "segment", "final"); // parameters->previewBackground.setFromPixels(backgroundReference.getPixels()); // this does not need to run every update, so its placed when updating backgroundReference data } @@ -366,7 +378,7 @@ void Camera::processCameraFrame(ofxCvGrayscaleImage &frame, ofxCvGrayscaleImage if (backgroundReferenceTaken) { cvAbsDiff(processedImage.getCvImage(), backgroundNewFrame.getCvImage(), segment.getCvImage()); // this works great for a single background frame of reference! - saveDebugImage(processedImage, "processedImage", "removed background absdiff"); + saveDebugImage(segment, "segment", "removed background absdiff"); } else { segment = processedImage; @@ -422,6 +434,8 @@ void Camera::processCameraFrame(ofxCvGrayscaleImage &frame, ofxCvGrayscaleImage getContourPolygonsFromImage(processedImage, &polygons); } + saveDebugImage(processedImage, "processedImage", "pre-gpuBlur"); + // add gaussian blur to the silouetes // (this step was originaly performed by the simulation on the sorounds of each particle, its here now to test if the performance is better) gpuBlur(processedImage, parameters->gaussianBlur); @@ -582,7 +596,7 @@ void Camera::startBackgroundReferenceSampling() { } void Camera::startBackgroundReferenceSampling(int samples) { - ofLogNotice("Camera::startBackgroundReferenceSampling()") << "Starting background"; + ofLogNotice(__FUNCTION__) << "Starting background sampling for futher substraction"; isTakingBackgroundReference = true; backgroundReferenceTaken = false; clearBackgroundReference(); @@ -591,7 +605,7 @@ void Camera::startBackgroundReferenceSampling(int samples) { } void Camera::clearBackgroundReference() { - ofLogNotice("Camera::clearBackgroundReference()") << "Clearing background"; + ofLogNotice("Camera::clearBackgroundReference()") << "Cleaning background"; backgroundReference.set(0); parameters->previewBackground.clear(); } @@ -618,8 +632,10 @@ void Camera::saveBackgroundReference(ofxCvGrayscaleImage image) { /// True if loads the reference successfully bool Camera::restoreBackgroundReference(ofxCvGrayscaleImage & outputImage) { ofLogNotice(__FUNCTION__) << "Attempting to load a background reference at data/" + BG_REFERENCE_FILENAME; + ofPixels pixels; bool loaded = ofLoadImage(pixels, BG_REFERENCE_FILENAME); + if (loaded) { // validate if the bg ref and the camera frame have the same size to continue, otherwise delete the bg ref for sanity if (pixels.getWidth() != IMG_WIDTH || pixels.getHeight() != IMG_HEIGHT) { @@ -678,10 +694,16 @@ void convertToTransparent(ofxCvGrayscaleImage &grayImage, ofImage &rgbaImage) { void Camera::saveDebugImage(ofxCvGrayscaleImage img, string name, string step) { #ifdef DEBUG_IMAGES if (parameters->saveDebugImages) { - const string& filename = ofGetTimestampString() + "_" + name + "_" + step + ".png"; + const string& filename = "raw_recording\\" + ofGetTimestampString() + "_" + name + "_" + step + ".png"; ofSaveImage(img.getPixels(), filename); } -#endif +} + +void Camera::saveDebugImage(ofPixels img, string name, string step) { + if (parameters->saveDebugImages) { + const string& filename = "raw_recording\\" + ofGetTimestampString() + "_" + name + "_" + step + ".png"; + ofSaveImage(img, filename); + } } void Camera::recordTestingFrames(ofxCvGrayscaleImage img) { diff --git a/src/camera/Camera.h b/src/camera/Camera.h index f66b88c..6190e78 100644 --- a/src/camera/Camera.h +++ b/src/camera/Camera.h @@ -9,10 +9,10 @@ class Camera { enum class VideoSources : int { - VIDEOSOURCE_ORBBEC, - VIDEOSOURCE_VIDEOFILE, - VIDEOSOURCE_WEBCAM, - VIDEOSOURCE_NONE + VIDEOSOURCE_ORBBEC = 1, + VIDEOSOURCE_VIDEOFILE = 2, + VIDEOSOURCE_WEBCAM = 3, + VIDEOSOURCE_NONE = 0 }; public: @@ -34,6 +34,7 @@ class Camera void getContourPolygonsFromImage(ofxCvGrayscaleImage image, vector* output); void saveDebugImage(ofxCvGrayscaleImage img, string name, string step); + void saveDebugImage(ofPixels img, string name, string step); void recordTestingFrames(ofxCvGrayscaleImage frame); uint64 recordTestingFramesCounter = 0; void saveMeshFrame(); diff --git a/src/gui/GuiApp.h b/src/gui/GuiApp.h index c3ee2fd..81a0552 100644 --- a/src/gui/GuiApp.h +++ b/src/gui/GuiApp.h @@ -3,6 +3,7 @@ #include "ofMain.h" #include "ofxGuiExtended.h" #include "ofxOpenCv.h" +#include "ofxEasing.h" #include "ofxPresetsParametersBase.h" #include "EsenciaParameters.h" diff --git a/src/gui/panels/VideoSourcePanel.h b/src/gui/panels/VideoSourcePanel.h index 7dadf7b..324e812 100644 --- a/src/gui/panels/VideoSourcePanel.h +++ b/src/gui/panels/VideoSourcePanel.h @@ -2,6 +2,10 @@ #include "EsenciaPanelBase.h" +#define DEBUG_IMAGES false +//#define RECORD_TESTING_VIDEO true + + class VideoSourcePanel : public EsenciaPanelBase { const bool SOURCE_ORBBEC = { false }; @@ -64,12 +68,14 @@ class VideoSourcePanel : public EsenciaPanelBase { +#if defined DEBUG_IMAGES || defined RECORD_TESTING_VIDEO + ofxGuiGroup* debug = panel->addGroup("debug"); +#endif #ifdef DEBUG_IMAGES - cameraGroup.add(ofParameter().set("DEBUG")); - camera.add(cameraParameters.saveDebugImages.set("save debug images", false)); + debug->add(params.saveDebugImages.set("save debug images", false)); #endif #ifdef RECORD_TESTING_VIDEO - camera.add(cameraParameters.recordTestingVideo.set("record testing video", false)); + debug.add(cameraParameters.recordTestingVideo.set("record testing video", false)); #endif From d9bd9367005a981340420c19eca96a5b8d6314cf Mon Sep 17 00:00:00 2001 From: luis toledo Date: Fri, 2 May 2025 12:10:39 +0200 Subject: [PATCH 10/11] Update/fix source switching --- src/camera/Camera.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/camera/Camera.cpp b/src/camera/Camera.cpp index 20f92db..7fd6306 100644 --- a/src/camera/Camera.cpp +++ b/src/camera/Camera.cpp @@ -290,7 +290,14 @@ void Camera::draw() { void Camera::update() { // acquire frame - if (currentVideosource == VideoSources::VIDEOSOURCE_VIDEOFILE) { + if (currentVideosource == VideoSources::VIDEOSOURCE_ORBBEC) { + orbbecCam.update(); + + if (orbbecCam.isFrameNewDepth()) { + source.setFromPixels(orbbecCam.getDepthPixels()); + } + } + else if (currentVideosource == VideoSources::VIDEOSOURCE_VIDEOFILE) { prerecordedVideo.update(); if (prerecordedVideo.isFrameNew()) { @@ -298,15 +305,13 @@ void Camera::update() { source = colorFrame; } } - - if (currentVideosource == VideoSources::VIDEOSOURCE_ORBBEC) { - orbbecCam.update(); - - if (orbbecCam.isFrameNewDepth()) { - source.setFromPixels(orbbecCam.getDepthPixels()); - } + else { + // no source, no update + return; } + parameters->previewSource.setFromPixels(source.getPixels()); + // to-do: make backgroundref an object with state? if (isTakingBackgroundReference) { addSampleToBackgroundReference(source, backgroundReference, BG_SAMPLE_FRAMES); @@ -316,9 +321,7 @@ void Camera::update() { // all the processing from source to extract the final segment processCameraFrame(source, backgroundReference); - // update the preview images on the shared parameters data structure for the GUI - parameters->previewSource.setFromPixels(source.getPixels()); convertToTransparent(segment, parameters->previewSegment); // to-do: should be called only when accessed saveDebugImage(segment, "segment", "final"); @@ -692,7 +695,6 @@ void convertToTransparent(ofxCvGrayscaleImage &grayImage, ofImage &rgbaImage) { /// object name id (background, camera, cleared..) /// additional id, i.e. sequence step void Camera::saveDebugImage(ofxCvGrayscaleImage img, string name, string step) { -#ifdef DEBUG_IMAGES if (parameters->saveDebugImages) { const string& filename = "raw_recording\\" + ofGetTimestampString() + "_" + name + "_" + step + ".png"; ofSaveImage(img.getPixels(), filename); From 85a11001eec5197114ccafaab66c2d5d27a696cb Mon Sep 17 00:00:00 2001 From: luis toledo Date: Fri, 2 May 2025 18:21:30 +0200 Subject: [PATCH 11/11] Fix background substraction issue when bgref and source has different sizes --- src/camera/Camera.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/camera/Camera.cpp b/src/camera/Camera.cpp index 7fd6306..a9683d4 100644 --- a/src/camera/Camera.cpp +++ b/src/camera/Camera.cpp @@ -56,6 +56,8 @@ void Camera::onGUIChangeSource(bool& _) { } else { stopCurrentSource(); } + + restoreBackgroundReference(backgroundReference); } @@ -175,6 +177,12 @@ void Camera::loadVideoFile() { currentVideosource = VideoSources::VIDEOSOURCE_VIDEOFILE; ofLogNotice(__FUNCTION__) << "Video file loaded"; + + // copy the the associated precaptured background into the right folder + ofFile::removeFile(BG_REFERENCE_FILENAME); + ofFile::copyFromTo("video_mocks\\" + BG_REFERENCE_FILENAME, BG_REFERENCE_FILENAME); + restoreBackgroundReference(backgroundReference); + ofLogNotice(__FUNCTION__) << "Load precaptured background reference from video file"; } /// @@ -197,6 +205,7 @@ void Camera::setupOrbbecCamera() { orbbecSettings.depthFrameSize.requestWidth = orbbecRequestedWidth; orbbecSettings.rotation = OB_ROTATE_DEGREE_90; orbbecCam.open(orbbecSettings); + currentVideosource = VideoSources::VIDEOSOURCE_ORBBEC; } @@ -379,6 +388,8 @@ void Camera::processCameraFrame(ofxCvGrayscaleImage &frame, ofxCvGrayscaleImage // --------- // remove the background from the frame (save the output in maskImage object) if (backgroundReferenceTaken) { + if (backgroundNewFrame.width != processedImage.width) return; + cvAbsDiff(processedImage.getCvImage(), backgroundNewFrame.getCvImage(), segment.getCvImage()); // this works great for a single background frame of reference! saveDebugImage(segment, "segment", "removed background absdiff"); @@ -607,10 +618,17 @@ void Camera::startBackgroundReferenceSampling(int samples) { backgroundReferenceLeftFrames = samples; } +/// +/// Clear the background reference image +/// will remove the previous bg reference file +/// wil set the Taken flag to false +/// void Camera::clearBackgroundReference() { ofLogNotice("Camera::clearBackgroundReference()") << "Cleaning background"; backgroundReference.set(0); parameters->previewBackground.clear(); + ofFile::removeFile(BG_REFERENCE_FILENAME); + backgroundReferenceTaken = false; }