From 025928ff7375a7fa96069c4a152dd52c1098b4ad Mon Sep 17 00:00:00 2001 From: Image Builder Date: Fri, 24 Oct 2025 11:48:47 +0000 Subject: [PATCH 1/2] switch to iostream with big buffer --- src/Stream.cpp | 4 +++- src/Stream.hpp | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Stream.cpp b/src/Stream.cpp index 2dc00ad..09525f5 100644 --- a/src/Stream.cpp +++ b/src/Stream.cpp @@ -1,10 +1,12 @@ #include "Stream.hpp" #include -#include #include + + pocolog_cpp::Stream::Stream(const pocolog_cpp::StreamDescription& desc, pocolog_cpp::Index& index) : desc(desc), index(index) { + fileStream.rdbuf()->pubsetbuf(ReadBuffer,16777216); fileStream.open(desc.getFileName().c_str(), std::ifstream::binary | std::ifstream::in); if(!fileStream.good()) throw std::runtime_error("Error, could not open logfile for stream " + desc.getName()); diff --git a/src/Stream.hpp b/src/Stream.hpp index 35f0910..26cd85f 100644 --- a/src/Stream.hpp +++ b/src/Stream.hpp @@ -7,6 +7,8 @@ #include "Index.hpp" #include "FileStream.hpp" +#include + namespace pocolog_cpp { @@ -17,7 +19,10 @@ class Stream const StreamDescription &desc; Index &index; - FileStream fileStream; + //FileStream fileStream; + std::ifstream fileStream; + char ReadBuffer[16777216]; //16MB + Stream(const StreamDescription &desc, Index &index); bool loadSampleHeader(std::streampos pos, pocolog_cpp::SampleHeaderData& header); @@ -75,7 +80,7 @@ class Stream return index.getNumSamples(); } - const FileStream& getFileStream() const + const std::ifstream& getFileStream() const { return fileStream; } From 9a26e139e0b0f550bf19f0368cc45ae96e6b2533 Mon Sep 17 00:00:00 2001 From: Steffen Planthaber Date: Mon, 27 Oct 2025 15:56:37 +0100 Subject: [PATCH 2/2] use unbuffered iostreams --- src/Index.hpp | 5 +++-- src/IndexFile.cpp | 4 ++-- src/LogFile.cpp | 8 +++++++- src/LogFile.hpp | 6 ++++-- src/Stream.cpp | 3 ++- src/Stream.hpp | 2 +- src/StreamDescription.hpp | 2 +- 7 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/Index.hpp b/src/Index.hpp index 0909de5..1e4b0e4 100644 --- a/src/Index.hpp +++ b/src/Index.hpp @@ -7,7 +7,8 @@ #include #include #include "StreamDescription.hpp" -#include "FileStream.hpp" +// #include "FileStream.hpp" +#include namespace pocolog_cpp { @@ -87,7 +88,7 @@ class Index bool firstAdd; size_t curSampleNr; IndexInfo curIndexInfo; - FileStream indexFile; + std::ifstream indexFile; std::vector buildBuffer; IndexPrologue prologue; base::Time firstSampleTime; diff --git a/src/IndexFile.cpp b/src/IndexFile.cpp index 6c7e6c7..50a4c1f 100644 --- a/src/IndexFile.cpp +++ b/src/IndexFile.cpp @@ -155,7 +155,7 @@ bool IndexFile::createIndexFile(std::string indexFileName, LogFile& logFile) { throw std::runtime_error("IndexFile: Error building Index, Unexpected stream index"); } - foundIndices.emplace_back(newStream, descPos); + //foundIndices.emplace_back(newStream, descPos); break; } case DataBlockType: @@ -203,7 +203,7 @@ bool IndexFile::createIndexFile(std::string indexFileName, LogFile& logFile) off_t curProloguePos = sizeof(IndexFileHeader); off_t curDataPos = foundIndices.size() * Index::getPrologueSize() + sizeof(IndexFileHeader); - for ( auto curIdx : foundIndices ) + for ( auto &curIdx : foundIndices ) { LOG_INFO_S << "Writing index for stream " << curIdx.getName() << " , num samples " << curIdx.getNumSamples(); diff --git a/src/LogFile.cpp b/src/LogFile.cpp index 79fb00a..632372d 100644 --- a/src/LogFile.cpp +++ b/src/LogFile.cpp @@ -13,13 +13,19 @@ namespace pocolog_cpp LogFile::LogFile(const std::string& fileName, bool verbose) : filename(fileName) { + logFile.rdbuf()->pubsetbuf(0,0); logFile.open(fileName.c_str(), std::ifstream::binary | std::ifstream::in); + if (!logFile.good()){ std::cerr << "\ncould not load " << fileName.c_str() << std::endl; perror("stat"); throw std::runtime_error("Error, empty File"); } + fileSize = logFile.tellg(); + logFile.seekg( 0, std::ios::end ); + fileSize = logFile.tellg() - fileSize; + // Initialize position attributes, read or create the log file rewind(); IndexFile *indexFile = new IndexFile(*this); @@ -234,7 +240,7 @@ bool LogFile::readSampleHeader() bool LogFile::checkSampleComplete() { - return (logFile.size() >= nextBlockHeaderPos); + return (fileSize >= nextBlockHeaderPos); } std::streampos LogFile::getSamplePos() const diff --git a/src/LogFile.hpp b/src/LogFile.hpp index 3023656..d9641dc 100644 --- a/src/LogFile.hpp +++ b/src/LogFile.hpp @@ -5,7 +5,8 @@ #include #include "Stream.hpp" #include "Format.hpp" -#include "FileStream.hpp" +// #include "FileStream.hpp" +#include #include "OwnedValue.hpp" namespace pocolog_cpp @@ -20,8 +21,9 @@ class LogFile std::streampos nextBlockHeaderPos; std::streampos curBlockHeaderPos; std::streampos curSampleHeaderPos; - FileStream logFile; + std::ifstream logFile; std::vector indexFiles; + std::streampos fileSize; std::vector streams; std::vector descriptions; diff --git a/src/Stream.cpp b/src/Stream.cpp index 09525f5..5128ee2 100644 --- a/src/Stream.cpp +++ b/src/Stream.cpp @@ -6,7 +6,7 @@ pocolog_cpp::Stream::Stream(const pocolog_cpp::StreamDescription& desc, pocolog_cpp::Index& index) : desc(desc), index(index) { - fileStream.rdbuf()->pubsetbuf(ReadBuffer,16777216); + fileStream.rdbuf()->pubsetbuf(0,0); fileStream.open(desc.getFileName().c_str(), std::ifstream::binary | std::ifstream::in); if(!fileStream.good()) throw std::runtime_error("Error, could not open logfile for stream " + desc.getName()); @@ -27,6 +27,7 @@ bool pocolog_cpp::Stream::loadSampleHeader(std::streampos pos, SampleHeaderData bool pocolog_cpp::Stream::getSampleData(std::vector< uint8_t >& result, size_t sampleNr) { + printf("%s:%i %li \n", __PRETTY_FUNCTION__, __LINE__, sampleNr); std::streampos samplePos = index.getSamplePos(sampleNr); std::streampos sampleHeaderPos = samplePos; sampleHeaderPos -= sizeof(SampleHeaderData); diff --git a/src/Stream.hpp b/src/Stream.hpp index 26cd85f..8001e04 100644 --- a/src/Stream.hpp +++ b/src/Stream.hpp @@ -5,7 +5,7 @@ #include "Format.hpp" #include "StreamDescription.hpp" #include "Index.hpp" -#include "FileStream.hpp" +// #include "FileStream.hpp" #include diff --git a/src/StreamDescription.hpp b/src/StreamDescription.hpp index 13ddf1b..de49ebe 100644 --- a/src/StreamDescription.hpp +++ b/src/StreamDescription.hpp @@ -7,7 +7,7 @@ #include #include "Format.hpp" -#include "FileStream.hpp" +// #include "FileStream.hpp" #include namespace pocolog_cpp