Skip to content

Add agent_data table #433

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Grufoony wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Add agent_data table #433

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2bba407
Add `agent_data` table if enabled
Grufoony Apr 8, 2026
4284091
Add tests
Grufoony Apr 8, 2026
687a52d
Bump version
Grufoony Apr 8, 2026
dc413a4
Fix macos
Grufoony Apr 8, 2026
956ece0
Copilot fixes
Grufoony Apr 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions CITATION.cff
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ authors:
repository-code: 'https://github.com/physycom/DynamicalSystemFramework'
url: 'https://physycom.github.io/DynamicalSystemFramework/'
license: CC-BY-NC-SA-4.0
version: 5.3.1
date-released: '2026-03-06'
version: 5.3.2
date-released: '2026-04-08'
4 changes: 3 additions & 1 deletion src/dsf/bindings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -749,14 +749,16 @@ PYBIND11_MODULE(dsf_cpp, m) {
pybind11::arg("save_average_stats") = false,
pybind11::arg("save_street_data") = false,
pybind11::arg("save_travel_data") = false,
pybind11::arg("save_agent_data") = false,
"Configure data saving during simulation.\n\n"
"Args:\n"
" saving_interval: Interval in time steps between data saves\n"
" save_average_stats: Whether to save average statistics (speed, density, "
"flow)\n"
" save_street_data: Whether to save per-street data (density, speed, coil "
"counts)\n"
" save_travel_data: Whether to save travel data (distance, travel time)")
" save_travel_data: Whether to save travel data (distance, travel time)\n"
" save_agent_data: Whether to save per-agent edge traversal data")
.def(
"summary",
[](dsf::mobility::FirstOrderDynamics& self) {
Expand Down
2 changes: 1 addition & 1 deletion src/dsf/dsf.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

static constexpr uint8_t DSF_VERSION_MAJOR = 5;
static constexpr uint8_t DSF_VERSION_MINOR = 3;
static constexpr uint8_t DSF_VERSION_PATCH = 1;
static constexpr uint8_t DSF_VERSION_PATCH = 2;

static auto const DSF_VERSION =
std::format("{}.{}.{}", DSF_VERSION_MAJOR, DSF_VERSION_MINOR, DSF_VERSION_PATCH);
Expand Down
60 changes: 53 additions & 7 deletions src/dsf/mobility/FirstOrderDynamics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -893,6 +893,23 @@ namespace dsf::mobility {

spdlog::info("Initialized travel_data table in the database.");
}
void FirstOrderDynamics::m_initAgentDataTable() const {
if (!this->database()) {
throw std::runtime_error(
"No database connected. Call connectDataBase() before saving data.");
}
// Create table if it doesn't exist
this->database()->exec(
"CREATE TABLE IF NOT EXISTS agent_data ("
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
"simulation_id INTEGER NOT NULL, "
"agent_id INTEGER NOT NULL, "
"edge_id INTEGER NOT NULL, "
"time_step_in INTEGER NOT NULL, "
"time_step_out INTEGER NOT NULL)");

spdlog::info("Initialized agent_data table in the database.");
}
void FirstOrderDynamics::m_dumpSimInfo() const {
// Dump simulation info (parameters) to the database, if connected
if (!this->database()) {
Expand All @@ -914,16 +931,18 @@ namespace dsf::mobility {
"force_priorities BOOLEAN, "
"save_avg_stats BOOLEAN, "
"save_road_data BOOLEAN, "
"save_travel_data BOOLEAN)");
"save_travel_data BOOLEAN, "
"save_agent_data BOOLEAN)");
createTableStmt.exec();
// Insert simulation parameters into the simulations table
SQLite::Statement insertSimStmt(
*this->database(),
"INSERT INTO simulations (id, name, speed_function, weight_function, "
"weight_threshold, error_probability, passage_probability, "
"mean_travel_distance_m, mean_travel_time_s, stagnant_tolerance_factor, "
"force_priorities, save_avg_stats, save_road_data, save_travel_data) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
"force_priorities, save_avg_stats, save_road_data, save_travel_data, "
"save_agent_data) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
insertSimStmt.bind(1, static_cast<std::int64_t>(this->id()));
insertSimStmt.bind(2, this->name());
insertSimStmt.bind(3, this->m_speedFunctionDescription);
Expand Down Expand Up @@ -968,6 +987,7 @@ namespace dsf::mobility {
insertSimStmt.bind(12, this->m_bSaveAverageStats);
insertSimStmt.bind(13, this->m_bSaveStreetData);
insertSimStmt.bind(14, this->m_bSaveTravelData);
insertSimStmt.bind(15, this->m_bSaveAgentData);
insertSimStmt.exec();
}
void FirstOrderDynamics::m_dumpNetwork() const {
Expand Down Expand Up @@ -1199,11 +1219,13 @@ namespace dsf::mobility {
void FirstOrderDynamics::saveData(std::time_t const savingInterval,
bool const saveAverageStats,
bool const saveStreetData,
bool const saveTravelData) {
bool const saveTravelData,
bool const saveAgentData) {
m_savingInterval = savingInterval;
m_bSaveAverageStats = saveAverageStats;
m_bSaveStreetData = saveStreetData;
m_bSaveTravelData = saveTravelData;
m_bSaveAgentData = saveAgentData;

// Initialize the required tables
if (saveStreetData) {
Expand All @@ -1215,17 +1237,21 @@ namespace dsf::mobility {
if (saveTravelData) {
m_initTravelDataTable();
}

if (saveAgentData) {
m_initAgentDataTable();
Street::acquireAgentData();
}
this->m_dumpSimInfo();
this->m_dumpNetwork();

spdlog::info(
"Data saving configured: interval={}s, avg_stats={}, street_data={}, "
"travel_data={}",
"travel_data={}, agent_data={}",
savingInterval,
saveAverageStats,
saveStreetData,
saveTravelData);
saveTravelData,
saveAgentData);
Comment on lines 1247 to +1254
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The log message uses interval={}s, but savingInterval is documented as "in time steps" (not seconds). Consider adjusting the unit in the message to avoid misleading logs (e.g., interval={} steps).

Copilot uses AI. Check for mistakes.
}

void FirstOrderDynamics::setDestinationNodes(
Expand Down Expand Up @@ -1601,6 +1627,25 @@ namespace dsf::mobility {
m_travelDTs.clear();
}

if (m_bSaveAgentData) {
auto agentData = Street::agentData();
SQLite::Statement insertStmt(*this->database(),
"INSERT INTO agent_data (simulation_id, "
"agent_id, edge_id, time_step_in, time_step_out)"
"VALUES (?, ?, ?, ?, ?)");
for (auto const& [edge_id, data] : agentData) {
for (auto const& [agent_id, ts_in, ts_out] : data) {
insertStmt.bind(1, simulationId);
insertStmt.bind(2, static_cast<std::int64_t>(agent_id));
insertStmt.bind(3, static_cast<std::int64_t>(edge_id));
insertStmt.bind(4, static_cast<std::int64_t>(ts_in));
insertStmt.bind(5, static_cast<std::int64_t>(ts_out));
insertStmt.exec();
insertStmt.reset();
Comment on lines +1637 to +1644
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ts_in/ts_out are std::time_t, but they’re bound without an explicit cast. On some platforms time_t is not an int64_t, which can lead to narrowing/overload surprises. Cast ts_in/ts_out to std::int64_t when binding to match the INTEGER schema.

Copilot uses AI. Check for mistakes.
}
Comment on lines +1630 to +1645
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agent-data inserts are not covered by the hasWritePayload/transaction logic (so when only m_bSaveAgentData is enabled, inserts run in autocommit mode). This can drastically slow down writes and makes this block behave differently from the other tables. Update the transaction condition to include agent-data payload and ensure these inserts run inside the same transaction.

Suggested change
if (m_bSaveAgentData) {
auto agentData = Street::agentData();
SQLite::Statement insertStmt(*this->database(),
"INSERT INTO agent_data (simulation_id, "
"agent_id, edge_id, time_step_in, time_step_out)"
"VALUES (?, ?, ?, ?, ?)");
for (auto const& [edge_id, data] : agentData) {
for (auto const& [agent_id, ts_in, ts_out] : data) {
insertStmt.bind(1, simulationId);
insertStmt.bind(2, static_cast<std::int64_t>(agent_id));
insertStmt.bind(3, static_cast<std::int64_t>(edge_id));
insertStmt.bind(4, static_cast<std::int64_t>(ts_in));
insertStmt.bind(5, static_cast<std::int64_t>(ts_out));
insertStmt.exec();
insertStmt.reset();
}
auto agentData = Street::agentData();
if (m_bSaveAgentData && !agentData.empty()) {
this->database()->exec("SAVEPOINT agent_data_insert");
try {
SQLite::Statement insertStmt(*this->database(),
"INSERT INTO agent_data (simulation_id, "
"agent_id, edge_id, time_step_in, time_step_out)"
"VALUES (?, ?, ?, ?, ?)");
for (auto const& [edge_id, data] : agentData) {
for (auto const& [agent_id, ts_in, ts_out] : data) {
insertStmt.bind(1, simulationId);
insertStmt.bind(2, static_cast<std::int64_t>(agent_id));
insertStmt.bind(3, static_cast<std::int64_t>(edge_id));
insertStmt.bind(4, static_cast<std::int64_t>(ts_in));
insertStmt.bind(5, static_cast<std::int64_t>(ts_out));
insertStmt.exec();
insertStmt.reset();
}
}
this->database()->exec("RELEASE SAVEPOINT agent_data_insert");
} catch (...) {
this->database()->exec("ROLLBACK TO SAVEPOINT agent_data_insert");
this->database()->exec("RELEASE SAVEPOINT agent_data_insert");
throw;

Copilot uses AI. Check for mistakes.
}
}

if (m_bSaveAverageStats) { // Average Stats Table
auto const validEdges = nValidEdges.load();
auto const edgeCount = static_cast<double>(numEdges);
Expand Down Expand Up @@ -1652,6 +1697,7 @@ namespace dsf::mobility {
m_bSaveStreetData = false;
m_bSaveTravelData = false;
m_bSaveAverageStats = false;
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When saveAgentData is enabled, Street::acquireAgentData() makes Street start collecting agent events based on m_agentData.has_value(). However, in the one-shot savingInterval == 0 reset path only m_bSaveAgentData is cleared; the static Street::m_agentData remains engaged, so streets will keep accumulating agent events even though saving is disabled. Add an explicit teardown/clear (e.g., Street::releaseAgentData()/resetAgentData()), or gate collection on a flag that is reset alongside m_bSaveAgentData.

Suggested change
m_bSaveAverageStats = false;
m_bSaveAverageStats = false;
Street::releaseAgentData();

Copilot uses AI. Check for mistakes.
m_bSaveAgentData = false;
}
}

Expand Down
14 changes: 13 additions & 1 deletion src/dsf/mobility/FirstOrderDynamics.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ namespace dsf::mobility {
bool m_bSaveStreetData{false};
bool m_bSaveTravelData{false};
bool m_bSaveAverageStats{false};
bool m_bSaveAgentData{false};

private:
/// @brief Kill an agent
Expand Down Expand Up @@ -167,6 +168,15 @@ namespace dsf::mobility {
/// - distance_m: The distance travelled by the agent in meters
/// - travel_time_s: The travel time of the agent in seconds
void m_initTravelDataTable() const;
/// @brief Initialize the agent data table.
/// This table contains the agent data of the agents. Columns are:
/// - id: The entry id (auto-incremented)
/// - simulation_id: The simulation id
/// - agent_id: The id of the agent
/// - edge_id: The id of the edge
/// - time_step_in: The time step of the data entry
/// - time_step_out: The time step of the data entry
void m_initAgentDataTable() const;

/// @brief Dump simulation metadata into the database.
/// @details Ensures the `simulations` table exists and inserts one row with the
Expand Down Expand Up @@ -282,10 +292,12 @@ namespace dsf::mobility {
/// @param saveAverageStats If true, saves the average stats of the simulation (default is false)
/// @param saveStreetData If true, saves the street data (default is false)
/// @param saveTravelData If true, saves the travel data of the agents (default is false)
/// @param saveAgentData If true, saves the individual data of the agents (default is false)
void saveData(std::time_t const savingInterval,
bool const saveAverageStats = false,
bool const saveStreetData = false,
bool const saveTravelData = false);
bool const saveTravelData = false,
bool const saveAgentData = false);

/// @brief Update the paths of the itineraries based on the given weight function
/// @param throw_on_empty If true, throws an exception if an itinerary has an empty path (default is true)
Expand Down
13 changes: 11 additions & 2 deletions src/dsf/mobility/Street.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@
#include <spdlog/spdlog.h>

namespace dsf::mobility {
std::optional<
tbb::concurrent_unordered_map<Id,
std::vector<std::tuple<Id, std::time_t, std::time_t>>>>
Street::m_agentData = std::nullopt;

void Street::m_updateLaneMapping(int const nLanes) {
m_laneMapping.clear();
switch (nLanes) {
Expand Down Expand Up @@ -152,9 +157,13 @@ namespace dsf::mobility {
assert(!m_exitQueues[index].empty());
auto pAgent{m_exitQueues[index].extract_front()};
// Keep track of average speed
m_avgSpeeds.push_back(m_length /
(currentTime - m_agentsInsertionTimes[pAgent->id()]));
auto const insertionTime{m_agentsInsertionTimes[pAgent->id()]};
m_avgSpeeds.push_back(m_length / (currentTime - insertionTime));
m_agentsInsertionTimes.erase(pAgent->id());
if (m_agentData.has_value()) {
m_agentData->operator[](m_id).emplace_back(
pAgent->id(), insertionTime, currentTime);
}

if (m_counter.has_value() && m_counterPosition == CounterPosition::EXIT) {
++(*m_counter);
Expand Down
18 changes: 18 additions & 0 deletions src/dsf/mobility/Street.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
#include <unordered_map>
#include <vector>

#include <tbb/concurrent_unordered_map.h>

namespace dsf::mobility {

class AgentComparator {
Expand Down Expand Up @@ -58,6 +60,10 @@ namespace dsf::mobility {
std::optional<Counter> m_counter;
CounterPosition m_counterPosition{CounterPosition::EXIT};
double m_stationaryWeight{1.0};
static std::optional<tbb::concurrent_unordered_map<
Id,
std::vector<std::tuple<Id, std::time_t, std::time_t>>>>
m_agentData;

/// @brief Update the street's lane mapping
/// @param nLanes The street's number of lanes
Expand Down Expand Up @@ -87,6 +93,11 @@ namespace dsf::mobility {
Street(Street const&) = delete;
bool operator==(Street const& other) const;

static inline void acquireAgentData() {
m_agentData = tbb::concurrent_unordered_map<
Id,
std::vector<std::tuple<Id, std::time_t, std::time_t>>>();
};
/// @brief Set the street's lane mapping
/// @param laneMapping The street's lane mapping
void setLaneMapping(std::vector<Direction> const& laneMapping);
Expand Down Expand Up @@ -154,6 +165,13 @@ namespace dsf::mobility {
movingAgents() {
return m_movingAgents;
}
static inline auto agentData() {
if (!m_agentData.has_value()) {
throw std::runtime_error(
"agentData not initialized. Please call acquireAgentData() first.");
}
return std::move(*m_agentData);
}
/// @brief Get the number of of moving agents, i.e. agents not yet enqueued
/// @return std::size_t The number of moving agents
std::size_t nMovingAgents() const;
Expand Down
64 changes: 63 additions & 1 deletion test/mobility/Test_dynamics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1305,6 +1305,50 @@
std::filesystem::remove(testDbPath);
}

WHEN("We connect a database and configure saveData with agent data") {
dynamics.connectDataBase(testDbPath);
// Configure saving: interval=1, saveAgentData=true
dynamics.saveData(1, false, false, false, true);

// Evolve a few times to generate edge crossing events
for (int iter = 0; iter < 50; ++iter) {
dynamics.evolve(true);
}

THEN("The agent_data table is created with correct data") {
SQLite::Database db(testDbPath, SQLite::OPEN_READONLY);
SQLite::Statement query(db, "SELECT COUNT(*) FROM agent_data");
REQUIRE(query.executeStep());
CHECK(query.getColumn(0).getInt() >= 1);

SQLite::Statement schema(db, "PRAGMA table_info(agent_data)");
std::set<std::string> columns;
while (schema.executeStep()) {

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 14.4 rule Note test

MISRA 14.4 rule
columns.insert(schema.getColumn(1).getString());
}
CHECK(columns.count("id") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(columns.count("simulation_id") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(columns.count("agent_id") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(columns.count("edge_id") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(columns.count("time_step_in") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(columns.count("time_step_out") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule

SQLite::Statement rows(
db,
"SELECT simulation_id, agent_id, edge_id, time_step_in, time_step_out "
"FROM agent_data");
while (rows.executeStep()) {

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 14.4 rule Note test

MISRA 14.4 rule
CHECK(rows.getColumn(0).getInt() >= 0);
CHECK(rows.getColumn(1).getInt() >= 0);
CHECK(rows.getColumn(2).getInt() >= 0);
CHECK(rows.getColumn(3).getInt64() >= 0);
CHECK(rows.getColumn(4).getInt64() >= rows.getColumn(3).getInt64());
}
}

std::filesystem::remove(testDbPath);
}

WHEN("We connect a database and configure saveData with average stats") {
dynamics.connectDataBase(testDbPath);
// Configure saving: interval=1, saveAverageStats=true
Expand Down Expand Up @@ -1339,7 +1383,7 @@
WHEN("We configure saveData with all options enabled") {
dynamics.connectDataBase(testDbPath);
// Configure saving: interval=1, all data types enabled
dynamics.saveData(1, true, true, true);
dynamics.saveData(1, true, true, true, true);

// Add agents and evolve until some reach destination
dynamics.addAgents(10, AgentInsertionMethod::RANDOM);
Expand Down Expand Up @@ -1411,6 +1455,23 @@
CHECK(travelColumns.count("distance_m") == 1);
CHECK(travelColumns.count("travel_time_s") == 1);

// Check agent_data table
SQLite::Statement agentQuery(db, "SELECT COUNT(*) FROM agent_data");
REQUIRE(agentQuery.executeStep());
CHECK(agentQuery.getColumn(0).getInt() >= 1);

SQLite::Statement agentSchema(db, "PRAGMA table_info(agent_data)");
std::set<std::string> agentColumns;
while (agentSchema.executeStep()) {

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 14.4 rule Note test

MISRA 14.4 rule
agentColumns.insert(agentSchema.getColumn(1).getString());
}
CHECK(agentColumns.count("id") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(agentColumns.count("simulation_id") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(agentColumns.count("agent_id") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(agentColumns.count("edge_id") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(agentColumns.count("time_step_in") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule
CHECK(agentColumns.count("time_step_out") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule

// Check simulations table
SQLite::Statement simQuery(
db,
Expand All @@ -1436,6 +1497,7 @@
CHECK(simColumns.count("save_avg_stats") == 1);
CHECK(simColumns.count("save_road_data") == 1);
CHECK(simColumns.count("save_travel_data") == 1);
CHECK(simColumns.count("save_agent_data") == 1);

Check notice

Code scanning / Cppcheck (reported by Codacy)

MISRA 10.4 rule Note test

MISRA 10.4 rule

// Check edges table exists
SQLite::Statement edgesQuery(
Expand Down
Loading