feat(api): full program serialization supported

Author: rcarcasses

libs/api/include/rtbot/FactoryOp.h

@@ -57,15 +57,9 @@ class FactoryOp {
 
   static Program createProgram(string const& json_string) { return Program(json_string); }
 
-  Bytes collect(string const& programId) {
-    if (this->programs.count(programId) == 0) throw runtime_error("Program " + programId + " was not found");
-    return programs.at(programId).collect();
-  }
+  Bytes serialize(string const& programId);
 
-  void restore(string const& programId, Bytes const& bytes) {
-    if (this->programs.count(programId) == 0) throw runtime_error("Program " + programId + " was not found");
-    programs.at(programId).restore(bytes);
-  }
+  string createProgram(string const& id, Bytes const& bytes);
 
   string createProgram(string const& id, string const& json_program);
 

libs/api/include/rtbot/Program.h

@@ -4,6 +4,7 @@
 #include <map>
 #include <memory>
 #include <optional>
+#include <string>
 
 #include "rtbot/Operator.h"
 
@@ -13,41 +14,46 @@ using namespace std;
 
 struct Program {
  private:
+  string program_json;
   map<string, Op_ptr<uint64_t, double>> all_op;  // from id to operator
   string entryOperator;                          // id of the entry operator
   map<string, vector<string>> outputFilter;
 
  public:
   explicit Program(string const& json_string);
+  explicit Program(Bytes const& bytes) {
+    // create an iterator
+    Bytes::const_iterator bytes_it = bytes.begin();
+    // read the size of the program json
+    uint64_t size = *reinterpret_cast<const uint64_t*>(&(*bytes_it));
+    bytes_it += sizeof(size);
+    // read the program json
+    program_json = string(bytes_it, bytes_it + size);
+    bytes_it += size;
+
+    // init the program
+    init();
+
+    // load the state of the operators
+    while (bytes_it != bytes.end()) {
+      // read the size of the operator id
+      size = *reinterpret_cast<const uint64_t*>(&(*bytes_it));
+      bytes_it += sizeof(size);
+      string opId(bytes_it, bytes_it + size);
+      bytes_it += size;
+
+      // read the operator state
+      all_op.at(opId)->restore(bytes_it);
+    }
+  }
 
   Program(Program const&) = delete;
   void operator=(Program const&) = delete;
   Program(Program&& other) = default;
 
-  Bytes collect() {
-    Bytes bytes;
-    for (auto it = this->all_op.begin(); it != this->all_op.end(); ++it) {
-      Bytes opBytes = it->second->collect();
-      bytes.insert(bytes.end(), opBytes.begin(), opBytes.end());
-    }
-
-    return bytes;
-  }
-
-  void restore(Bytes const& bytes) {
-    Bytes::const_iterator bytes_it = bytes.begin();
-    for (auto it = this->all_op.begin(); it != this->all_op.end(); ++it) {
-      it->second->restore(bytes_it);
-    }
-  }
+  void init();
 
-  string debug() {
-    string toReturn;
-    for (auto it = this->all_op.begin(); it != this->all_op.end(); ++it) {
-      toReturn += "\n  " + it->second->debug("");
-    }
-    return toReturn;
-  }
+  Bytes serialize();
 
   string getProgramEntryOperatorId() { return entryOperator; }
 

libs/api/include/rtbot/bindings.h

@@ -13,9 +13,8 @@
 using namespace std;
 using namespace rtbot;
 
-Bytes collect(string const& programId);
-// after creating a program, we can restore it with the a collected state
-void restore(string const& programId, Bytes const& bytes);
+Bytes serializeProgram(string const& programId);
+void createProgram(string const& programId, Bytes const& bytes);
 
 ProgramMessage<uint64_t, double> processMessageMapNative(string const& programId,
                                                          const OperatorMessage<uint64_t, double>& messagesMap);

libs/api/src/FactoryOp.cpp

@@ -401,9 +401,20 @@ void from_json(const json& j, Pipeline<T, V>& p) {
 
 /* Operators serialization - deserialization - end */
 
+Bytes FactoryOp::serialize(string const& programId) {
+  if (this->programs.count(programId) == 0) throw runtime_error("Program " + programId + " was not found");
+  return programs.at(programId).serialize();
+}
+
+string FactoryOp::createProgram(string const& id, Bytes const& bytes) {
+  if (this->programs.count(id) > 0) throw runtime_error("Program " + id + " already exists");
+  programs.emplace(id, Program(bytes));
+  return id;
+}
+
 string FactoryOp::createProgram(string const& id, string const& json_program) {
   try {
-    programs.emplace(id, createProgram(json_program));
+    programs.emplace(id, Program(json_program));
     return "";
   } catch (const json::parse_error& e) {
     // output exception information

libs/api/src/Program.cpp

@@ -34,8 +34,12 @@ inline void from_json(const json& j, OpConnection& p) {
 }
 
 Program::Program(const string& json_string) {
-  auto j = json::parse(json_string);
+  program_json = json_string;
+  init();
+}
 
+void Program::init() {
+  auto j = json::parse(program_json);
   this->entryOperator = j.value("entryOperator", "");
 
   if (this->entryOperator.empty()) throw runtime_error("Entry operator was not specified");
@@ -72,6 +76,31 @@ Program::Program(const string& json_string) {
   }
 }
 
+Bytes Program::serialize() {
+  Bytes bytes;
+
+  // save the program json definition, starting with the size of the string
+  uint64_t size = program_json.size();
+  bytes.insert(bytes.end(), reinterpret_cast<const unsigned char*>(&size),
+               reinterpret_cast<const unsigned char*>(&size) + sizeof(size));
+  bytes.insert(bytes.end(), program_json.begin(), program_json.end());
+
+  // save the state of the operators
+  for (auto& [opId, op] : all_op) {
+    // save the size of the operator id
+    size = opId.size();
+    bytes.insert(bytes.end(), reinterpret_cast<const unsigned char*>(&size),
+                 reinterpret_cast<const unsigned char*>(&size) + sizeof(size));
+    bytes.insert(bytes.end(), opId.begin(), opId.end());
+
+    // save the operator state
+    Bytes opBytes = op->collect();
+    bytes.insert(bytes.end(), opBytes.begin(), opBytes.end());
+  }
+
+  return bytes;
+}
+
 string Program::getProgram() {
   string out = "";
   for (auto it = this->all_op.begin(); it != this->all_op.end(); ++it) {

libs/api/src/bindings.cpp

@@ -40,9 +40,9 @@ void from_json(const json& j, Message<T, V>& p) {
 
 }  // namespace rtbot
 
-Bytes collect(string const& programId) { return factory.collect(programId); }
+Bytes serializeProgram(string const& programId) { return factory.serialize(programId); }
 
-void restore(string const& programId, Bytes const& bytes) { factory.restore(programId, bytes); }
+void createProgram(string const& programId, Bytes const& bytes) { factory.createProgram(programId, bytes); }
 
 ProgramMessage<uint64_t, double> processMessageMapNative(string const& programId,
                                                          const OperatorMessage<uint64_t, double>& messagesMap) {

libs/api/test/test_program_collect_restore.cpp

@@ -46,10 +46,9 @@ TEST_CASE("Program collect and restore") {
       auto output = program.receive(messagesMap);
     }
 
-    Bytes bytes = program.collect();
+    Bytes bytes = program.serialize();
     // create a new program
-    auto program2 = FactoryOp::createProgram(json.dump().c_str());
-    program2.restore(bytes);
+    auto program2 = Program(bytes);
 
     // send the second half of the data
     for (int i = half; i < v.size(); i++) {