Merge pull request #67 from rtbot-dev/RB-214

RB-214 Implement serialization of the state of an rtbot program

Author: rcarcasses

.bazelrc

@@ -19,20 +19,24 @@ test --notest_verbose_timeout_warnings
 build --action_env=EMCC_CFLAGS="-fexceptions  -Wno-c++11-narrowing -Wno-unused-const-variable"
 build --incompatible_enable_cc_toolchain_resolution
 
+build --enable_platform_specific_config
+build:macos --apple_crosstool_top=@local_config_apple_cc//:toolchain
+build:macos --crosstool_top=@local_config_apple_cc//:toolchain
+build:macos --host_crosstool_top=@local_config_apple_cc//:toolchain
 
 # -- platform specific --
-# linux
-build:linux --workspace_status_command=tools/bazel_workspace_status.sh
-build:linux --action_env=BAZEL_CXXOPTS="-std=c++17"
-
-# macos
-build:macos --workspace_status_command=tools/bazel_workspace_status.sh
-build:macos --cxxopt='-std=c++17'
+# unix
+build --workspace_status_command=tools/bazel_workspace_status.sh
+build --action_env=BAZEL_CXXOPTS="-std=c++17"
+build --cxxopt=-std=c++17
+build --repo_env=CC=clang
 
 # windows
 # MSVC
 build:windows --cxxopt='/std:c++17'
 # see https://bazel.build/configure/windows
 startup --windows_enable_symlinks
 
+# allow to use network while building
+build --sandbox_default_allow_network
 

.clangd

@@ -0,0 +1,21 @@
+# this file is added mainly for intellisense purposes
+If:
+  PathMatch: .*\.h
+CompileFlags:
+  Add:
+    - -DLOCAL
+    - -Wall
+    - -O2
+    - -Wextra
+    - -Wshadow
+    - -Wconversion
+    - -Wfloat-equal
+    - -Wno-unused-const-variable
+    - -Wno-sign-conversion
+    - -std=c++17
+    - -I./dist/bin/libs/api/proto
+    - -I./libs/api/include
+    - -I./libs/core/include
+    - -I./libs/std/include
+Index:
+  Background: Skip

.gitignore

@@ -43,3 +43,4 @@ venv/
 settings.json
 .firebase
 .envrc
+compile_commands.json

WORKSPACE

@@ -4,6 +4,42 @@ load("//:deps.bzl", "deps")
 
 deps()
 
+load(
+    "@build_bazel_apple_support//lib:repositories.bzl",
+    "apple_support_dependencies",
+)
+
+apple_support_dependencies()
+
+load("@bazel_features//:deps.bzl", "bazel_features_deps")
+
+bazel_features_deps()
+
+load("@rules_buf//buf:repositories.bzl", "rules_buf_dependencies", "rules_buf_toolchains")
+
+rules_buf_dependencies()
+
+rules_buf_toolchains(
+    version = "v1.32.1",
+)
+
+# rules_proto
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies")
+
+rules_proto_dependencies()
+
+load("@rules_proto//proto:setup.bzl", "rules_proto_setup")
+
+rules_proto_setup()
+
+load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
+
+rules_proto_toolchains()
+
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+
+protobuf_deps()
+
 load("@pybind11_bazel//:python_configure.bzl", "python_configure")
 
 python_configure(
@@ -27,7 +63,7 @@ load("@rules_nodejs//nodejs:repositories.bzl", "nodejs_register_toolchains")
 
 nodejs_register_toolchains(
     name = "nodejs",
-    node_version = '18.13.0',
+    node_version = "18.13.0",
 )
 
 load("@aspect_rules_js//npm:npm_import.bzl", "npm_translate_lock")

deps.bzl

@@ -115,12 +115,12 @@ def deps():
         strip_prefix = "bazel-lib-1.34.0",
         url = "https://github.com/aspect-build/bazel-lib/releases/download/v1.34.0/bazel-lib-v1.34.0.tar.gz",
     )
-    
+
     http_archive(
         name = "emsdk",
-        strip_prefix = "emsdk-3.1.34/bazel",
-        sha256 = "e03ed88f9f8075c78f49c43241e3586798300593e7ea873f766e9403e662516c",
-        url = "https://github.com/emscripten-core/emsdk/archive/refs/tags/3.1.34.tar.gz",
+        sha256 = "97f21c32c0c1eecb963d19d1cacf58538086ce28eb28274993ab21d3673b5c29",
+        strip_prefix = "emsdk-3.1.68/bazel",
+        url = "https://github.com/emscripten-core/emsdk/archive/refs/tags/3.1.68.tar.gz",
     )
 
     http_archive(
@@ -142,3 +142,42 @@ def deps():
         sha256 = "aeec52381284ec3752505a220d36096954c869da4573c2e1df3642d2f0a4aac6",
         strip_prefix = "benchmark-1.7.1",
     )
+
+    http_archive(
+        name = "build_bazel_apple_support",
+        sha256 = "b53f6491e742549f13866628ddffcc75d1f3b2d6987dc4f14a16b242113c890b",
+        url = "https://github.com/bazelbuild/apple_support/releases/download/1.17.1/apple_support.1.17.1.tar.gz",
+    )
+
+    http_archive(
+        name = "com_google_protobuf",
+        # notice that when we change the version here we also have to change the version as well for
+        # the buf plugin buf.build/protocolbuffers/cpp in the files:
+        #  - libs/api/proto/buf.gen.yaml
+        url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protobuf-27.3.zip",
+        strip_prefix = "protobuf-27.3",
+        sha256 = "a49147217f69e8d19aab0cc5c0059d6201261f5cb62145f8ab4ac8b94e7ffa86",
+    )
+
+    http_archive(
+        name = "rules_buf",
+        integrity = "sha256-Hr64Q/CaYr0E3ptAjEOgdZd1yc+cBjp7OG1wzuf3DIs=",
+        strip_prefix = "rules_buf-0.3.0",
+        urls = [
+            "https://github.com/bufbuild/rules_buf/archive/refs/tags/v0.3.0.zip",
+        ],
+    )
+
+    http_archive(
+        name = "rules_proto",
+        sha256 = "6fb6767d1bef535310547e03247f7518b03487740c11b6c6adb7952033fe1295",
+        strip_prefix = "rules_proto-6.0.2",
+        url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.2/rules_proto-6.0.2.tar.gz",
+    )
+
+    http_archive(
+        name = "protovalidate",
+        sha256 = "6e49ffdf4502d32472b568cf1b279d113cc8548ffb19134f800f903d5148184b",
+        url = "https://github.com/bufbuild/protovalidate/archive/refs/tags/v0.8.2.zip",
+        strip_prefix = "protovalidate-0.8.2",
+    )

libs/api/BUILD.bazel

@@ -1,9 +1,8 @@
 load("@aspect_bazel_lib//lib:copy_file.bzl", "copy_file")
 load("@rules_cc//cc:defs.bzl", "cc_library")
 load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary")
-load("@aspect_rules_js//npm:defs.bzl", "npm_package")
+load("@aspect_rules_js//npm:defs.bzl", "npm_package", "stamped_package_json")
 load("//tools/generator:generator.bzl", "rtbot_generate")
-load("@aspect_rules_js//npm:defs.bzl", "stamped_package_json")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -38,8 +37,9 @@ cc_library(
     includes = ["include"],
     visibility = ["//visibility:public"],
     deps = [
-        "//libs/finance:rtbot-finance",
+        "//libs/api/proto",
         "//libs/core:rtbot",
+        "//libs/finance:rtbot-finance",
         "//libs/std:rtbot-std",
         "@json-cpp//:lib",
         "@json-schema-validator//:lib",
@@ -50,11 +50,13 @@ cc_library(
 BASE_LINKOPTS = [
     "-O3",
     "-lembind",  # Enable embind
-    "-s MODULARIZE",
-    "--embind-emit-tsd",
-    "-s ALLOW_MEMORY_GROWTH",
+    "-sMODULARIZE",
+    # TODO: enable typescript emit, although this is non blocking
+    #"--emit-tsd",
+    "-sALLOW_MEMORY_GROWTH",
     #"-s MEMORY64=2",
-    "-s NO_DISABLE_EXCEPTION_CATCHING",
+    "-sWASM_BIGINT",
+    "-sNO_DISABLE_EXCEPTION_CATCHING",
     #"-s EXPORT_ES6=1",
 ]
 
@@ -101,14 +103,14 @@ stamped_package_json(
 npm_package(
     name = "js",
     srcs = [
-        ":package",
         ":declarations",
+        ":package",
         ":rtbot-wasm",
     ],
+    # this is needed if we want to use this package in another bazel workspace
+    include_external_repositories = ["**"],
     package = "@rtbot-dev/wasm",
     replace_prefixes = {
-      "rtbot-wasm": "."
+        "rtbot-wasm": ".",
     },
-    # this is needed if we want to use this package in another bazel workspace
-    include_external_repositories = ["**"]
 )

libs/api/include/rtbot/FactoryOp.h

@@ -16,7 +16,7 @@ using namespace std;
 
 class FactoryOp {
   map<string, Program> programs;
-  map<string, map<string, vector<Message<uint64_t, double>>>> messageBuffer;
+  ProgramMessage<uint64_t, double> messageBuffer;
 
  public:
   FactoryOp();
@@ -54,8 +54,13 @@ class FactoryOp {
 
   static Op_ptr<uint64_t, double> readOp(string const& json_string);
   static string writeOp(Op_ptr<uint64_t, double> const& op);
+
   static Program createProgram(string const& json_string) { return Program(json_string); }
 
+  Bytes serialize(string const& programId);
+
+  string createProgram(string const& id, Bytes const& bytes);
+
   string createProgram(string const& id, string const& json_program);
 
   bool deleteProgram(string const& id) { return (programs.erase(id) == 1) ? true : false; }
@@ -66,7 +71,7 @@ class FactoryOp {
     return true;
   }
 
-  map<string, map<string, vector<Message<uint64_t, double>>>> processMessageBuffer(const string& apId) {
+  ProgramMessage<uint64_t, double> processMessageBuffer(const string& apId) {
     if (this->programs.count(apId) == 0) throw runtime_error("Program " + apId + " was not found");
     if (this->messageBuffer.count(apId) > 0) {
       if (!this->messageBuffer.at(apId).empty()) {
@@ -79,7 +84,7 @@ class FactoryOp {
     return {};
   }
 
-  map<string, map<string, vector<Message<uint64_t, double>>>> processMessageBufferDebug(const string& apId) {
+  ProgramMessage<uint64_t, double> processMessageBufferDebug(const string& apId) {
     if (this->programs.count(apId) == 0) throw runtime_error("Program " + apId + " was not found");
     if (this->messageBuffer.count(apId) > 0) {
       if (!this->messageBuffer.at(apId).empty()) {
@@ -107,15 +112,15 @@ class FactoryOp {
     return this->programs.at(apId).getProgramOutputFilter();
   }
 
-  map<string, map<string, vector<Message<uint64_t, double>>>> processMessageMap(
-      string const& apId, const map<string, vector<Message<uint64_t, double>>>& messagesMap) {
+  ProgramMessage<uint64_t, double> processMessageMap(string const& apId,
+                                                     const OperatorMessage<uint64_t, double>& messagesMap) {
     auto it = programs.find(apId);
     if (it == programs.end()) return {};
     return it->second.receive(messagesMap);
   }
 
-  map<string, map<string, vector<Message<uint64_t, double>>>> processMessageMapDebug(
-      string const& apId, const map<string, vector<Message<uint64_t, double>>>& messagesMap) {
+  ProgramMessage<uint64_t, double> processMessageMapDebug(string const& apId,
+                                                          const OperatorMessage<uint64_t, double>& messagesMap) {
     auto it = programs.find(apId);
     if (it == programs.end()) return {};
     return it->second.receiveDebug(messagesMap);