From ac2cedeb138780ef9219763259460e105de3cb06 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 30 Jun 2025 09:34:54 +0100 Subject: [PATCH 01/10] Add cluster support (#228) ## Release notes: usage and product changes Introduce cluster support for TypeDB 3.x, featuring server replicas and the new version message. ## Implementation Remove database replicas. Instead, add server replicas (by extending the usual `Server` message) with a similar set of fields. Instead of adding a `is_primary` flag, introduce an extensible enum `ReplicaType` for more granular split between primary and supporting nodes. To reduce the network overhead, this new information is provided in the initial server's connection response. --- proto/BUILD | 1 + proto/connection.proto | 9 ++------- proto/database.proto | 20 ++++---------------- proto/error.proto | 2 +- proto/server.proto | 20 +++++++++++++++++++- proto/typedb-service.proto | 3 +++ 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/proto/BUILD b/proto/BUILD index 1eaa241..d6e31ac 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -66,6 +66,7 @@ proto_library( deps = [ ":authentication-proto", ":database-proto", + ":server-proto", ":version-proto" ], ) diff --git a/proto/connection.proto b/proto/connection.proto index d288eb4..b083d7f 100644 --- a/proto/connection.proto +++ b/proto/connection.proto @@ -6,35 +6,30 @@ syntax = "proto3"; import "proto/authentication.proto"; import "proto/database.proto"; +import "proto/server.proto"; import "proto/version.proto"; package typedb.protocol; message Connection { - message Open { message Req { Version version = 1; ExtensionVersion extension_version = 5; string driver_lang = 2; string driver_version = 3; - Authentication.Token.Create.Req authentication = 4; } message Res { uint64 server_duration_millis = 1; ConnectionID connection_id = 2; - - // pre-send all databases and replica info - DatabaseManager.All.Res databases_all = 3; - + ServerManager.All.Res servers_all = 3; Authentication.Token.Create.Res authentication = 4; } } } -// Connection ID and Token are expected in all message metadata message ConnectionID { bytes id = 1; } diff --git a/proto/database.proto b/proto/database.proto index eefa6d4..4694092 100644 --- a/proto/database.proto +++ b/proto/database.proto @@ -16,14 +16,14 @@ message DatabaseManager { } message Res { - DatabaseReplicas database = 1; + Database database = 1; } } message All { message Req {} message Res { - repeated DatabaseReplicas databases = 1; + repeated Database databases = 1; } } @@ -43,7 +43,7 @@ message DatabaseManager { } message Res { - DatabaseReplicas database = 1; + Database database = 1; } } @@ -58,20 +58,8 @@ message DatabaseManager { } } -message DatabaseReplicas { - - string name = 1; - repeated Replica replicas = 2; - - message Replica { - string address = 1; - bool primary = 2; - bool preferred = 3; - int64 term = 4; - } -} - message Database { + string name = 1; message Schema { message Req { diff --git a/proto/error.proto b/proto/error.proto index eaa7a55..7543d28 100644 --- a/proto/error.proto +++ b/proto/error.proto @@ -7,7 +7,7 @@ syntax = "proto3"; package typedb.protocol; // This is an emulation of the google ErrorDetails message. Generally, ErrorDetails are submitted via the GRPC error -// mechanism, but a manual error sending is required in streams +// mechanism, but a manual error sending is useful to differentiate error levels in streams message Error { string error_code = 1; string domain = 2; diff --git a/proto/server.proto b/proto/server.proto index 40d2c98..b1c0cc3 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -7,7 +7,6 @@ syntax = "proto3"; package typedb.protocol; message ServerManager { - message All { message Req {} message Res { @@ -18,4 +17,23 @@ message ServerManager { message Server { string address = 1; + optional ReplicaStatus replica_status = 2; + + message ReplicaStatus { + ReplicaType replica_type = 1; + int64 term = 2; + + enum ReplicaType { + Primary = 0; + Secondary = 1; + } + } + + message Version { + message Req {} + message Res { + string distribution = 1; + string version = 2; + } + } } diff --git a/proto/typedb-service.proto b/proto/typedb-service.proto index 95f0c23..79e6936 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb-service.proto @@ -25,6 +25,9 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + // Server API + rpc server_version (Server.Version.Req) returns (Server.Version.Res); + // User Manager API rpc users_get (UserManager.Get.Req) returns (UserManager.Get.Res); rpc users_all (UserManager.All.Req) returns (UserManager.All.Res); From 5f14a1133e0422a9580048d8032b1b15a6035c1e Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Thu, 3 Jul 2025 16:20:59 +0100 Subject: [PATCH 02/10] Add server register/deregister endpoint --- proto/server.proto | 16 ++++++++++++++++ proto/typedb-service.proto | 2 ++ 2 files changed, 18 insertions(+) diff --git a/proto/server.proto b/proto/server.proto index b1c0cc3..7dd1914 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -13,6 +13,21 @@ message ServerManager { repeated Server servers = 1; } } + + message Register { + message Req { + uint64 replica_id = 1; + string address = 2; + } + message Res {} + } + + message Deregister { + message Req { + uint64 replica_id = 1; + } + message Res {} + } } message Server { @@ -20,6 +35,7 @@ message Server { optional ReplicaStatus replica_status = 2; message ReplicaStatus { + uint64 replica_id = 3; ReplicaType replica_type = 1; int64 term = 2; diff --git a/proto/typedb-service.proto b/proto/typedb-service.proto index 79e6936..3b401c7 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb-service.proto @@ -24,6 +24,8 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + rpc servers_register(ServerManager.Register.Req) returns (ServerManager.Register.Res); + rpc servers_deregister(ServerManager.Deregister.Req) returns (ServerManager.Deregister.Res); // Server API rpc server_version (Server.Version.Req) returns (Server.Version.Res); From 10779b9bc1b85a1b6554491673b5221c9f464d09 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Thu, 3 Jul 2025 16:24:49 +0100 Subject: [PATCH 03/10] Update the numbering of ReplicaStatus fields --- proto/server.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proto/server.proto b/proto/server.proto index 7dd1914..980731f 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -35,9 +35,9 @@ message Server { optional ReplicaStatus replica_status = 2; message ReplicaStatus { - uint64 replica_id = 3; - ReplicaType replica_type = 1; - int64 term = 2; + uint64 replica_id = 1; + ReplicaType replica_type = 2; + int64 term = 3; enum ReplicaType { Primary = 0; From 21afc68ed5d1e68e9329ef121bc1e401a3b91225 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Fri, 4 Jul 2025 09:29:27 +0100 Subject: [PATCH 04/10] Change type of term from in64 to uint64 --- proto/server.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/server.proto b/proto/server.proto index 980731f..07afb72 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -37,7 +37,7 @@ message Server { message ReplicaStatus { uint64 replica_id = 1; ReplicaType replica_type = 2; - int64 term = 3; + uint64 term = 3; enum ReplicaType { Primary = 0; From 1508d02f5989918d78ddb6e60b8444913e583c64 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Wed, 9 Jul 2025 13:51:43 +0100 Subject: [PATCH 05/10] Move raft peering protocol to this repo --- grpc/rust/BUILD | 44 +++++++++++++++++--------------- grpc/rust/build.rs | 2 ++ proto/BUILD | 29 +++++++++++++++------ proto/raft-peering-service.proto | 9 +++++++ proto/raft-peering.proto | 12 +++++++++ 5 files changed, 67 insertions(+), 29 deletions(-) create mode 100644 proto/raft-peering-service.proto create mode 100644 proto/raft-peering.proto diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index d3e84fe..6d9fbdd 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -10,64 +10,66 @@ load("@typedb_dependencies//builder/proto_grpc/rust:compile.bzl", "rust_tonic_co rust_tonic_compile( name = "typedb_protocol_src", - packages = ["typedb.protocol"], srcs = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", + "//proto:analyze-proto", + "//proto:analyzed-conjunction-proto", "//proto:answer-proto", + "//proto:authentication-proto", "//proto:concept-proto", "//proto:connection-proto", - "//proto:authentication-proto", + "//proto:database-proto", + "//proto:error-proto", + "//proto:migration-proto", "//proto:options-proto", "//proto:query-proto", + "//proto:raft-peering-proto", + "//proto:raft-peering-service", + "//proto:server-proto", "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", + "//proto:typedb-service", + "//proto:user-proto", "//proto:version-proto", - ] + ], + packages = ["typedb.protocol"], ) rust_library( name = "typedb_protocol", - crate_root = ":typedb_protocol_src", srcs = [ ":typedb_protocol_src", ], - deps = [ - "@crates//:tonic", - "@crates//:prost", - ], + crate_root = ":typedb_protocol_src", tags = [ "crate-name=typedb-protocol", ], visibility = ["//visibility:public"], + deps = [ + "@crates//:prost", + "@crates//:tonic", + ], ) assemble_crate( name = "assemble_crate", - target = ":typedb_protocol", description = "TypeDB Protocol", homepage = "https://github.com/typedb/typedb-protocol", - readme_file = "//:README.md", license = "MPL-2.0", license_file = "//:LICENSE", + readme_file = "//:README.md", repository = "https://github.com/typedb/typedb-protocol", + target = ":typedb_protocol", ) deploy_crate( name = "deploy_crate", - target = ":assemble_crate", + release = deployment["crate"]["release"], snapshot = deployment["crate"]["snapshot"], - release = deployment["crate"]["release"] + target = ":assemble_crate", ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/grpc/rust/build.rs b/grpc/rust/build.rs index 8ea60f8..ad54f3c 100644 --- a/grpc/rust/build.rs +++ b/grpc/rust/build.rs @@ -20,6 +20,8 @@ fn main() -> std::io::Result<()> { "../../proto/typedb-service.proto", "../../proto/user.proto", "../../proto/version.proto", + "../../proto/raft-peering.proto", + "../../proto/raft-peering-service.proto", ]; tonic_build::configure() diff --git a/proto/BUILD b/proto/BUILD index d6e31ac..fbdd41b 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -6,20 +6,33 @@ package(default_visibility = ["//visibility:public"]) load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") +proto_library( + name = "raft-peering-service", + srcs = [":raft-peering-service.proto"], + deps = [ + ":raft-peering-proto", + ], +) + proto_library( name = "typedb-service", srcs = [":typedb-service.proto"], deps = [ - ":connection-proto", ":authentication-proto", - ":server-proto", - ":user-proto", + ":connection-proto", ":database-proto", - ":transaction-proto", ":migration-proto", + ":server-proto", + ":transaction-proto", + ":user-proto", ], ) +proto_library( + name = "raft-peering-proto", + srcs = [":raft-peering.proto"], +) + proto_library( name = "server-proto", srcs = [":server.proto"], @@ -67,7 +80,7 @@ proto_library( ":authentication-proto", ":database-proto", ":server-proto", - ":version-proto" + ":version-proto", ], ) @@ -103,7 +116,7 @@ proto_library( ":error-proto", ":options-proto", ":query-proto", - ] + ], ) proto_library( @@ -146,12 +159,12 @@ filegroup( "query.proto", "transaction.proto", "version.proto", - ] + ], ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/proto/raft-peering-service.proto b/proto/raft-peering-service.proto new file mode 100644 index 0000000..e87a8e0 --- /dev/null +++ b/proto/raft-peering-service.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +import "proto/raft-peering.proto"; + +package typedb.protocol; + +service RaftPeering { + rpc Peering(PeeringRequest) returns (PeeringResponse); +} \ No newline at end of file diff --git a/proto/raft-peering.proto b/proto/raft-peering.proto new file mode 100644 index 0000000..33a1add --- /dev/null +++ b/proto/raft-peering.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package typedb.protocol; + +message PeeringRequest { + uint64 from_id = 1; + string from_address = 2; + bytes message = 3; +} + +message PeeringResponse { +} From 3c5d8cbe37145a4d6089015a00b91ee48c7a6091 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 21 Jul 2025 16:46:09 +0100 Subject: [PATCH 06/10] Add server_status message to the raft peering service (#229) ## Release notes: usage and product changes Add server_status message to the raft peering service and rename the old raft peering messages to adjust to the existing naming conventions. ## Implementation --- proto/BUILD | 3 +++ proto/raft-peering-service.proto | 7 ++++++- proto/raft-peering.proto | 27 ++++++++++++++++++++++----- proto/server.proto | 3 ++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/proto/BUILD b/proto/BUILD index fbdd41b..4be34bf 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -31,6 +31,9 @@ proto_library( proto_library( name = "raft-peering-proto", srcs = [":raft-peering.proto"], + deps = [ + ":server-proto", + ], ) proto_library( diff --git a/proto/raft-peering-service.proto b/proto/raft-peering-service.proto index e87a8e0..82301fd 100644 --- a/proto/raft-peering-service.proto +++ b/proto/raft-peering-service.proto @@ -1,3 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + syntax = "proto3"; import "proto/raft-peering.proto"; @@ -5,5 +9,6 @@ import "proto/raft-peering.proto"; package typedb.protocol; service RaftPeering { - rpc Peering(PeeringRequest) returns (PeeringResponse); + rpc peering (Peering.Req) returns (Peering.Res); + rpc server_status (ServerStatus.Req) returns (ServerStatus.Res); } \ No newline at end of file diff --git a/proto/raft-peering.proto b/proto/raft-peering.proto index 33a1add..cc6411e 100644 --- a/proto/raft-peering.proto +++ b/proto/raft-peering.proto @@ -1,12 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + syntax = "proto3"; package typedb.protocol; -message PeeringRequest { - uint64 from_id = 1; - string from_address = 2; - bytes message = 3; +import "proto/server.proto"; + +// TODO: Rename to Replication +message Peering { + message Req { + uint64 from_id = 1; + string from_address = 2; + bytes message = 3; + } + + message Res {} } -message PeeringResponse { +message ServerStatus { + message Req {} + + message Res { + string connection_address = 1; + Server.ReplicaStatus replica_status = 2; + } } diff --git a/proto/server.proto b/proto/server.proto index 07afb72..e29d01e 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -41,7 +41,8 @@ message Server { enum ReplicaType { Primary = 0; - Secondary = 1; + Candidate = 1; + Secondary = 2; } } From 96fc29ac070536cf3785369e3fb8586de65f47cf Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Thu, 14 Aug 2025 14:47:42 +0100 Subject: [PATCH 07/10] Split protocol into 3 separate services (#230) Split the protocol into 3 services: * TypeDBService (contains common TypeDB logic): server, database, user, and transaction management, etc. * TypeDBClusteringService (contains TypeDB logic regarding clustering): replica registration, server status. * RaftService (contains Raft logic): replication operations. Split the old RaftPeeringService into TypeDBClusteringService and RaftService. Move separate services and their implementation files into separate subpackages. Rename methods. Introduce `ServersGet` method for single server status retrieval. Introduce `registration` method to verify the replica registration operation before submitting it to Raft. --- grpc/java/BUILD | 34 ++-- grpc/nodejs/BUILD | 34 ++-- grpc/rust/BUILD | 36 +++-- grpc/rust/build.rs | 36 +++-- proto/BUILD | 152 +----------------- proto/raft_service/BUILD | 28 ++++ proto/raft_service/raft_service.proto | 14 ++ .../replication.proto} | 14 +- proto/typedb_clustering_service/BUILD | 30 ++++ .../typedb_clustering.proto | 25 +++ .../typedb_clustering_service.proto} | 8 +- proto/typedb_service/BUILD | 134 +++++++++++++++ proto/{ => typedb_service}/analyze.proto | 8 +- .../analyzed-conjunction.proto | 4 +- proto/{ => typedb_service}/answer.proto | 2 +- .../{ => typedb_service}/authentication.proto | 0 proto/{ => typedb_service}/concept.proto | 0 proto/{ => typedb_service}/connection.proto | 6 +- proto/{ => typedb_service}/database.proto | 2 +- proto/{ => typedb_service}/error.proto | 0 proto/{ => typedb_service}/migration.proto | 2 +- proto/{ => typedb_service}/options.proto | 0 proto/{ => typedb_service}/query.proto | 8 +- proto/{ => typedb_service}/server.proto | 7 + proto/{ => typedb_service}/transaction.proto | 8 +- .../typedb_service.proto} | 15 +- proto/{ => typedb_service}/user.proto | 0 27 files changed, 355 insertions(+), 252 deletions(-) create mode 100644 proto/raft_service/BUILD create mode 100644 proto/raft_service/raft_service.proto rename proto/{raft-peering.proto => raft_service/replication.proto} (61%) create mode 100644 proto/typedb_clustering_service/BUILD create mode 100644 proto/typedb_clustering_service/typedb_clustering.proto rename proto/{raft-peering-service.proto => typedb_clustering_service/typedb_clustering_service.proto} (66%) create mode 100644 proto/typedb_service/BUILD rename proto/{ => typedb_service}/analyze.proto (95%) rename proto/{ => typedb_service}/analyzed-conjunction.proto (97%) rename proto/{ => typedb_service}/answer.proto (97%) rename proto/{ => typedb_service}/authentication.proto (100%) rename proto/{ => typedb_service}/concept.proto (100%) rename proto/{ => typedb_service}/connection.proto (84%) rename proto/{ => typedb_service}/database.proto (97%) rename proto/{ => typedb_service}/error.proto (100%) rename proto/{ => typedb_service}/migration.proto (98%) rename proto/{ => typedb_service}/options.proto (100%) rename proto/{ => typedb_service}/query.proto (92%) rename proto/{ => typedb_service}/server.proto (91%) rename proto/{ => typedb_service}/transaction.proto (93%) rename proto/{typedb-service.proto => typedb_service/typedb_service.proto} (86%) rename proto/{ => typedb_service}/user.proto (100%) diff --git a/grpc/java/BUILD b/grpc/java/BUILD index cf6f231..a793641 100644 --- a/grpc/java/BUILD +++ b/grpc/java/BUILD @@ -9,22 +9,26 @@ load("@rules_proto_grpc//java:defs.bzl", "java_grpc_library") java_grpc_library( name = "typedb-protocol", protos = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", - "//proto:answer-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:authentication-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ], # TypeDB Core bundles JARs by maven coordinate, we can remove this when Core is rewritten in Rust tags = ["maven_coordinates=com.typedb:typedb-protocol:{pom_version}"], diff --git a/grpc/nodejs/BUILD b/grpc/nodejs/BUILD index c9b8827..8e00bae 100644 --- a/grpc/nodejs/BUILD +++ b/grpc/nodejs/BUILD @@ -25,22 +25,26 @@ protoc_gen_ts_bin.protoc_gen_ts_binary( ts_grpc_compile( name = "typedb-protocol-src", deps = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", - "//proto:answer-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:authentication-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ] ) diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index 6d9fbdd..200ec89 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -11,24 +11,26 @@ load("@typedb_dependencies//builder/proto_grpc/rust:compile.bzl", "rust_tonic_co rust_tonic_compile( name = "typedb_protocol_src", srcs = [ - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", - "//proto:answer-proto", - "//proto:authentication-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:database-proto", - "//proto:error-proto", - "//proto:migration-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:raft-peering-proto", - "//proto:raft-peering-service", - "//proto:server-proto", - "//proto:transaction-proto", - "//proto:typedb-service", - "//proto:user-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ], packages = ["typedb.protocol"], ) diff --git a/grpc/rust/build.rs b/grpc/rust/build.rs index ad54f3c..8ba15f0 100644 --- a/grpc/rust/build.rs +++ b/grpc/rust/build.rs @@ -4,24 +4,26 @@ fn main() -> std::io::Result<()> { let protos = vec![ - "../../proto/analyze.proto", - "../../proto/answer.proto", - "../../proto/authentication.proto", - "../../proto/concept.proto", - "../../proto/connection.proto", - "../../proto/analyzed-conjunction.proto", - "../../proto/database.proto", - "../../proto/error.proto", - "../../proto/migration.proto", - "../../proto/options.proto", - "../../proto/query.proto", - "../../proto/server.proto", - "../../proto/transaction.proto", - "../../proto/typedb-service.proto", - "../../proto/user.proto", + "../../proto/raft_service/replication.proto", + "../../proto/raft_service/raft_service.proto", + "../../proto/typedb_clustering_service/typedb_clustering.proto", + "../../proto/typedb_clustering_service/typedb_clustering_service.proto", + "../../proto/typedb_service/analyze.proto", + "../../proto/typedb_service/analyzed-conjunction.proto", + "../../proto/typedb_service/answer.proto", + "../../proto/typedb_service/authentication.proto", + "../../proto/typedb_service/concept.proto", + "../../proto/typedb_service/connection.proto", + "../../proto/typedb_service/database.proto", + "../../proto/typedb_service/error.proto", + "../../proto/typedb_service/migration.proto", + "../../proto/typedb_service/options.proto", + "../../proto/typedb_service/query.proto", + "../../proto/typedb_service/server.proto", + "../../proto/typedb_service/transaction.proto", + "../../proto/typedb_service/typedb_service.proto", + "../../proto/typedb_service/user.proto", "../../proto/version.proto", - "../../proto/raft-peering.proto", - "../../proto/raft-peering-service.proto", ]; tonic_build::configure() diff --git a/proto/BUILD b/proto/BUILD index 4be34bf..3b6d63b 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -6,142 +6,6 @@ package(default_visibility = ["//visibility:public"]) load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") -proto_library( - name = "raft-peering-service", - srcs = [":raft-peering-service.proto"], - deps = [ - ":raft-peering-proto", - ], -) - -proto_library( - name = "typedb-service", - srcs = [":typedb-service.proto"], - deps = [ - ":authentication-proto", - ":connection-proto", - ":database-proto", - ":migration-proto", - ":server-proto", - ":transaction-proto", - ":user-proto", - ], -) - -proto_library( - name = "raft-peering-proto", - srcs = [":raft-peering.proto"], - deps = [ - ":server-proto", - ], -) - -proto_library( - name = "server-proto", - srcs = [":server.proto"], -) - -proto_library( - name = "database-proto", - srcs = [":database.proto"], - deps = [ - ":migration-proto", - ], -) - -proto_library( - name = "migration-proto", - srcs = [":migration.proto"], - deps = [":concept-proto"], -) - -proto_library( - name = "user-proto", - srcs = [":user.proto"], -) - -proto_library( - name = "answer-proto", - srcs = ["answer.proto"], - deps = [":concept-proto"], -) - -proto_library( - name = "authentication-proto", - srcs = ["authentication.proto"], -) - -proto_library( - name = "concept-proto", - srcs = ["concept.proto"], -) - -proto_library( - name = "connection-proto", - srcs = ["connection.proto"], - deps = [ - ":authentication-proto", - ":database-proto", - ":server-proto", - ":version-proto", - ], -) - -proto_library( - name = "options-proto", - srcs = ["options.proto"], -) - -proto_library( - name = "query-proto", - srcs = ["query.proto"], - deps = [ - ":analyze-proto", - ":answer-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ], -) - -proto_library( - name = "error-proto", - srcs = ["error.proto"], -) - -proto_library( - name = "transaction-proto", - srcs = ["transaction.proto"], - deps = [ - ":analyze-proto", - ":answer-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ":query-proto", - ], -) - -proto_library( - name = "analyze-proto", - srcs = ["analyze.proto"], - deps = [ - ":analyzed-conjunction-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ], -) - -proto_library( - name = "analyzed-conjunction-proto", - srcs = ["analyzed-conjunction.proto"], - deps = [ - ":answer-proto", - ":concept-proto", - ], -) - proto_library( name = "version-proto", srcs = ["version.proto"], @@ -153,14 +17,14 @@ proto_library( filegroup( name = "proto-raw-buffers", srcs = [ - "analyze.proto", - "answer.proto", - "concept.proto", - "connection.proto", - "analyzed-conjunction.proto", - "options.proto", - "query.proto", - "transaction.proto", + "//proto/typedb_service:analyze.proto", + "//proto/typedb_service:analyzed-conjunction.proto", + "//proto/typedb_service:answer.proto", + "//proto/typedb_service:concept.proto", + "//proto/typedb_service:connection.proto", + "//proto/typedb_service:options.proto", + "//proto/typedb_service:query.proto", + "//proto/typedb_service:transaction.proto", "version.proto", ], ) diff --git a/proto/raft_service/BUILD b/proto/raft_service/BUILD new file mode 100644 index 0000000..62000b0 --- /dev/null +++ b/proto/raft_service/BUILD @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "raft-service", + srcs = ["raft_service.proto"], + deps = [ + ":replication-proto", + ], +) + +proto_library( + name = "replication-proto", + srcs = ["replication.proto"], + deps = [], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/raft_service/raft_service.proto b/proto/raft_service/raft_service.proto new file mode 100644 index 0000000..51886f2 --- /dev/null +++ b/proto/raft_service/raft_service.proto @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +import "proto/raft_service/replication.proto"; + +package typedb.protocol; + +// TODO: If the protocol is exposed, maybe it's actually `TypeDBRaft`? +service Raft { + rpc replication (Replication.Req) returns (Replication.Res); +} diff --git a/proto/raft-peering.proto b/proto/raft_service/replication.proto similarity index 61% rename from proto/raft-peering.proto rename to proto/raft_service/replication.proto index cc6411e..7d9dc92 100644 --- a/proto/raft-peering.proto +++ b/proto/raft_service/replication.proto @@ -6,10 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/server.proto"; - -// TODO: Rename to Replication -message Peering { +message Replication { message Req { uint64 from_id = 1; string from_address = 2; @@ -18,12 +15,3 @@ message Peering { message Res {} } - -message ServerStatus { - message Req {} - - message Res { - string connection_address = 1; - Server.ReplicaStatus replica_status = 2; - } -} diff --git a/proto/typedb_clustering_service/BUILD b/proto/typedb_clustering_service/BUILD new file mode 100644 index 0000000..4a5caf0 --- /dev/null +++ b/proto/typedb_clustering_service/BUILD @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "typedb-clustering-service", + srcs = ["typedb_clustering_service.proto"], + deps = [ + ":typedb-clustering-proto", + ], +) + +proto_library( + name = "typedb-clustering-proto", + srcs = ["typedb_clustering.proto"], + deps = [ + "//proto/typedb_service:server-proto", + ], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/typedb_clustering_service/typedb_clustering.proto b/proto/typedb_clustering_service/typedb_clustering.proto new file mode 100644 index 0000000..74ab151 --- /dev/null +++ b/proto/typedb_clustering_service/typedb_clustering.proto @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +package typedb.protocol; + +import "proto/typedb_service/server.proto"; + +message CanRegister { + message Req { + uint64 replica_id = 1; + } + + message Res {} +} + +message ServerStatus { + message Req {} + + message Res { + Server server = 1; + } +} diff --git a/proto/raft-peering-service.proto b/proto/typedb_clustering_service/typedb_clustering_service.proto similarity index 66% rename from proto/raft-peering-service.proto rename to proto/typedb_clustering_service/typedb_clustering_service.proto index 82301fd..cc83c61 100644 --- a/proto/raft-peering-service.proto +++ b/proto/typedb_clustering_service/typedb_clustering_service.proto @@ -4,11 +4,11 @@ syntax = "proto3"; -import "proto/raft-peering.proto"; +import "proto/typedb_clustering_service/typedb_clustering.proto"; package typedb.protocol; -service RaftPeering { - rpc peering (Peering.Req) returns (Peering.Res); +service TypeDBClustering { + rpc can_register (CanRegister.Req) returns (CanRegister.Res); rpc server_status (ServerStatus.Req) returns (ServerStatus.Res); -} \ No newline at end of file +} diff --git a/proto/typedb_service/BUILD b/proto/typedb_service/BUILD new file mode 100644 index 0000000..7246b15 --- /dev/null +++ b/proto/typedb_service/BUILD @@ -0,0 +1,134 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "typedb-service", + srcs = ["typedb_service.proto"], + deps = [ + ":authentication-proto", + ":connection-proto", + ":database-proto", + ":migration-proto", + ":server-proto", + ":transaction-proto", + ":user-proto", + ], +) + +proto_library( + name = "analyze-proto", + srcs = ["analyze.proto"], + deps = [ + ":analyzed-conjunction-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ], +) + +proto_library( + name = "analyzed-conjunction-proto", + srcs = ["analyzed-conjunction.proto"], + deps = [ + ":answer-proto", + ":concept-proto", + ], +) + +proto_library( + name = "answer-proto", + srcs = ["answer.proto"], + deps = [":concept-proto"], +) + +proto_library( + name = "authentication-proto", + srcs = ["authentication.proto"], +) + +proto_library( + name = "concept-proto", + srcs = ["concept.proto"], +) + +proto_library( + name = "connection-proto", + srcs = ["connection.proto"], + deps = [ + ":authentication-proto", + ":database-proto", + ":server-proto", + "//proto:version-proto", + ], +) + +proto_library( + name = "database-proto", + srcs = ["database.proto"], + deps = [ + ":migration-proto", + ], +) + +proto_library( + name = "error-proto", + srcs = ["error.proto"], +) + +proto_library( + name = "migration-proto", + srcs = ["migration.proto"], + deps = [":concept-proto"], +) + +proto_library( + name = "options-proto", + srcs = ["options.proto"], +) + +proto_library( + name = "query-proto", + srcs = ["query.proto"], + deps = [ + ":analyze-proto", + ":answer-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ], +) + +proto_library( + name = "server-proto", + srcs = ["server.proto"], +) + +proto_library( + name = "transaction-proto", + srcs = ["transaction.proto"], + deps = [ + ":analyze-proto", + ":answer-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ":query-proto", + ], +) + +proto_library( + name = "user-proto", + srcs = ["user.proto"], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/analyze.proto b/proto/typedb_service/analyze.proto similarity index 95% rename from proto/analyze.proto rename to proto/typedb_service/analyze.proto index 7e3eb92..1d3dee8 100644 --- a/proto/analyze.proto +++ b/proto/typedb_service/analyze.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyzed-conjunction.proto"; -import "proto/concept.proto"; -import "proto/error.proto"; -import "proto/options.proto"; +import "proto/typedb_service/analyzed-conjunction.proto"; +import "proto/typedb_service/concept.proto"; +import "proto/typedb_service/error.proto"; +import "proto/typedb_service/options.proto"; package typedb.protocol; diff --git a/proto/analyzed-conjunction.proto b/proto/typedb_service/analyzed-conjunction.proto similarity index 97% rename from proto/analyzed-conjunction.proto rename to proto/typedb_service/analyzed-conjunction.proto index eef88d0..a28f01e 100644 --- a/proto/analyzed-conjunction.proto +++ b/proto/typedb_service/analyzed-conjunction.proto @@ -4,8 +4,8 @@ syntax = "proto3"; -import "proto/answer.proto"; -import "proto/concept.proto"; +import "proto/typedb_service/answer.proto"; +import "proto/typedb_service/concept.proto"; package typedb.protocol; diff --git a/proto/answer.proto b/proto/typedb_service/answer.proto similarity index 97% rename from proto/answer.proto rename to proto/typedb_service/answer.proto index 91d4817..2e88e54 100644 --- a/proto/answer.proto +++ b/proto/typedb_service/answer.proto @@ -4,7 +4,7 @@ syntax = "proto3"; -import "proto/concept.proto"; +import "proto/typedb_service/concept.proto"; package typedb.protocol; diff --git a/proto/authentication.proto b/proto/typedb_service/authentication.proto similarity index 100% rename from proto/authentication.proto rename to proto/typedb_service/authentication.proto diff --git a/proto/concept.proto b/proto/typedb_service/concept.proto similarity index 100% rename from proto/concept.proto rename to proto/typedb_service/concept.proto diff --git a/proto/connection.proto b/proto/typedb_service/connection.proto similarity index 84% rename from proto/connection.proto rename to proto/typedb_service/connection.proto index b083d7f..d4d3207 100644 --- a/proto/connection.proto +++ b/proto/typedb_service/connection.proto @@ -4,9 +4,9 @@ syntax = "proto3"; -import "proto/authentication.proto"; -import "proto/database.proto"; -import "proto/server.proto"; +import "proto/typedb_service/authentication.proto"; +import "proto/typedb_service/database.proto"; +import "proto/typedb_service/server.proto"; import "proto/version.proto"; package typedb.protocol; diff --git a/proto/database.proto b/proto/typedb_service/database.proto similarity index 97% rename from proto/database.proto rename to proto/typedb_service/database.proto index 4694092..e3a2ca0 100644 --- a/proto/database.proto +++ b/proto/typedb_service/database.proto @@ -6,7 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/migration.proto"; +import "proto/typedb_service/migration.proto"; message DatabaseManager { diff --git a/proto/error.proto b/proto/typedb_service/error.proto similarity index 100% rename from proto/error.proto rename to proto/typedb_service/error.proto diff --git a/proto/migration.proto b/proto/typedb_service/migration.proto similarity index 98% rename from proto/migration.proto rename to proto/typedb_service/migration.proto index 46ec24c..ba82617 100644 --- a/proto/migration.proto +++ b/proto/typedb_service/migration.proto @@ -6,7 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/concept.proto"; +import "proto/typedb_service/concept.proto"; message Migration { message Export { diff --git a/proto/options.proto b/proto/typedb_service/options.proto similarity index 100% rename from proto/options.proto rename to proto/typedb_service/options.proto diff --git a/proto/query.proto b/proto/typedb_service/query.proto similarity index 92% rename from proto/query.proto rename to proto/typedb_service/query.proto index 8fd640b..c8e1347 100644 --- a/proto/query.proto +++ b/proto/typedb_service/query.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyze.proto"; -import "proto/answer.proto"; -import "proto/options.proto"; -import "proto/error.proto"; +import "proto/typedb_service/analyze.proto"; +import "proto/typedb_service/answer.proto"; +import "proto/typedb_service/options.proto"; +import "proto/typedb_service/error.proto"; package typedb.protocol; diff --git a/proto/server.proto b/proto/typedb_service/server.proto similarity index 91% rename from proto/server.proto rename to proto/typedb_service/server.proto index e29d01e..695a409 100644 --- a/proto/server.proto +++ b/proto/typedb_service/server.proto @@ -14,6 +14,13 @@ message ServerManager { } } + message Get { + message Req {} + message Res { + Server server = 1; + } + } + message Register { message Req { uint64 replica_id = 1; diff --git a/proto/transaction.proto b/proto/typedb_service/transaction.proto similarity index 93% rename from proto/transaction.proto rename to proto/typedb_service/transaction.proto index ce5f648..a47d07b 100644 --- a/proto/transaction.proto +++ b/proto/typedb_service/transaction.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyze.proto"; -import "proto/error.proto"; -import "proto/options.proto"; -import "proto/query.proto"; +import "proto/typedb_service/analyze.proto"; +import "proto/typedb_service/error.proto"; +import "proto/typedb_service/options.proto"; +import "proto/typedb_service/query.proto"; package typedb.protocol; diff --git a/proto/typedb-service.proto b/proto/typedb_service/typedb_service.proto similarity index 86% rename from proto/typedb-service.proto rename to proto/typedb_service/typedb_service.proto index 3b401c7..03d6648 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb_service/typedb_service.proto @@ -4,13 +4,13 @@ syntax = "proto3"; -import "proto/authentication.proto"; -import "proto/connection.proto"; -import "proto/database.proto"; -import "proto/server.proto"; -import "proto/user.proto"; -import "proto/transaction.proto"; -import "proto/migration.proto"; +import "proto/typedb_service/authentication.proto"; +import "proto/typedb_service/connection.proto"; +import "proto/typedb_service/database.proto"; +import "proto/typedb_service/server.proto"; +import "proto/typedb_service/user.proto"; +import "proto/typedb_service/transaction.proto"; +import "proto/typedb_service/migration.proto"; package typedb.protocol; @@ -24,6 +24,7 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + rpc servers_get (ServerManager.Get.Req) returns (ServerManager.Get.Res); rpc servers_register(ServerManager.Register.Req) returns (ServerManager.Register.Res); rpc servers_deregister(ServerManager.Deregister.Req) returns (ServerManager.Deregister.Res); diff --git a/proto/user.proto b/proto/typedb_service/user.proto similarity index 100% rename from proto/user.proto rename to proto/typedb_service/user.proto From aa011f69af9ef8d41a6882e8b3bb148264774632 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Fri, 24 Oct 2025 14:26:58 +0100 Subject: [PATCH 08/10] Add protobuf schema for Raft's 'state mutation request' (#232) Add protobuf schema for Raft's 'state mutation request'. The schema defines how the data is transported over the network and stored in the Raft log. - Add a protobuf definition for State Mutation Request - Create a mock RPC endpoint to handle that message as a hack to get the code generator to work --- grpc/java/BUILD | 6 ++- grpc/nodejs/BUILD | 34 ++++++++----- grpc/rust/BUILD | 3 +- proto/raft_service/BUILD | 7 +++ proto/raft_service/raft_service.proto | 2 + proto/raft_service/request.proto | 72 +++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 proto/raft_service/request.proto diff --git a/grpc/java/BUILD b/grpc/java/BUILD index a793641..b35e96a 100644 --- a/grpc/java/BUILD +++ b/grpc/java/BUILD @@ -3,6 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. package(default_visibility = ["//visibility:public"]) + load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") load("@rules_proto_grpc//java:defs.bzl", "java_grpc_library") @@ -12,8 +13,9 @@ java_grpc_library( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", @@ -36,7 +38,7 @@ java_grpc_library( checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/grpc/nodejs/BUILD b/grpc/nodejs/BUILD index 8e00bae..4db3294 100644 --- a/grpc/nodejs/BUILD +++ b/grpc/nodejs/BUILD @@ -3,6 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. package(default_visibility = ["//visibility:public"]) + load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") load("@typedb_bazel_distribution//npm:rules.bzl", "assemble_npm", "deploy_npm") load("@typedb_dependencies//distribution:deployment.bzl", "deployment") @@ -14,7 +15,10 @@ load("//grpc/nodejs:rules.bzl", "ts_grpc_compile") npm_link_all_packages(name = "node_modules") -exports_files(["package.json", "pnpm-lock.yaml"]) +exports_files([ + "package.json", + "pnpm-lock.yaml", +]) load("@typedb_protocol_npm//grpc/nodejs:protoc-gen-ts/package_json.bzl", protoc_gen_ts_bin = "bin") @@ -28,8 +32,9 @@ ts_grpc_compile( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", @@ -45,12 +50,14 @@ ts_grpc_compile( "//proto/typedb_service:transaction-proto", "//proto/typedb_service:typedb-service", "//proto/typedb_service:user-proto", - ] + ], ) ts_project( name = "typedb-protocol", srcs = [":typedb-protocol-src"], + declaration = True, + transpiler = "tsc", tsconfig = { "compilerOptions": { "target": "es2019", @@ -60,26 +67,27 @@ ts_project( }, "include": [ "proto", - ] + ], }, - declaration = True, + visibility = ["//visibility:public"], deps = [ ":node_modules/@grpc/grpc-js", - ":node_modules/google-protobuf", ":node_modules/@types/google-protobuf", ":node_modules/@types/node", + ":node_modules/google-protobuf", ":node_modules/typescript", ], - transpiler = "tsc", - visibility = ["//visibility:public"], ) npm_package( name = "typedb-protocol-package", - srcs = [":typedb-protocol", "package.json"], + srcs = [ + "package.json", + ":typedb-protocol", + ], + include_external_repositories = ["typedb_protocol"], include_runfiles = False, package = "typedb-protocol", - include_external_repositories = ["typedb_protocol"], visibility = ["//visibility:public"], ) @@ -90,18 +98,18 @@ assemble_npm( deploy_npm( name = "deploy-npm", - target = ":assemble-npm", - snapshot = deployment["npm"]["snapshot"], release = deployment["npm"]["release"], + snapshot = deployment["npm"]["snapshot"], + target = ":assemble-npm", ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), exclude = [ "package.json", "pnpm-lock.yaml", ], license_type = "mpl-header", - size = "small", ) diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index 200ec89..c3f0595 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -14,8 +14,9 @@ rust_tonic_compile( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", diff --git a/proto/raft_service/BUILD b/proto/raft_service/BUILD index 62000b0..c9fc4f2 100644 --- a/proto/raft_service/BUILD +++ b/proto/raft_service/BUILD @@ -11,6 +11,7 @@ proto_library( srcs = ["raft_service.proto"], deps = [ ":replication-proto", + ":request-proto", ], ) @@ -20,6 +21,12 @@ proto_library( deps = [], ) +proto_library( + name = "request-proto", + srcs = ["request.proto"], + deps = [], +) + checkstyle_test( name = "checkstyle", size = "small", diff --git a/proto/raft_service/raft_service.proto b/proto/raft_service/raft_service.proto index 51886f2..d934687 100644 --- a/proto/raft_service/raft_service.proto +++ b/proto/raft_service/raft_service.proto @@ -5,10 +5,12 @@ syntax = "proto3"; import "proto/raft_service/replication.proto"; +import "proto/raft_service/request.proto"; package typedb.protocol; // TODO: If the protocol is exposed, maybe it's actually `TypeDBRaft`? service Raft { rpc replication (Replication.Req) returns (Replication.Res); + rpc request (Request) returns (Request); } diff --git a/proto/raft_service/request.proto b/proto/raft_service/request.proto new file mode 100644 index 0000000..2296a36 --- /dev/null +++ b/proto/raft_service/request.proto @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +package typedb.protocol; + +message Request { + oneof request { + Load load = 1; + DatabaseManager database_manager = 2; + DatabaseCommit database_commit = 3; + UserManager user_manager = 4; + } + + message Load {} + + message DatabaseManager { + oneof database_manager { + All all = 1; + Get get = 2; + GetUnrestricted get_unrestricted = 3; + Create create = 4; + CreateUnrestricted create_unrestricted = 5; + Delete delete = 6; + } + + message All {} + message Get { string name = 1; } + message GetUnrestricted { string name = 1; } + message Create { string name = 1; } + message CreateUnrestricted { string name = 1; } + message Delete { string name = 1; } + } + + message DatabaseCommit { + oneof database_commit { + SchemaCommit schema = 1; + DataCommit data = 2; + } + + message SchemaCommit { + string name = 1; + bytes commit_record = 2; + } + + message DataCommit { + string name = 1; + bytes commit_record = 2; + } + } + + message UserManager { + oneof user_manager { + All all = 1; + Get get = 2; + Contains contains = 3; + Create create = 4; + Update update = 5; + Delete delete = 6; + } + + message All { string accessor = 1; } + message Get { string name = 1; string accessor = 2; } + message Contains { string name = 1; } + message Create { bytes commit_record = 1; } + message Update { bytes commit_record = 1; } + message Delete { bytes commit_record = 1; } + } +} + From 065cb7ed26e6fbecee8b68e7e9c6ebf51fa9c701 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Wed, 19 Nov 2025 17:49:45 +0000 Subject: [PATCH 09/10] Mirror TypeDB service methods in Raft service (#238) ## Release notes: usage and product changes Reorganize Raft service's request messages to mirror the structure of TypeDB service's methods. Additionally, optionalize the Server message to allow returning pure Raft Agent id without other server information available for Clustered operations (in case the replica is not available). ## Implementation I just did it! --------- Co-authored-by: Ganeshwara Hananda Co-authored-by: Ganeshwara Hananda --- proto/raft_service/request.proto | 105 ++++++++++++++++------ proto/typedb_service/server.proto | 6 +- proto/typedb_service/typedb_service.proto | 1 - 3 files changed, 82 insertions(+), 30 deletions(-) diff --git a/proto/raft_service/request.proto b/proto/raft_service/request.proto index 2296a36..ff88beb 100644 --- a/proto/raft_service/request.proto +++ b/proto/raft_service/request.proto @@ -10,8 +10,9 @@ message Request { oneof request { Load load = 1; DatabaseManager database_manager = 2; - DatabaseCommit database_commit = 3; + Database database = 3; UserManager user_manager = 4; + User user = 5; } message Load {} @@ -19,25 +20,50 @@ message Request { message DatabaseManager { oneof database_manager { All all = 1; - Get get = 2; - GetUnrestricted get_unrestricted = 3; - Create create = 4; - CreateUnrestricted create_unrestricted = 5; - Delete delete = 6; + Contains contains = 2; + Get get = 3; + GetUnrestricted get_unrestricted = 4; + Create create = 5; + CreateUnrestricted create_unrestricted = 6; } message All {} - message Get { string name = 1; } - message GetUnrestricted { string name = 1; } - message Create { string name = 1; } - message CreateUnrestricted { string name = 1; } - message Delete { string name = 1; } + + message Contains { + string name = 1; + } + + message Get { + string name = 1; + } + + message GetUnrestricted { + string name = 1; + } + message Create { + string name = 1; + } + + message CreateUnrestricted { + string name = 1; + } } - message DatabaseCommit { - oneof database_commit { - SchemaCommit schema = 1; - DataCommit data = 2; + message Database { + oneof database { + Schema schema = 1; + TypeSchema type_schema = 2; + SchemaCommit schema_commit = 3; + DataCommit data_commit = 4; + Delete delete = 5; + } + + message Schema { + string name = 1; + } + + message TypeSchema { + string name = 1; } message SchemaCommit { @@ -49,24 +75,51 @@ message Request { string name = 1; bytes commit_record = 2; } + + message Delete { + string name = 1; + } } message UserManager { oneof user_manager { All all = 1; - Get get = 2; - Contains contains = 3; + Contains contains = 2; + Get get = 3; Create create = 4; - Update update = 5; - Delete delete = 6; } - message All { string accessor = 1; } - message Get { string name = 1; string accessor = 2; } - message Contains { string name = 1; } - message Create { bytes commit_record = 1; } - message Update { bytes commit_record = 1; } - message Delete { bytes commit_record = 1; } + message All { + string accessor = 1; + } + + message Contains { + string name = 1; + string accessor = 2; + } + + message Get { + string name = 1; + string accessor = 2; + } + + message Create { + bytes commit_record = 1; + } } -} + message User { + oneof user { + Update update = 1; + Delete delete = 2; + } + + message Update { + bytes commit_record = 1; + } + + message Delete { + bytes commit_record = 1; + } + } +} diff --git a/proto/typedb_service/server.proto b/proto/typedb_service/server.proto index 695a409..f75f5f3 100644 --- a/proto/typedb_service/server.proto +++ b/proto/typedb_service/server.proto @@ -38,13 +38,13 @@ message ServerManager { } message Server { - string address = 1; + optional string address = 1; optional ReplicaStatus replica_status = 2; message ReplicaStatus { uint64 replica_id = 1; - ReplicaType replica_type = 2; - uint64 term = 3; + optional ReplicaType replica_type = 2; + optional uint64 term = 3; enum ReplicaType { Primary = 0; diff --git a/proto/typedb_service/typedb_service.proto b/proto/typedb_service/typedb_service.proto index 03d6648..2dec0f6 100644 --- a/proto/typedb_service/typedb_service.proto +++ b/proto/typedb_service/typedb_service.proto @@ -59,5 +59,4 @@ service TypeDB { // requests and responses back-and-forth. The first transaction client message must // be {Transaction.Open.Req}. Closing the stream closes the transaction. rpc transaction (stream Transaction.Client) returns (stream Transaction.Server); - } From b007ebc43e307ca4b6354fedbbfc3223361044dc Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Thu, 20 Nov 2025 14:14:32 +0000 Subject: [PATCH 10/10] Rename ReplicaType to ReplicaRole --- proto/typedb_service/server.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proto/typedb_service/server.proto b/proto/typedb_service/server.proto index f75f5f3..fa524fa 100644 --- a/proto/typedb_service/server.proto +++ b/proto/typedb_service/server.proto @@ -43,10 +43,10 @@ message Server { message ReplicaStatus { uint64 replica_id = 1; - optional ReplicaType replica_type = 2; + optional ReplicaRole replica_role = 2; optional uint64 term = 3; - enum ReplicaType { + enum ReplicaRole { Primary = 0; Candidate = 1; Secondary = 2;