Skip to content

Split protocol into 3 separate services #230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
30 changes: 17 additions & 13 deletions grpc/java/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,24 @@ load("@rules_proto_grpc//java:defs.bzl", "java_grpc_library")
java_grpc_library(
name = "typedb-protocol",
protos = [
"//proto:typedb-service",
"//proto:server-proto",
"//proto:user-proto",
"//proto:database-proto",
"//proto:migration-proto",
"//proto:error-proto",
"//proto:answer-proto",
"//proto:concept-proto",
"//proto:connection-proto",
"//proto:authentication-proto",
"//proto:options-proto",
"//proto:query-proto",
"//proto:transaction-proto",
"//proto:version-proto",
"//proto/raft_service:raft-service",
"//proto/raft_service:replication-proto",
"//proto/typedb_clustering_service:typedb-clustering-service",
"//proto/typedb_clustering_service:typedb-clustering-proto",
"//proto/typedb_service:answer-proto",
"//proto/typedb_service:authentication-proto",
"//proto/typedb_service:concept-proto",
"//proto/typedb_service:connection-proto",
"//proto/typedb_service:database-proto",
"//proto/typedb_service:error-proto",
"//proto/typedb_service:migration-proto",
"//proto/typedb_service:options-proto",
"//proto/typedb_service:query-proto",
"//proto/typedb_service:server-proto",
"//proto/typedb_service:transaction-proto",
"//proto/typedb_service:typedb-service",
"//proto/typedb_service:user-proto",
],
# TypeDB Core bundles JARs by maven coordinate, we can remove this when Core is rewritten in Rust
tags = ["maven_coordinates=com.typedb:typedb-protocol:{pom_version}"],
Expand Down
30 changes: 17 additions & 13 deletions grpc/nodejs/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,24 @@ protoc_gen_ts_bin.protoc_gen_ts_binary(
ts_grpc_compile(
name = "typedb-protocol-src",
deps = [
"//proto:typedb-service",
"//proto:server-proto",
"//proto:user-proto",
"//proto:database-proto",
"//proto:migration-proto",
"//proto:error-proto",
"//proto:answer-proto",
"//proto:concept-proto",
"//proto:connection-proto",
"//proto:authentication-proto",
"//proto:options-proto",
"//proto:query-proto",
"//proto:transaction-proto",
"//proto:version-proto",
"//proto/raft_service:raft-service",
"//proto/raft_service:replication-proto",
"//proto/typedb_clustering_service:typedb-clustering-service",
"//proto/typedb_clustering_service:typedb-clustering-proto",
"//proto/typedb_service:answer-proto",
"//proto/typedb_service:authentication-proto",
"//proto/typedb_service:concept-proto",
"//proto/typedb_service:connection-proto",
"//proto/typedb_service:database-proto",
"//proto/typedb_service:error-proto",
"//proto/typedb_service:migration-proto",
"//proto/typedb_service:options-proto",
"//proto/typedb_service:query-proto",
"//proto/typedb_service:server-proto",
"//proto/typedb_service:transaction-proto",
"//proto/typedb_service:typedb-service",
"//proto/typedb_service:user-proto",
]
)

Expand Down
32 changes: 17 additions & 15 deletions grpc/rust/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,24 @@ load("@typedb_dependencies//builder/proto_grpc/rust:compile.bzl", "rust_tonic_co
rust_tonic_compile(
name = "typedb_protocol_src",
srcs = [
"//proto:answer-proto",
"//proto:authentication-proto",
"//proto:concept-proto",
"//proto:connection-proto",
"//proto:database-proto",
"//proto:error-proto",
"//proto:migration-proto",
"//proto:options-proto",
"//proto:query-proto",
"//proto:raft-peering-proto",
"//proto:raft-peering-service",
"//proto:server-proto",
"//proto:transaction-proto",
"//proto:typedb-service",
"//proto:user-proto",
"//proto:version-proto",
"//proto/raft_service:raft-service",
"//proto/raft_service:replication-proto",
"//proto/typedb_clustering_service:typedb-clustering-service",
"//proto/typedb_clustering_service:typedb-clustering-proto",
"//proto/typedb_service:answer-proto",
"//proto/typedb_service:authentication-proto",
"//proto/typedb_service:concept-proto",
"//proto/typedb_service:connection-proto",
"//proto/typedb_service:database-proto",
"//proto/typedb_service:error-proto",
"//proto/typedb_service:migration-proto",
"//proto/typedb_service:options-proto",
"//proto/typedb_service:query-proto",
"//proto/typedb_service:server-proto",
"//proto/typedb_service:transaction-proto",
"//proto/typedb_service:typedb-service",
"//proto/typedb_service:user-proto",
],
packages = ["typedb.protocol"],
)
Expand Down
32 changes: 17 additions & 15 deletions grpc/rust/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,24 @@

fn main() -> std::io::Result<()> {
let protos = vec![
"../../proto/answer.proto",
"../../proto/authentication.proto",
"../../proto/concept.proto",
"../../proto/connection.proto",
"../../proto/database.proto",
"../../proto/error.proto",
"../../proto/migration.proto",
"../../proto/options.proto",
"../../proto/query.proto",
"../../proto/server.proto",
"../../proto/transaction.proto",
"../../proto/typedb-service.proto",
"../../proto/user.proto",
"../../proto/raft_service/replication.proto",
"../../proto/raft_service/raft_service.proto",
"../../proto/typedb_service/answer.proto",
"../../proto/typedb_service/authentication.proto",
"../../proto/typedb_service/concept.proto",
"../../proto/typedb_service/connection.proto",
"../../proto/typedb_service/database.proto",
"../../proto/typedb_service/error.proto",
"../../proto/typedb_service/migration.proto",
"../../proto/typedb_service/options.proto",
"../../proto/typedb_service/query.proto",
"../../proto/typedb_service/server.proto",
"../../proto/typedb_service/transaction.proto",
"../../proto/typedb_service/typedb_service.proto",
"../../proto/typedb_service/user.proto",
"../../proto/typedb_clustering_service/typedb_clustering.proto",
"../../proto/typedb_clustering_service/typedb_clustering_service.proto",
"../../proto/version.proto",
"../../proto/raft-peering.proto",
"../../proto/raft-peering-service.proto",
];

tonic_build::configure()
Expand Down
126 changes: 6 additions & 120 deletions proto/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -6,120 +6,6 @@ package(default_visibility = ["//visibility:public"])

load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test")

proto_library(
name = "raft-peering-service",
srcs = [":raft-peering-service.proto"],
deps = [
":raft-peering-proto",
],
)

proto_library(
name = "typedb-service",
srcs = [":typedb-service.proto"],
deps = [
":authentication-proto",
":connection-proto",
":database-proto",
":migration-proto",
":server-proto",
":transaction-proto",
":user-proto",
],
)

proto_library(
name = "raft-peering-proto",
srcs = [":raft-peering.proto"],
deps = [
":server-proto",
],
)

proto_library(
name = "server-proto",
srcs = [":server.proto"],
)

proto_library(
name = "database-proto",
srcs = [":database.proto"],
deps = [
":migration-proto",
],
)

proto_library(
name = "migration-proto",
srcs = [":migration.proto"],
deps = [":concept-proto"],
)

proto_library(
name = "user-proto",
srcs = [":user.proto"],
)

proto_library(
name = "answer-proto",
srcs = ["answer.proto"],
deps = [":concept-proto"],
)

proto_library(
name = "authentication-proto",
srcs = ["authentication.proto"],
)

proto_library(
name = "concept-proto",
srcs = ["concept.proto"],
)

proto_library(
name = "connection-proto",
srcs = ["connection.proto"],
deps = [
":authentication-proto",
":database-proto",
":server-proto",
":version-proto",
],
)

proto_library(
name = "options-proto",
srcs = ["options.proto"],
)

proto_library(
name = "query-proto",
srcs = ["query.proto"],
deps = [
":answer-proto",
":concept-proto",
":error-proto",
":options-proto",
],
)

proto_library(
name = "error-proto",
srcs = ["error.proto"],
)

proto_library(
name = "transaction-proto",
srcs = ["transaction.proto"],
deps = [
":answer-proto",
":concept-proto",
":error-proto",
":options-proto",
":query-proto",
],
)

proto_library(
name = "version-proto",
srcs = ["version.proto"],
Expand All @@ -131,12 +17,12 @@ proto_library(
filegroup(
name = "proto-raw-buffers",
srcs = [
"answer.proto",
"concept.proto",
"connection.proto",
"options.proto",
"query.proto",
"transaction.proto",
"//proto/typedb_service:answer.proto",
"//proto/typedb_service:concept.proto",
"//proto/typedb_service:connection.proto",
"//proto/typedb_service:options.proto",
"//proto/typedb_service:query.proto",
"//proto/typedb_service:transaction.proto",
"version.proto",
],
)
Expand Down
28 changes: 28 additions & 0 deletions proto/raft_service/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

package(default_visibility = ["//visibility:public"])

load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test")

proto_library(
name = "raft-service",
srcs = ["raft_service.proto"],
deps = [
":replication-proto",
],
)

proto_library(
name = "replication-proto",
srcs = ["replication.proto"],
deps = [],
)

checkstyle_test(
name = "checkstyle",
size = "small",
include = glob(["*"]),
license_type = "mpl-header",
)
14 changes: 14 additions & 0 deletions proto/raft_service/raft_service.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

syntax = "proto3";

import "proto/raft_service/replication.proto";

package typedb.protocol;

// TODO: If the protocol is exposed, maybe it's actually `TypeDBRaft`?
service Raft {
rpc replication (Replication.Req) returns (Replication.Res);
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@ syntax = "proto3";

package typedb.protocol;

import "proto/server.proto";

// TODO: Rename to Replication
message Peering {
message Replication {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now that I'm looking into this as a reader, I think the message should be called Raft rather than Replication. Similarly, RaftService should have one rpc endpoint, raft.

It's because the message that we exchange are not just replication message, but any messages as defined by the Raft protocol. They can either be "request vote", "append entries" (ie., the message responsible for replication), and "cluster conf change" (ie., add / remove node).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmmm okay. I thought that we considered extending this protocol and adding more methods. Your point about vote requests, etc. is reasonable, I'll rename it now.

message Req {
uint64 from_id = 1;
string from_address = 2;
Expand All @@ -18,12 +15,3 @@ message Peering {

message Res {}
}

message ServerStatus {
message Req {}

message Res {
string connection_address = 1;
Server.ReplicaStatus replica_status = 2;
}
}
30 changes: 30 additions & 0 deletions proto/typedb_clustering_service/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

package(default_visibility = ["//visibility:public"])

load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test")

proto_library(
name = "typedb-clustering-service",
srcs = ["typedb_clustering_service.proto"],
deps = [
":typedb-clustering-proto",
],
)

proto_library(
name = "typedb-clustering-proto",
srcs = ["typedb_clustering.proto"],
deps = [
"//proto/typedb_service:server-proto",
],
)

checkstyle_test(
name = "checkstyle",
size = "small",
include = glob(["*"]),
license_type = "mpl-header",
)
Loading