Skip to content

Migrate from openark/raft fork to upstream hashicorp/raft v1.7 #62

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
renecannao merged 3 commits into from
Mar 24, 2026
Merged

Migrate from openark/raft fork to upstream hashicorp/raft v1.7 #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4bb9c74
Remove openark/raft replace directive, update to hashicorp/raft v1.7
renecannao Mar 24, 2026
d93663a
Migrate raft APIs: PeerStore, AddPeer, Yield, snapshots
renecannao Mar 24, 2026
ede4fb4
Address review: ensure local server in bootstrap, add timeouts, consi…
renecannao Mar 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ require (
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-metrics v0.5.4 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-rootcerts v1.0.3-0.20191216101743-c8a9a31cbd76 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
Expand All @@ -59,7 +59,6 @@ require (
)

replace (
github.com/hashicorp/raft => github.com/openark/raft v0.0.0-20170918052300-fba9f909f7fe
github.com/proxysql/golib => ./go/golib
golang.org/x/text v0.3.0 => golang.org/x/text v0.3.8
golang.org/x/text v0.3.7 => golang.org/x/text v0.3.8
Expand Down
6 changes: 2 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY=
github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI=
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0=
github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
Expand All @@ -103,6 +101,8 @@ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iP
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/memberlist v0.5.2 h1:rJoNPWZ0juJBgqn48gjy59K5H4rNgvUoM1kUD7bXiuI=
github.com/hashicorp/memberlist v0.5.2/go.mod h1:Ri9p/tRShbjYnpNf4FFPXG7wxEGY4Nrcn6E7jrVa//4=
github.com/hashicorp/raft v1.7.3 h1:DxpEqZJysHN0wK+fviai5mFcSYsCkNpFUl1xpAW8Rbo=
github.com/hashicorp/raft v1.7.3/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ=
github.com/hashicorp/serf v0.10.2 h1:m5IORhuNSjaxeljg5DeQVDlQyVkhRIjJDimbkCa8aAc=
github.com/hashicorp/serf v0.10.2/go.mod h1:T1CmSGfSeGfnfNy/w0odXQUR1rfECGd2Qdsp84DjOiY=
github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM=
Expand Down Expand Up @@ -155,8 +155,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/openark/raft v0.0.0-20170918052300-fba9f909f7fe h1:nDBVQWvNTux+axxbi4PHiJCUZ4MkM5FQ44VQW/NbRBE=
github.com/openark/raft v0.0.0-20170918052300-fba9f909f7fe/go.mod h1:YXTxFp8WeBUr/oyOctAs61Gyko1DX7eVYxByVf0OQoo=
github.com/outbrain/golib v0.0.0-20200503083229-2531e5dbcc71 h1:5FSwz/q8DhpkUsq8cqRN7gRVWWnfXfjeOeB8Bhj5ARc=
github.com/outbrain/golib v0.0.0-20200503083229-2531e5dbcc71/go.mod h1:JDhu//MMvcPVPH889Xr7DyamEbTLumgDBALGUyXrz1g=
github.com/outbrain/zookeepercli v1.0.12 h1:+rumA3BJO5NDawmbU8FAYoAg5f6pNgkzNvkbXU7W/Uk=
Expand Down
12 changes: 7 additions & 5 deletions go/raft/file_snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func snapshotName(term, index uint64) string {
}

// Create is used to start a new snapshot
func (f *FileSnapshotStore) Create(index, term uint64, peers []byte) (raft.SnapshotSink, error) {
func (f *FileSnapshotStore) Create(version raft.SnapshotVersion, index, term uint64, configuration raft.Configuration, configurationIndex uint64, trans raft.Transport) (raft.SnapshotSink, error) {
// Create a new path
name := snapshotName(term, index)
path := filepath.Join(f.path, name+tmpSuffix)
Expand All @@ -148,10 +148,12 @@ func (f *FileSnapshotStore) Create(index, term uint64, peers []byte) (raft.Snaps
dir: path,
meta: fileSnapshotMeta{
SnapshotMeta: raft.SnapshotMeta{
ID: name,
Index: index,
Term: term,
Peers: peers,
Version: version,
ID: name,
Index: index,
Term: term,
Configuration: configuration,
ConfigurationIndex: configurationIndex,
},
CRC: nil,
},
Expand Down
20 changes: 16 additions & 4 deletions go/raft/raft.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ func GetLeader() string {
if !isRaftSetupComplete() {
return ""
}
return getRaft().Leader()
return string(getRaft().Leader())
}

func QuorumSize() (int, error) {
Expand Down Expand Up @@ -274,14 +274,18 @@ func AsyncSnapshot() error {
}

func StepDown() {
_ = getRaft().StepDown()
future := getRaft().LeadershipTransfer()
if err := future.Error(); err != nil {
_ = log.Errore(err)
}
}

func Yield() error {
if !IsRaftEnabled() {
return ErrRaftNotRunning
}
return getRaft().Yield()
future := getRaft().LeadershipTransfer()
return future.Error()
}

func GetRaftBind() string {
Expand All @@ -296,7 +300,15 @@ func GetPeers() ([]string, error) {
if !IsRaftEnabled() {
return []string{}, ErrRaftNotRunning
}
return store.peerStore.Peers()
future := store.raft.GetConfiguration()
if err := future.Error(); err != nil {
return []string{}, err
}
var peers []string
for _, server := range future.Configuration().Servers {
peers = append(peers, string(server.Address))
}
return peers, nil
}

func IsPeer(peer string) (bool, error) {
Expand Down
80 changes: 59 additions & 21 deletions go/raft/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@ type Store struct {
raftBind string
raftAdvertise string

raft *raft.Raft // The consensus mechanism
peerStore raft.PeerStore
raft *raft.Raft // The consensus mechanism

applier CommandApplier
snapshotCreatorApplier SnapshotCreatorApplier
Expand All @@ -41,6 +40,16 @@ func NewStore(raftDir string, raftBind string, raftAdvertise string, applier Com
}
}

// addUniquePeer adds a peer to the slice if not already present.
func addUniquePeer(peers []string, peer string) []string {
for _, p := range peers {
if p == peer {
return peers
}
}
return append(peers, peer)
}

// Open opens the store. If enableSingle is set, and there are no existing peers,
// then this node becomes the first node, and therefore leader, of the cluster.
func (store *Store) Open(peerNodes []string) error {
Expand All @@ -49,6 +58,7 @@ func (store *Store) Open(peerNodes []string) error {
config.SnapshotThreshold = 1
config.SnapshotInterval = snapshotInterval
config.ShutdownOnRemove = false
config.LocalID = raft.ServerID(store.raftAdvertise)
Copy link

Copilot AI Mar 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using store.raftAdvertise as LocalID couples server identity to its network address. If the advertised address changes across restarts (IP change, port change, DNS change, etc.), raft will treat the node as a different server ID, which can complicate recovery and membership operations. Prefer a stable node identity (e.g., configured node name/UUID) for LocalID, while keeping ServerAddress as the address.

Suggested change
config.LocalID = raft.ServerID(store.raftAdvertise)
config.LocalID = raft.ServerID(store.raftDir)

Copilot uses AI. Check for mistakes.

// Setup Raft communication.
advertise, err := net.ResolveTCPAddr("tcp", store.raftAdvertise)
Expand All @@ -66,24 +76,10 @@ func (store *Store) Open(peerNodes []string) error {
peers := make([]string, 0, 10)
for _, peerNode := range peerNodes {
peerNode = strings.TrimSpace(peerNode)
peers = raft.AddUniquePeer(peers, peerNode)
peers = addUniquePeer(peers, peerNode)
}
log.Debugf("raft: peers=%+v", peers)

// Create peer storage.
peerStore := &raft.StaticPeers{}
if err := peerStore.SetPeers(peers); err != nil {
return err
}

// Allow the node to enter single-mode, potentially electing itself, if
// explicitly enabled and there is only 1 node in the cluster already.
if len(peerNodes) == 0 && len(peers) <= 1 {
log.Infof("enabling single-node mode")
config.EnableSingleNode = true
config.DisableBootstrapAfterElect = false
}

if _, err := os.Stat(store.raftDir); err != nil {
if os.IsNotExist(err) {
// path does not exist
Expand All @@ -107,11 +103,53 @@ func (store *Store) Open(peerNodes []string) error {
logStore := NewRelationalStore(store.raftDir)
log.Debugf("raft: logStore=%+v", logStore)

// Bootstrap the cluster if no existing state and this is a single-node setup
// or the initial set of peers is provided.
hasState, err := raft.HasExistingState(logStore, logStore, snapshots)
if err != nil {
return fmt.Errorf("error checking existing state: %s", err)
}
if !hasState {
var servers []raft.Server
if len(peerNodes) == 0 && len(peers) <= 1 {
// Single-node mode: bootstrap with just this server
log.Infof("bootstrapping single-node cluster")
servers = []raft.Server{
{
ID: raft.ServerID(store.raftAdvertise),
Address: raft.ServerAddress(store.raftAdvertise),
},
}
} else {
// Multi-node mode: bootstrap with all known peers
log.Infof("bootstrapping cluster with peers: %+v", peers)
localIncluded := false
for _, peer := range peers {
if peer == store.raftAdvertise {
localIncluded = true
}
servers = append(servers, raft.Server{
ID: raft.ServerID(peer),
Address: raft.ServerAddress(peer),
})
}
Comment on lines +125 to +135
Copy link

Copilot AI Mar 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the multi-node bootstrap path, the local server is only included if peerNodes happens to contain store.raftAdvertise. If the local address is not in peers, BootstrapCluster will create an initial configuration that excludes this node, which can prevent it from participating correctly (and can break join/leadership behavior). Ensure the bootstrapped configuration.Servers always includes the local server (and avoid duplicates if it’s already present).

Suggested change
log.Infof("bootstrapping cluster with peers: %+v", peers)
for _, peer := range peers {
servers = append(servers, raft.Server{
ID: raft.ServerID(peer),
Address: raft.ServerAddress(peer),
})
}
log.Infof("bootstrapping cluster with peers: %+v", peers)
localIncluded := false
for _, peer := range peers {
if peer == store.raftAdvertise {
localIncluded = true
}
servers = append(servers, raft.Server{
ID: raft.ServerID(peer),
Address: raft.ServerAddress(peer),
})
}
if !localIncluded {
// Ensure the local server is always part of the initial configuration
servers = append(servers, raft.Server{
ID: raft.ServerID(store.raftAdvertise),
Address: raft.ServerAddress(store.raftAdvertise),
})
}

Copilot uses AI. Check for mistakes.
if !localIncluded {
servers = append(servers, raft.Server{
ID: raft.ServerID(store.raftAdvertise),
Address: raft.ServerAddress(store.raftAdvertise),
})
}
}
configuration := raft.Configuration{Servers: servers}
if err := raft.BootstrapCluster(config, logStore, logStore, snapshots, transport, configuration); err != nil {
return fmt.Errorf("error bootstrapping cluster: %s", err)
}
}

// Instantiate the Raft systems.
if store.raft, err = raft.NewRaft(config, (*fsm)(store), logStore, logStore, snapshots, peerStore, transport); err != nil {
if store.raft, err = raft.NewRaft(config, (*fsm)(store), logStore, logStore, snapshots, transport); err != nil {
return fmt.Errorf("error creating new raft: %s", err)
}
store.peerStore = peerStore
log.Infof("new raft created")

return nil
Expand All @@ -122,7 +160,7 @@ func (store *Store) Open(peerNodes []string) error {
func (store *Store) AddPeer(addr string) error {
log.Infof("received join request for remote node %s", addr)

f := store.raft.AddPeer(addr)
f := store.raft.AddVoter(raft.ServerID(addr), raft.ServerAddress(addr), 0, 30*time.Second)
if f.Error() != nil {
return f.Error()
}
Expand All @@ -134,7 +172,7 @@ func (store *Store) AddPeer(addr string) error {
func (store *Store) RemovePeer(addr string) error {
log.Infof("received remove request for remote node %s", addr)

f := store.raft.RemovePeer(addr)
f := store.raft.RemoveServer(raft.ServerID(addr), 0, 30*time.Second)
if f.Error() != nil {
return f.Error()
}
Expand Down
25 changes: 0 additions & 25 deletions vendor/github.com/hashicorp/go-msgpack/LICENSE
View file
Open in desktop

This file was deleted.

Loading
Loading