From 823bef8d5553d5ff51d181a54786ccf69eb0c690 Mon Sep 17 00:00:00 2001 From: Michael McCarty Date: Thu, 29 Jan 2026 17:40:50 -0800 Subject: [PATCH 01/12] =?UTF-8?q?=F0=9F=9A=80=20Implement=20concurrent=20c?= =?UTF-8?q?ontract=20save=20queue=20with=20deduplication?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/boost/boost.go | 3 ++ src/boost/boost_datastore.go | 76 ++++++++++++++++++++++++++---------- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/src/boost/boost.go b/src/boost/boost.go index edfb9b74..619a214b 100644 --- a/src/boost/boost.go +++ b/src/boost/boost.go @@ -347,6 +347,9 @@ func init() { if err == nil { Contracts = c } + + // Start the background save queue worker + startSaveQueueWorker() } func changeContractState(contract *Contract, newstate int) { diff --git a/src/boost/boost_datastore.go b/src/boost/boost_datastore.go index b9a08cbd..a788fdde 100644 --- a/src/boost/boost_datastore.go +++ b/src/boost/boost_datastore.go @@ -8,6 +8,7 @@ import ( "fmt" "log" "strings" + "sync" "time" "github.com/peterbourgon/diskv/v3" @@ -21,6 +22,11 @@ var ctx = context.Background() var ddl string var queries *Queries +// Save queue infrastructure +var saveQueue = make(chan string, 100) // Buffer up to 100 save requests +var saveQueueMutex sync.Mutex +var pendingSaves = make(map[string]bool) // Track contracts pending save + func sqliteInit() { db, _ := sql.Open("sqlite", "ttbb-data/ContractData.sqlite?_busy_timeout=5000") @@ -31,6 +37,21 @@ func sqliteInit() { queries = New(db) } +// startSaveQueueWorker initializes the background worker that processes save requests +func startSaveQueueWorker() { + go func() { + for contractHash := range saveQueue { + // Mark as no longer pending + saveQueueMutex.Lock() + delete(pendingSaves, contractHash) + saveQueueMutex.Unlock() + + // Process the actual save + processSingleContractSave(contractHash) + } + }() +} + // SaveAllData will save all contract data to disk func SaveAllData() { log.Print("Saving contract data") @@ -69,32 +90,34 @@ func InverseTransform(pathKey *diskv.PathKey) (key string) { func saveData(contractHash string) { if contractHash != "" { - contract := FindContractByHash(contractHash) - if contract == nil { - return + // Queue individual contract save with deduplication + saveQueueMutex.Lock() + // If already pending, no need to add another request + if !pendingSaves[contractHash] { + pendingSaves[contractHash] = true + saveQueue <- contractHash } - - /* - if contract.State == ContractStateSignup { - if time.Since(contract.LastSaveTime) < 30*time.Second && len(contract.Boosters) < contract.CoopSize { - // Only save signup contracts every 30 seconds during signup - return - } - } else { - if time.Since(contract.LastSaveTime) < 15*time.Second { - // Only save non-signup contracts every 15 seconds - return - } - } - */ - contract.LastSaveTime = time.Now() - saveSqliteData(contract) + saveQueueMutex.Unlock() return } + // Save all contracts - queue each one individually for _, c := range Contracts { - saveSqliteData(c) - c.LastSaveTime = time.Now() + contractHash := c.ContractHash + saveQueueMutex.Lock() + if !pendingSaves[contractHash] { + pendingSaves[contractHash] = true + select { + case saveQueue <- contractHash: + // Successfully queued + default: + // Queue is full, skip this one (it will be retried in the next save cycle) + log.Printf("Save queue full, skipping contract: %s", contractHash) + saveQueueMutex.Unlock() + break + } + } + saveQueueMutex.Unlock() } // Legacy disk store backup @@ -102,6 +125,17 @@ func saveData(contractHash string) { //_ = dataStore.Write("EggsBackup", b) } +// processSingleContractSave handles the actual database write for a single contract +func processSingleContractSave(contractHash string) { + contract := FindContractByHash(contractHash) + if contract == nil { + return + } + + contract.LastSaveTime = time.Now() + saveSqliteData(contract) +} + /* func saveEndData(c *Contract) error { //diskmutex.Lock() From 3389f72188a3bc05aebdc3b5c2943d7fcf88d0ee Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:46:17 +0000 Subject: [PATCH 02/12] Initial plan From 60926e3d768ac1acc3dd30e05bfda5fec93df121 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:48:59 +0000 Subject: [PATCH 03/12] Fix double mutex unlock bug in saveData loop Co-authored-by: mkmccarty <57630+mkmccarty@users.noreply.github.com> --- src/boost/boost_datastore.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/boost/boost_datastore.go b/src/boost/boost_datastore.go index a788fdde..ef0bb849 100644 --- a/src/boost/boost_datastore.go +++ b/src/boost/boost_datastore.go @@ -113,8 +113,7 @@ func saveData(contractHash string) { default: // Queue is full, skip this one (it will be retried in the next save cycle) log.Printf("Save queue full, skipping contract: %s", contractHash) - saveQueueMutex.Unlock() - break + delete(pendingSaves, contractHash) // Remove from pending since we couldn't queue it } } saveQueueMutex.Unlock() From 11e80347359f63f95813d7d8c0f3cdc59fafce9a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:49:23 +0000 Subject: [PATCH 04/12] Initial plan From b385d8470582ab39c13537f719be490dca5b5c73 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:49:54 +0000 Subject: [PATCH 05/12] Initial plan From 72f38757e6c88d3667a97601493c2edde6fd1b9e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:50:08 +0000 Subject: [PATCH 06/12] Initial plan From 522cb8a2e1f278370c6854cf16a8abd0338e599f Mon Sep 17 00:00:00 2001 From: Michael McCarty Date: Thu, 29 Jan 2026 17:50:16 -0800 Subject: [PATCH 07/12] Update src/boost/boost_datastore.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/boost/boost_datastore.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/boost/boost_datastore.go b/src/boost/boost_datastore.go index a788fdde..5b05b3a6 100644 --- a/src/boost/boost_datastore.go +++ b/src/boost/boost_datastore.go @@ -132,7 +132,9 @@ func processSingleContractSave(contractHash string) { return } + contract.mutex.Lock() contract.LastSaveTime = time.Now() + contract.mutex.Unlock() saveSqliteData(contract) } From a719037c06c58e6a9fdee3894234f6b5a5737404 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:50:50 +0000 Subject: [PATCH 08/12] Initial plan From 81bf5b6b96fc0d664d88ebd1939271c3f9eca94a Mon Sep 17 00:00:00 2001 From: Michael McCarty Date: Thu, 29 Jan 2026 17:50:58 -0800 Subject: [PATCH 09/12] Update src/boost/boost_datastore.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/boost/boost_datastore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/boost/boost_datastore.go b/src/boost/boost_datastore.go index 5b05b3a6..e03640e2 100644 --- a/src/boost/boost_datastore.go +++ b/src/boost/boost_datastore.go @@ -106,14 +106,14 @@ func saveData(contractHash string) { contractHash := c.ContractHash saveQueueMutex.Lock() if !pendingSaves[contractHash] { - pendingSaves[contractHash] = true select { case saveQueue <- contractHash: // Successfully queued + pendingSaves[contractHash] = true default: // Queue is full, skip this one (it will be retried in the next save cycle) log.Printf("Save queue full, skipping contract: %s", contractHash) - saveQueueMutex.Unlock() + // Do not mark as pending so it can be retried in a future save cycle break } } From a26b90d1c0997edba95bbbf0118fe0917ef531e2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:52:05 +0000 Subject: [PATCH 10/12] Fix double mutex unlock bug by using continue instead of break Co-authored-by: mkmccarty <57630+mkmccarty@users.noreply.github.com> --- src/boost/boost_datastore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/boost/boost_datastore.go b/src/boost/boost_datastore.go index a788fdde..c3904911 100644 --- a/src/boost/boost_datastore.go +++ b/src/boost/boost_datastore.go @@ -114,7 +114,7 @@ func saveData(contractHash string) { // Queue is full, skip this one (it will be retried in the next save cycle) log.Printf("Save queue full, skipping contract: %s", contractHash) saveQueueMutex.Unlock() - break + continue } } saveQueueMutex.Unlock() From 2946caeb4c840963ba9bda5a998b438377cf1b81 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:52:20 +0000 Subject: [PATCH 11/12] Fix race condition by moving pendingSaves deletion after processing Co-authored-by: mkmccarty <57630+mkmccarty@users.noreply.github.com> --- src/boost/boost_datastore.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/boost/boost_datastore.go b/src/boost/boost_datastore.go index a788fdde..b5fc84d5 100644 --- a/src/boost/boost_datastore.go +++ b/src/boost/boost_datastore.go @@ -41,13 +41,15 @@ func sqliteInit() { func startSaveQueueWorker() { go func() { for contractHash := range saveQueue { - // Mark as no longer pending + // Process the actual save + processSingleContractSave(contractHash) + + // Mark as no longer pending after processing completes + // This ensures that if the contract is modified during processing, + // a new save request will be queued rather than being skipped saveQueueMutex.Lock() delete(pendingSaves, contractHash) saveQueueMutex.Unlock() - - // Process the actual save - processSingleContractSave(contractHash) } }() } From b209dca2b2a804fac715a63c5a758e11a56817e2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:58:42 +0000 Subject: [PATCH 12/12] Add mutex protection for contract access during save Co-authored-by: mkmccarty <57630+mkmccarty@users.noreply.github.com> --- src/boost/boost_datastore.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/boost/boost_datastore.go b/src/boost/boost_datastore.go index b5fc84d5..4265d9bf 100644 --- a/src/boost/boost_datastore.go +++ b/src/boost/boost_datastore.go @@ -134,8 +134,11 @@ func processSingleContractSave(contractHash string) { return } + // Lock the contract to ensure thread-safe access during save + contract.mutex.Lock() contract.LastSaveTime = time.Now() saveSqliteData(contract) + contract.mutex.Unlock() } /*