From e4480ecff950940bb52f0b84a8ccb96c7cbf5901 Mon Sep 17 00:00:00 2001 From: Mygod Date: Tue, 31 Mar 2026 15:58:05 -0400 Subject: [PATCH 01/10] Multi-battle support --- config.toml.example | 1 + config/config.go | 1 + config/reader.go | 13 +- decoder/api_fort.go | 32 +- decoder/api_station.go | 118 ++++--- decoder/fortRtree.go | 24 +- decoder/gmo_decode.go | 1 + decoder/main.go | 1 + decoder/preload.go | 21 +- decoder/station.go | 13 + decoder/station_battle.go | 570 +++++++++++++++++++++++++++++++++ decoder/station_battle_test.go | 492 ++++++++++++++++++++++++++++ decoder/station_decode.go | 2 + decoder/station_state.go | 121 ++++--- main.go | 4 + sql/54_station_battle.up.sql | 23 ++ stats.go | 24 ++ 17 files changed, 1342 insertions(+), 119 deletions(-) create mode 100644 decoder/station_battle.go create mode 100644 decoder/station_battle_test.go create mode 100644 sql/54_station_battle.up.sql diff --git a/config.toml.example b/config.toml.example index a3b10a15..30dde492 100644 --- a/config.toml.example +++ b/config.toml.example @@ -15,6 +15,7 @@ bearer_token = "secret" [cleanup] pokemon = true # Keep pokemon table is kept nice and short incidents = true # Remove incidents after expiry +station_battles = true # Remove station battles after expiry quests = true # Remove quests after expiry tappables = true # Remove tappables after expiry stats = true # Enable/Disable stats history diff --git a/config/config.go b/config/config.go index b9af097e..c2c0f988 100644 --- a/config/config.go +++ b/config/config.go @@ -53,6 +53,7 @@ type cleanup struct { Pokemon bool `koanf:"pokemon"` Quests bool `koanf:"quests"` Incidents bool `koanf:"incidents"` + StationBattles bool `koanf:"station_battles"` Tappables bool `koanf:"tappables"` Stats bool `koanf:"stats"` StatsDays int `koanf:"stats_days"` diff --git a/config/reader.go b/config/reader.go index fae0a393..09875fa5 100644 --- a/config/reader.go +++ b/config/reader.go @@ -40,12 +40,13 @@ func ReadConfig() (configDefinition, error) { Compress: true, }, Cleanup: cleanup{ - Pokemon: true, - Quests: true, - Incidents: true, - Tappables: true, - StatsDays: 7, - DeviceHours: 24, + Pokemon: true, + Quests: true, + Incidents: true, + StationBattles: true, + Tappables: true, + StatsDays: 7, + DeviceHours: 24, }, Database: database{ MaxPool: 100, diff --git a/decoder/api_fort.go b/decoder/api_fort.go index 8a06b654..372b0215 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -198,14 +198,34 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn } } else if fortLookup.FortType == STATION { if filter.BattleLevel != nil || filter.BattlePokemon != nil { - // Check if battle has expired - if fortLookup.BattleEndTimestamp <= now { - return false + if len(fortLookup.StationBattles) == 0 { + if fortLookup.BattleEndTimestamp <= now { + return false + } + if filter.BattleLevel != nil && !slices.Contains(filter.BattleLevel, fortLookup.BattleLevel) { + return false + } + if filter.BattlePokemon != nil && !matchDnfIdPair(filter.BattlePokemon, fortLookup.BattlePokemonId, fortLookup.BattlePokemonForm) { + return false + } + return true } - if filter.BattleLevel != nil && !slices.Contains(filter.BattleLevel, fortLookup.BattleLevel) { - return false + + matchedBattle := false + for _, battle := range fortLookup.StationBattles { + if battle.BattleEndTimestamp <= now { + continue + } + if filter.BattleLevel != nil && !slices.Contains(filter.BattleLevel, battle.BattleLevel) { + continue + } + if filter.BattlePokemon != nil && !matchDnfIdPair(filter.BattlePokemon, battle.BattlePokemonId, battle.BattlePokemonForm) { + continue + } + matchedBattle = true + break } - if filter.BattlePokemon != nil && !matchDnfIdPair(filter.BattlePokemon, fortLookup.BattlePokemonId, fortLookup.BattlePokemonForm) { + if !matchedBattle { return false } } diff --git a/decoder/api_station.go b/decoder/api_station.go index 88d9c1ee..04142221 100644 --- a/decoder/api_station.go +++ b/decoder/api_station.go @@ -1,55 +1,81 @@ package decoder -import "github.com/guregu/null/v6" +import ( + "time" + + "github.com/guregu/null/v6" +) type ApiStationResult struct { - Id string `json:"id"` - Lat float64 `json:"lat"` - Lon float64 `json:"lon"` - Name string `json:"name"` - StartTime int64 `json:"start_time"` - EndTime int64 `json:"end_time"` - IsBattleAvailable bool `json:"is_battle_available"` - Updated int64 `json:"updated"` - BattleLevel null.Int `json:"battle_level"` - BattleStart null.Int `json:"battle_start"` - BattleEnd null.Int `json:"battle_end"` - BattlePokemonId null.Int `json:"battle_pokemon_id"` - BattlePokemonForm null.Int `json:"battle_pokemon_form"` - BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` - BattlePokemonGender null.Int `json:"battle_pokemon_gender"` - BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` - BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` - BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` - BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` - TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` - TotalStationedGmax null.Int `json:"total_stationed_gmax"` - StationedPokemon null.String `json:"stationed_pokemon"` + Id string `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + Name string `json:"name"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + IsBattleAvailable bool `json:"is_battle_available"` + Updated int64 `json:"updated"` + BattleLevel null.Int `json:"battle_level"` + BattleStart null.Int `json:"battle_start"` + BattleEnd null.Int `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` + TotalStationedGmax null.Int `json:"total_stationed_gmax"` + StationedPokemon null.String `json:"stationed_pokemon"` + Battles []ApiStationBattle `json:"battles,omitempty"` } func BuildStationResult(station *Station) ApiStationResult { - return ApiStationResult{ - Id: station.Id, - Lat: station.Lat, - Lon: station.Lon, - Name: station.Name, - StartTime: station.StartTime, - EndTime: station.EndTime, - IsBattleAvailable: station.IsBattleAvailable, - Updated: station.Updated, - BattleLevel: station.BattleLevel, - BattleStart: station.BattleStart, - BattleEnd: station.BattleEnd, - BattlePokemonId: station.BattlePokemonId, - BattlePokemonForm: station.BattlePokemonForm, - BattlePokemonCostume: station.BattlePokemonCostume, - BattlePokemonGender: station.BattlePokemonGender, - BattlePokemonAlignment: station.BattlePokemonAlignment, - BattlePokemonBreadMode: station.BattlePokemonBreadMode, - BattlePokemonMove1: station.BattlePokemonMove1, - BattlePokemonMove2: station.BattlePokemonMove2, - TotalStationedPokemon: station.TotalStationedPokemon, - TotalStationedGmax: station.TotalStationedGmax, - StationedPokemon: station.StationedPokemon, + now := time.Now().Unix() + battles := getKnownStationBattles(station.Id, station, now) + canonical := canonicalStationBattleFromSlice(battles, now) + _, hasBattleCache := stationBattleCache.Load(station.Id) + + result := ApiStationResult{ + Id: station.Id, + Lat: station.Lat, + Lon: station.Lon, + Name: station.Name, + StartTime: station.StartTime, + EndTime: station.EndTime, + IsBattleAvailable: station.IsBattleAvailable, + Updated: station.Updated, + TotalStationedPokemon: station.TotalStationedPokemon, + TotalStationedGmax: station.TotalStationedGmax, + StationedPokemon: station.StationedPokemon, + Battles: buildApiStationBattles(station, now), + } + if canonical != nil { + result.BattleLevel = null.IntFrom(int64(canonical.BattleLevel)) + result.BattleStart = null.IntFrom(canonical.BattleStart) + result.BattleEnd = null.IntFrom(canonical.BattleEnd) + result.BattlePokemonId = canonical.BattlePokemonId + result.BattlePokemonForm = canonical.BattlePokemonForm + result.BattlePokemonCostume = canonical.BattlePokemonCostume + result.BattlePokemonGender = canonical.BattlePokemonGender + result.BattlePokemonAlignment = canonical.BattlePokemonAlignment + result.BattlePokemonBreadMode = canonical.BattlePokemonBreadMode + result.BattlePokemonMove1 = canonical.BattlePokemonMove1 + result.BattlePokemonMove2 = canonical.BattlePokemonMove2 + } else if !hasBattleCache { + result.BattleLevel = station.BattleLevel + result.BattleStart = station.BattleStart + result.BattleEnd = station.BattleEnd + result.BattlePokemonId = station.BattlePokemonId + result.BattlePokemonForm = station.BattlePokemonForm + result.BattlePokemonCostume = station.BattlePokemonCostume + result.BattlePokemonGender = station.BattlePokemonGender + result.BattlePokemonAlignment = station.BattlePokemonAlignment + result.BattlePokemonBreadMode = station.BattlePokemonBreadMode + result.BattlePokemonMove1 = station.BattlePokemonMove1 + result.BattlePokemonMove2 = station.BattlePokemonMove2 } + return result } diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 7fb0f44b..39377a83 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -3,6 +3,7 @@ package decoder import ( "encoding/json" "sync" + "time" "github.com/guregu/null/v6" "github.com/puzpuzpuz/xsync/v3" @@ -57,6 +58,7 @@ type FortLookup struct { BattleLevel int8 BattlePokemonId int16 BattlePokemonForm int16 + StationBattles []FortLookupStationBattle } var fortLookupCache *xsync.MapOf[string, FortLookup] @@ -202,14 +204,28 @@ func updateGymLookup(gym *Gym) { } func updateStationLookup(station *Station) { + now := time.Now().Unix() + battles := buildFortLookupStationBattles(station, now) + canonical := canonicalStationBattleFromSlice(getKnownStationBattles(station.Id, station, now), now) + battleEndTimestamp := int64(0) + battleLevel := int8(0) + battlePokemonId := int16(0) + battlePokemonForm := int16(0) + if canonical != nil { + battleEndTimestamp = canonical.BattleEnd + battleLevel = int8(canonical.BattleLevel) + battlePokemonId = int16(canonical.BattlePokemonId.ValueOrZero()) + battlePokemonForm = int16(canonical.BattlePokemonForm.ValueOrZero()) + } fortLookupCache.Store(station.Id, FortLookup{ FortType: STATION, Lat: station.Lat, Lon: station.Lon, - BattleEndTimestamp: station.BattleEnd.ValueOrZero(), - BattleLevel: int8(station.BattleLevel.ValueOrZero()), - BattlePokemonId: int16(station.BattlePokemonId.ValueOrZero()), - BattlePokemonForm: int16(station.BattlePokemonForm.ValueOrZero()), + BattleEndTimestamp: battleEndTimestamp, + BattleLevel: battleLevel, + BattlePokemonId: battlePokemonId, + BattlePokemonForm: battlePokemonForm, + StationBattles: battles, }) } diff --git a/decoder/gmo_decode.go b/decoder/gmo_decode.go index 97ce0dd9..706a9a35 100644 --- a/decoder/gmo_decode.go +++ b/decoder/gmo_decode.go @@ -122,6 +122,7 @@ func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters Sca continue } station.updateFromStationProto(stationProto.Data, stationProto.Cell) + syncStationBattlesFromProto(ctx, db, station, stationProto.Data.BattleDetails) saveStationRecord(ctx, db, station) unlock() } diff --git a/decoder/main.go b/decoder/main.go index 9ba127d1..14a4ab66 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -170,6 +170,7 @@ func initDataCache() { }) initPokemonRtree() initFortRtree() + initStationBattleCache() incidentCache = ttlcache.New[string, *Incident]( ttlcache.WithTTL[string, *Incident](60 * time.Minute), diff --git a/decoder/preload.go b/decoder/preload.go index b14fe7f9..82317b84 100644 --- a/decoder/preload.go +++ b/decoder/preload.go @@ -11,13 +11,13 @@ import ( "golbat/db" ) -// Preload loads forts, stations, active incidents, and recent spawnpoints from DB into cache. +// Preload loads forts, stations, active station battles, active incidents, and recent spawnpoints from DB into cache. // If populateRtree is true, also builds the rtree index for forts. func Preload(dbDetails db.DbDetails, populateRtree bool) { startTime := time.Now() var wg sync.WaitGroup - var pokestopCount, gymCount, stationCount, incidentCount, spawnpointCount int32 + var pokestopCount, gymCount, stationCount, stationBattleCount, incidentCount, spawnpointCount int32 // Phase 1: Load forts and spawnpoints in parallel. // Forts must be loaded before incidents so that the fort lookup cache @@ -41,11 +41,20 @@ func Preload(dbDetails db.DbDetails, populateRtree bool) { }() wg.Wait() - // Phase 2: Load incidents (needs pokestop lookup entries to exist) - incidentCount = preloadIncidents(dbDetails, populateRtree) + // Phase 2: Load child state that depends on forts/stations already being present. + wg.Add(2) + go func() { + defer wg.Done() + stationBattleCount = preloadStationBattles(dbDetails, populateRtree) + }() + go func() { + defer wg.Done() + incidentCount = preloadIncidents(dbDetails, populateRtree) + }() + wg.Wait() - log.Infof("Preload: loaded %d pokestops, %d gyms, %d stations, %d incidents, %d spawnpoints in %v (rtree=%v)", - pokestopCount, gymCount, stationCount, incidentCount, spawnpointCount, time.Since(startTime), populateRtree) + log.Infof("Preload: loaded %d pokestops, %d gyms, %d stations, %d station battles, %d incidents, %d spawnpoints in %v (rtree=%v)", + pokestopCount, gymCount, stationCount, stationBattleCount, incidentCount, spawnpointCount, time.Since(startTime), populateRtree) } // PreloadForts loads all forts from DB into cache. diff --git a/decoder/station.go b/decoder/station.go index 02fb9ed3..691190d4 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -2,6 +2,7 @@ package decoder import ( "fmt" + "time" "github.com/guregu/null/v6" ) @@ -50,6 +51,7 @@ type Station struct { dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) + forceSave bool `db:"-" json:"-"` oldValues StationOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -57,12 +59,14 @@ type Station struct { // StationOldValues holds old field values for webhook comparison type StationOldValues struct { EndTime int64 + CanonicalBattleSeed int64 BattleEnd null.Int BattlePokemonId null.Int BattlePokemonForm null.Int BattlePokemonCostume null.Int BattlePokemonGender null.Int BattlePokemonBreadMode null.Int + BattleListSignature string } // IsDirty returns true if any field has been modified @@ -93,7 +97,11 @@ func (station *Station) Unlock() { // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (station *Station) snapshotOldValues() { + now := time.Now().Unix() + battles := getKnownStationBattles(station.Id, station, now) + canonical := canonicalStationBattleFromSlice(battles, now) station.oldValues = StationOldValues{ + CanonicalBattleSeed: canonicalBattleSeed(canonical), EndTime: station.EndTime, BattleEnd: station.BattleEnd, BattlePokemonId: station.BattlePokemonId, @@ -101,9 +109,14 @@ func (station *Station) snapshotOldValues() { BattlePokemonCostume: station.BattlePokemonCostume, BattlePokemonGender: station.BattlePokemonGender, BattlePokemonBreadMode: station.BattlePokemonBreadMode, + BattleListSignature: stationBattleSignatureFromSlice(battles), } } +func (station *Station) MarkBattleListChanged() { + station.forceSave = true +} + // --- Set methods with dirty tracking --- func (station *Station) SetId(v string) { diff --git a/decoder/station_battle.go b/decoder/station_battle.go new file mode 100644 index 00000000..93d800d3 --- /dev/null +++ b/decoder/station_battle.go @@ -0,0 +1,570 @@ +package decoder + +import ( + "context" + "fmt" + "slices" + "strings" + "time" + + "github.com/guregu/null/v6" + "github.com/puzpuzpuz/xsync/v3" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +type StationBattleData struct { + BreadBattleSeed int64 `db:"bread_battle_seed"` + StationId string `db:"station_id"` + BattleLevel int16 `db:"battle_level"` + BattleStart int64 `db:"battle_start"` + BattleEnd int64 `db:"battle_end"` + BattlePokemonId null.Int `db:"battle_pokemon_id"` + BattlePokemonForm null.Int `db:"battle_pokemon_form"` + BattlePokemonCostume null.Int `db:"battle_pokemon_costume"` + BattlePokemonGender null.Int `db:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `db:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `db:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `db:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `db:"battle_pokemon_move_2"` + BattlePokemonStamina null.Int `db:"battle_pokemon_stamina"` + BattlePokemonCpMultiplier null.Float `db:"battle_pokemon_cp_multiplier"` + Updated int64 `db:"updated"` +} + +type ApiStationBattle struct { + BreadBattleSeed int64 `json:"bread_battle_seed,omitempty"` + BattleLevel int16 `json:"battle_level"` + BattleStart int64 `json:"battle_start"` + BattleEnd int64 `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + BattlePokemonStamina null.Int `json:"battle_pokemon_stamina"` + BattlePokemonCpMultiplier null.Float `json:"battle_pokemon_cp_multiplier"` +} + +type StationBattleWebhook struct { + BreadBattleSeed int64 `json:"bread_battle_seed,omitempty"` + BattleLevel int16 `json:"battle_level"` + BattleStart int64 `json:"battle_start"` + BattleEnd int64 `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + BattlePokemonStamina null.Int `json:"battle_pokemon_stamina"` + BattlePokemonCpMultiplier null.Float `json:"battle_pokemon_cp_multiplier"` +} + +type FortLookupStationBattle struct { + BattleEndTimestamp int64 + BattleLevel int8 + BattlePokemonId int16 + BattlePokemonForm int16 +} + +const stationBattleSelectColumns = `bread_battle_seed, station_id, battle_level, battle_start, battle_end, + battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, + battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, + battle_pokemon_stamina, battle_pokemon_cp_multiplier, updated` + +var stationBattleCache *xsync.MapOf[string, []StationBattleData] +var upsertStationBattleRecordFunc = upsertStationBattleRecord + +func initStationBattleCache() { + stationBattleCache = xsync.NewMapOf[string, []StationBattleData]() +} + +func syncStationBattlesFromProto(ctx context.Context, dbDetails db.DbDetails, station *Station, battleDetail *pogo.BreadBattleDetailProto) { + now := time.Now().Unix() + cacheFresh := battleDetail == nil + if battle := stationBattleFromProto(station.Id, battleDetail, now); battle != nil { + if err := upsertStationBattleRecordFunc(ctx, dbDetails, *battle); err != nil { + log.Errorf("upsert station battle %s/%d: %v", station.Id, battle.BreadBattleSeed, err) + } else if upsertCachedStationBattle(*battle, now) { + station.MarkBattleListChanged() + cacheFresh = true + } else { + cacheFresh = true + } + } else if battleDetail == nil { + cacheFresh = true + } + + if !cacheFresh { + stationBattleCache.Delete(station.Id) + } + + battles := getKnownStationBattles(station.Id, station, now) + applyStationBattleProjection(station, canonicalStationBattleFromSlice(battles, now)) + if station.oldValues.BattleListSignature != stationBattleSignatureFromSlice(battles) { + station.MarkBattleListChanged() + } +} + +func stationBattleFromProto(stationId string, battleDetail *pogo.BreadBattleDetailProto, updated int64) *StationBattleData { + if stationId == "" || battleDetail == nil { + return nil + } + seed := battleDetail.GetBreadBattleSeed() + if seed == 0 { + return nil + } + battle := &StationBattleData{ + BreadBattleSeed: seed, + StationId: stationId, + BattleLevel: int16(battleDetail.GetBattleLevel()), + BattleStart: int64(battleDetail.GetBattleWindowStartMs() / 1000), + BattleEnd: int64(battleDetail.GetBattleWindowEndMs() / 1000), + Updated: updated, + } + if pokemon := battleDetail.GetBattlePokemon(); pokemon != nil { + battle.BattlePokemonId = null.IntFrom(int64(pokemon.GetPokemonId())) + battle.BattlePokemonMove1 = null.IntFrom(int64(pokemon.GetMove1())) + battle.BattlePokemonMove2 = null.IntFrom(int64(pokemon.GetMove2())) + battle.BattlePokemonForm = null.IntFrom(int64(pokemon.GetPokemonDisplay().GetForm())) + battle.BattlePokemonCostume = null.IntFrom(int64(pokemon.GetPokemonDisplay().GetCostume())) + battle.BattlePokemonGender = null.IntFrom(int64(pokemon.GetPokemonDisplay().GetGender())) + battle.BattlePokemonAlignment = null.IntFrom(int64(pokemon.GetPokemonDisplay().GetAlignment())) + battle.BattlePokemonBreadMode = null.IntFrom(int64(pokemon.GetPokemonDisplay().GetBreadModeEnum())) + battle.BattlePokemonStamina = null.IntFrom(int64(pokemon.GetStamina())) + battle.BattlePokemonCpMultiplier = null.FloatFrom(float64(pokemon.GetCpMultiplier())) + } + return battle +} + +func stationBattleFromStationProjection(station *Station) *StationBattleData { + if station == nil || !station.BattleEnd.Valid { + return nil + } + return &StationBattleData{ + StationId: station.Id, + BattleLevel: int16(station.BattleLevel.ValueOrZero()), + BattleStart: station.BattleStart.ValueOrZero(), + BattleEnd: station.BattleEnd.ValueOrZero(), + BattlePokemonId: station.BattlePokemonId, + BattlePokemonForm: station.BattlePokemonForm, + BattlePokemonCostume: station.BattlePokemonCostume, + BattlePokemonGender: station.BattlePokemonGender, + BattlePokemonAlignment: station.BattlePokemonAlignment, + BattlePokemonBreadMode: station.BattlePokemonBreadMode, + BattlePokemonMove1: station.BattlePokemonMove1, + BattlePokemonMove2: station.BattlePokemonMove2, + BattlePokemonStamina: station.BattlePokemonStamina, + BattlePokemonCpMultiplier: station.BattlePokemonCpMultiplier, + Updated: station.Updated, + } +} + +func cloneStationBattles(battles []StationBattleData) []StationBattleData { + if len(battles) == 0 { + return nil + } + return append([]StationBattleData(nil), battles...) +} + +func sortStationBattlesByEnd(battles []StationBattleData) { + slices.SortFunc(battles, func(a, b StationBattleData) int { + if a.BattleEnd != b.BattleEnd { + if a.BattleEnd > b.BattleEnd { + return -1 + } + return 1 + } + if a.BattleStart != b.BattleStart { + if a.BattleStart > b.BattleStart { + return -1 + } + return 1 + } + switch { + case a.BreadBattleSeed > b.BreadBattleSeed: + return -1 + case a.BreadBattleSeed < b.BreadBattleSeed: + return 1 + default: + return 0 + } + }) +} + +func stationBattleIsActive(battle StationBattleData, now int64) bool { + if battle.BattleEnd <= now { + return false + } + if battle.BattleStart == 0 { + return true + } + return battle.BattleStart <= now +} + +func activeStationBattlesFromSlice(battles []StationBattleData, now int64) []StationBattleData { + if len(battles) == 0 { + return nil + } + active := make([]StationBattleData, 0, len(battles)) + for _, battle := range battles { + if stationBattleIsActive(battle, now) { + active = append(active, battle) + } + } + sortStationBattlesByEnd(active) + return active +} + +func nonExpiredStationBattlesFromSlice(battles []StationBattleData, now int64) []StationBattleData { + if len(battles) == 0 { + return nil + } + current := make([]StationBattleData, 0, len(battles)) + for _, battle := range battles { + if battle.BattleEnd > now { + current = append(current, battle) + } + } + sortStationBattlesByEnd(current) + return current +} + +func stationBattlesEqual(a []StationBattleData, b []StationBattleData) bool { + if len(a) != len(b) { + return false + } + for i := range a { + left := a[i] + right := b[i] + left.Updated = 0 + right.Updated = 0 + if left != right { + return false + } + } + return true +} + +func upsertCachedStationBattle(battle StationBattleData, now int64) bool { + existing, _ := stationBattleCache.Load(battle.StationId) + next := make([]StationBattleData, 0, len(existing)+1) + replaced := false + for _, cached := range existing { + if cached.BreadBattleSeed == battle.BreadBattleSeed { + next = append(next, battle) + replaced = true + continue + } + if stationBattleIsActive(cached, now) || cached.BattleStart > now { + next = append(next, cached) + } + } + if !replaced && (stationBattleIsActive(battle, now) || battle.BattleStart > now) { + next = append(next, battle) + } + sortStationBattlesByEnd(next) + if stationBattlesEqual(existing, next) { + return false + } + if len(next) == 0 { + stationBattleCache.Delete(battle.StationId) + } else { + stationBattleCache.Store(battle.StationId, next) + } + return true +} + +func getKnownStationBattles(stationId string, station *Station, now int64) []StationBattleData { + if stationId != "" { + if cached, ok := stationBattleCache.Load(stationId); ok { + current := nonExpiredStationBattlesFromSlice(cached, now) + if !stationBattlesEqual(cached, current) { + if len(current) == 0 { + stationBattleCache.Delete(stationId) + } else { + stationBattleCache.Store(stationId, current) + } + } + if len(current) > 0 { + return cloneStationBattles(current) + } + } + } + if fallback := stationBattleFromStationProjection(station); fallback != nil && fallback.BattleEnd > now { + return []StationBattleData{*fallback} + } + return nil +} + +func getActiveStationBattles(stationId string, station *Station, now int64) []StationBattleData { + return activeStationBattlesFromSlice(getKnownStationBattles(stationId, station, now), now) +} + +func canonicalStationBattleFromSlice(battles []StationBattleData, now int64) *StationBattleData { + if len(battles) == 0 { + return nil + } + for _, battle := range battles { + if stationBattleIsActive(battle, now) { + current := battle + return ¤t + } + } + battle := battles[0] + return &battle +} + +func canonicalBattleSeed(battle *StationBattleData) int64 { + if battle == nil { + return 0 + } + return battle.BreadBattleSeed +} + +func clearStationBattleProjection(station *Station) { + station.SetBattleLevel(null.Int{}) + station.SetBattleStart(null.Int{}) + station.SetBattleEnd(null.Int{}) + station.SetBattlePokemonId(null.Int{}) + station.SetBattlePokemonForm(null.Int{}) + station.SetBattlePokemonCostume(null.Int{}) + station.SetBattlePokemonGender(null.Int{}) + station.SetBattlePokemonAlignment(null.Int{}) + station.SetBattlePokemonBreadMode(null.Int{}) + station.SetBattlePokemonMove1(null.Int{}) + station.SetBattlePokemonMove2(null.Int{}) + station.SetBattlePokemonStamina(null.Int{}) + station.SetBattlePokemonCpMultiplier(null.Float{}) +} + +func applyStationBattleProjection(station *Station, battle *StationBattleData) { + if battle == nil { + clearStationBattleProjection(station) + return + } + station.SetBattleLevel(null.IntFrom(int64(battle.BattleLevel))) + station.SetBattleStart(null.IntFrom(battle.BattleStart)) + station.SetBattleEnd(null.IntFrom(battle.BattleEnd)) + station.SetBattlePokemonId(battle.BattlePokemonId) + station.SetBattlePokemonForm(battle.BattlePokemonForm) + station.SetBattlePokemonCostume(battle.BattlePokemonCostume) + station.SetBattlePokemonGender(battle.BattlePokemonGender) + station.SetBattlePokemonAlignment(battle.BattlePokemonAlignment) + station.SetBattlePokemonBreadMode(battle.BattlePokemonBreadMode) + station.SetBattlePokemonMove1(battle.BattlePokemonMove1) + station.SetBattlePokemonMove2(battle.BattlePokemonMove2) + station.SetBattlePokemonStamina(battle.BattlePokemonStamina) + station.SetBattlePokemonCpMultiplier(battle.BattlePokemonCpMultiplier) +} + +func stationBattleSignature(station *Station, now int64) string { + return stationBattleSignatureFromSlice(getKnownStationBattles(station.Id, station, now)) +} + +func stationBattleSignatureFromSlice(battles []StationBattleData) string { + if len(battles) == 0 { + return "" + } + var builder strings.Builder + for _, battle := range battles { + builder.WriteString(fmt.Sprintf("%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%t:%t;", + battle.BreadBattleSeed, + battle.BattleLevel, + battle.BattleStart, + battle.BattleEnd, + battle.BattlePokemonId.ValueOrZero(), + battle.BattlePokemonForm.ValueOrZero(), + battle.BattlePokemonCostume.ValueOrZero(), + battle.BattlePokemonGender.ValueOrZero(), + battle.BattlePokemonAlignment.ValueOrZero(), + battle.BattlePokemonBreadMode.ValueOrZero(), + battle.BattlePokemonMove1.ValueOrZero(), + battle.BattlePokemonMove2.Valid, + battle.BattlePokemonCpMultiplier.Valid, + )) + builder.WriteString(fmt.Sprintf("%d:%g;", battle.BattlePokemonMove2.ValueOrZero(), battle.BattlePokemonCpMultiplier.ValueOrZero())) + } + return builder.String() +} + +func buildApiStationBattles(station *Station, now int64) []ApiStationBattle { + battles := getKnownStationBattles(station.Id, station, now) + if len(battles) == 0 { + return nil + } + result := make([]ApiStationBattle, 0, len(battles)) + for _, battle := range battles { + result = append(result, ApiStationBattle{ + BreadBattleSeed: battle.BreadBattleSeed, + BattleLevel: battle.BattleLevel, + BattleStart: battle.BattleStart, + BattleEnd: battle.BattleEnd, + BattlePokemonId: battle.BattlePokemonId, + BattlePokemonForm: battle.BattlePokemonForm, + BattlePokemonCostume: battle.BattlePokemonCostume, + BattlePokemonGender: battle.BattlePokemonGender, + BattlePokemonAlignment: battle.BattlePokemonAlignment, + BattlePokemonBreadMode: battle.BattlePokemonBreadMode, + BattlePokemonMove1: battle.BattlePokemonMove1, + BattlePokemonMove2: battle.BattlePokemonMove2, + BattlePokemonStamina: battle.BattlePokemonStamina, + BattlePokemonCpMultiplier: battle.BattlePokemonCpMultiplier, + }) + } + return result +} + +func buildStationWebhookBattles(station *Station, now int64) []StationBattleWebhook { + battles := getKnownStationBattles(station.Id, station, now) + if len(battles) == 0 { + return nil + } + result := make([]StationBattleWebhook, 0, len(battles)) + for _, battle := range battles { + result = append(result, StationBattleWebhook{ + BreadBattleSeed: battle.BreadBattleSeed, + BattleLevel: battle.BattleLevel, + BattleStart: battle.BattleStart, + BattleEnd: battle.BattleEnd, + BattlePokemonId: battle.BattlePokemonId, + BattlePokemonForm: battle.BattlePokemonForm, + BattlePokemonCostume: battle.BattlePokemonCostume, + BattlePokemonGender: battle.BattlePokemonGender, + BattlePokemonAlignment: battle.BattlePokemonAlignment, + BattlePokemonBreadMode: battle.BattlePokemonBreadMode, + BattlePokemonMove1: battle.BattlePokemonMove1, + BattlePokemonMove2: battle.BattlePokemonMove2, + BattlePokemonStamina: battle.BattlePokemonStamina, + BattlePokemonCpMultiplier: battle.BattlePokemonCpMultiplier, + }) + } + return result +} + +func buildFortLookupStationBattles(station *Station, now int64) []FortLookupStationBattle { + battles := getKnownStationBattles(station.Id, station, now) + if len(battles) == 0 { + return nil + } + result := make([]FortLookupStationBattle, 0, len(battles)) + for _, battle := range battles { + result = append(result, FortLookupStationBattle{ + BattleEndTimestamp: battle.BattleEnd, + BattleLevel: int8(battle.BattleLevel), + BattlePokemonId: int16(battle.BattlePokemonId.ValueOrZero()), + BattlePokemonForm: int16(battle.BattlePokemonForm.ValueOrZero()), + }) + } + return result +} + +func upsertStationBattleRecord(ctx context.Context, dbDetails db.DbDetails, battle StationBattleData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, ` + INSERT INTO station_battle ( + bread_battle_seed, station_id, battle_level, battle_start, battle_end, + battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, + battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, + battle_pokemon_stamina, battle_pokemon_cp_multiplier, updated + ) VALUES ( + :bread_battle_seed, :station_id, :battle_level, :battle_start, :battle_end, + :battle_pokemon_id, :battle_pokemon_form, :battle_pokemon_costume, :battle_pokemon_gender, + :battle_pokemon_alignment, :battle_pokemon_bread_mode, :battle_pokemon_move_1, :battle_pokemon_move_2, + :battle_pokemon_stamina, :battle_pokemon_cp_multiplier, :updated + ) + ON DUPLICATE KEY UPDATE + station_id = VALUES(station_id), + battle_level = VALUES(battle_level), + battle_start = VALUES(battle_start), + battle_end = VALUES(battle_end), + battle_pokemon_id = VALUES(battle_pokemon_id), + battle_pokemon_form = VALUES(battle_pokemon_form), + battle_pokemon_costume = VALUES(battle_pokemon_costume), + battle_pokemon_gender = VALUES(battle_pokemon_gender), + battle_pokemon_alignment = VALUES(battle_pokemon_alignment), + battle_pokemon_bread_mode = VALUES(battle_pokemon_bread_mode), + battle_pokemon_move_1 = VALUES(battle_pokemon_move_1), + battle_pokemon_move_2 = VALUES(battle_pokemon_move_2), + battle_pokemon_stamina = VALUES(battle_pokemon_stamina), + battle_pokemon_cp_multiplier = VALUES(battle_pokemon_cp_multiplier), + updated = VALUES(updated) + `, battle) + statsCollector.IncDbQuery("upsert station_battle", err) + return err +} + +func loadStationBattlesForStation(ctx context.Context, dbDetails db.DbDetails, stationId string, now int64) ([]StationBattleData, error) { + var battles []StationBattleData + err := dbDetails.GeneralDb.SelectContext(ctx, &battles, ` + SELECT `+stationBattleSelectColumns+` + FROM station_battle + WHERE station_id = ? AND battle_end > ? + ORDER BY battle_end DESC, battle_start DESC, bread_battle_seed DESC + `, stationId, now) + statsCollector.IncDbQuery("select station_battle station", err) + if err != nil { + return nil, err + } + return battles, nil +} + +func hydrateStationBattlesForStation(ctx context.Context, dbDetails db.DbDetails, stationId string, now int64) error { + if stationId == "" { + return nil + } + battles, err := loadStationBattlesForStation(ctx, dbDetails, stationId, now) + if err != nil { + return err + } + if len(battles) == 0 { + stationBattleCache.Delete(stationId) + return nil + } + stationBattleCache.Store(stationId, battles) + return nil +} + +func preloadStationBattles(dbDetails db.DbDetails, populateRtree bool) int32 { + now := time.Now().Unix() + query := "SELECT " + stationBattleSelectColumns + " FROM station_battle WHERE battle_end > ?" + rows, err := dbDetails.GeneralDb.Queryx(query, now) + statsCollector.IncDbQuery("select station_battle active", err) + if err != nil { + log.Errorf("Preload: failed to query station battles - %s", err) + return 0 + } + defer rows.Close() + + count := int32(0) + affected := make(map[string]struct{}) + for rows.Next() { + var battle StationBattleData + if err := rows.StructScan(&battle); err != nil { + log.Errorf("Preload: station battle scan error - %s", err) + continue + } + upsertCachedStationBattle(battle, now) + affected[battle.StationId] = struct{}{} + count++ + } + + if populateRtree { + for stationId := range affected { + station, unlock, _ := peekStationRecord(stationId, "preloadStationBattles") + if station == nil { + continue + } + updateStationLookup(station) + unlock() + } + } + return count +} diff --git a/decoder/station_battle_test.go b/decoder/station_battle_test.go new file mode 100644 index 00000000..5f5ccbf1 --- /dev/null +++ b/decoder/station_battle_test.go @@ -0,0 +1,492 @@ +package decoder + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/guregu/null/v6" + + "golbat/db" + "golbat/geo" + "golbat/pogo" + "golbat/stats_collector" + "golbat/webhooks" +) + +type recordingWebhooksSender struct { + messages []webhooks.WebhookType +} + +func (sender *recordingWebhooksSender) AddMessage(whType webhooks.WebhookType, _ any, _ []geo.AreaName) { + sender.messages = append(sender.messages, whType) +} + +type recordingStatsCollector struct { + stats_collector.StatsCollector + maxBattleLevels []int64 +} + +func (collector *recordingStatsCollector) UpdateMaxBattleCount(_ []geo.AreaName, level int64) { + collector.maxBattleLevels = append(collector.maxBattleLevels, level) +} + +func TestUpsertCachedStationBattleIgnoresUpdatedOnlyChange(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + battle := StationBattleData{ + BreadBattleSeed: 1, + StationId: "station-1", + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(527), + Updated: now, + } + + if !upsertCachedStationBattle(battle, now) { + t.Fatal("expected first insert to change cache") + } + + battle.Updated = now + 120 + if upsertCachedStationBattle(battle, now) { + t.Fatal("expected updated-only change to be ignored") + } +} + +func TestCanonicalStationBattleUsesLatestEnd(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: "station-1", + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, now) + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 2, + StationId: "station-1", + BattleLevel: 2, + BattleStart: now - 120, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(133), + }, now) + + battles := getKnownStationBattles("station-1", nil, now) + if len(battles) != 2 { + t.Fatalf("expected 2 battles, got %d", len(battles)) + } + if battles[0].BreadBattleSeed != 2 { + t.Fatalf("expected latest-ending battle first, got seed %d", battles[0].BreadBattleSeed) + } + + canonical := canonicalStationBattleFromSlice(battles, now) + if canonical == nil || canonical.BreadBattleSeed != 2 { + t.Fatalf("expected canonical seed 2, got %+v", canonical) + } +} + +func TestBuildStationResultUsesBattleCacheProjection(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + + station := &Station{ + StationData: StationData{ + Id: "station-1", + Name: "Station", + Lat: 1, + Lon: 2, + StartTime: now - 3600, + EndTime: now + 3600, + Updated: now, + BattleLevel: null.IntFrom(1), + BattleStart: null.IntFrom(now - 60), + BattleEnd: null.IntFrom(now + 1800), + BattlePokemonId: null.IntFrom(527), + }, + } + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, now) + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 2, + StationId: station.Id, + BattleLevel: 2, + BattleStart: now - 120, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(133), + }, now) + + result := BuildStationResult(station) + if result.BattlePokemonId.ValueOrZero() != 133 { + t.Fatalf("expected canonical pokemon 133, got %d", result.BattlePokemonId.ValueOrZero()) + } + if len(result.Battles) != 2 { + t.Fatalf("expected 2 battles, got %d", len(result.Battles)) + } +} + +func TestStationFortFilterMatchesSecondaryBattle(t *testing.T) { + now := time.Now().Unix() + filter := ApiFortDnfFilter{ + BattlePokemon: []ApiDnfId{{Pokemon: 133}}, + } + lookup := FortLookup{ + FortType: STATION, + StationBattles: []FortLookupStationBattle{ + {BattleEndTimestamp: now + 1800, BattleLevel: 1, BattlePokemonId: 527}, + {BattleEndTimestamp: now + 7200, BattleLevel: 2, BattlePokemonId: 133}, + }, + } + + if !isFortDnfMatch(STATION, &lookup, &filter, now) { + t.Fatal("expected station filter to match secondary battle") + } +} + +func TestGetActiveStationBattlesKeepsFutureBattleCached(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + future := StationBattleData{ + BreadBattleSeed: 1, + StationId: "station-1", + BattleLevel: 1, + BattleStart: now + 600, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(527), + } + + if !upsertCachedStationBattle(future, now) { + t.Fatal("expected future battle insert to change cache") + } + + if active := getActiveStationBattles("station-1", nil, now); len(active) != 0 { + t.Fatalf("expected no active battles, got %d", len(active)) + } + + cached, ok := stationBattleCache.Load("station-1") + if !ok || len(cached) != 1 { + t.Fatalf("expected future battle to remain cached, got ok=%t len=%d", ok, len(cached)) + } + if cached[0].BreadBattleSeed != future.BreadBattleSeed { + t.Fatalf("expected cached seed %d, got %d", future.BreadBattleSeed, cached[0].BreadBattleSeed) + } +} + +func TestCanonicalStationBattlePrefersActiveOverFuture(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: "station-1", + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, now) + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 2, + StationId: "station-1", + BattleLevel: 2, + BattleStart: now + 600, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(133), + }, now) + + battles := getKnownStationBattles("station-1", nil, now) + canonical := canonicalStationBattleFromSlice(battles, now) + if canonical == nil || canonical.BreadBattleSeed != 1 { + t.Fatalf("expected active battle seed 1 to override future battle, got %+v", canonical) + } +} + +func TestBuildStationResultProjectsFutureBattleFromCache(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + Name: "Station", + Lat: 1, + Lon: 2, + StartTime: now - 3600, + EndTime: now + 3600, + IsBattleAvailable: true, + Updated: now, + BattleLevel: null.IntFrom(1), + BattleStart: null.IntFrom(now + 600), + BattleEnd: null.IntFrom(now + 3600), + BattlePokemonId: null.IntFrom(527), + }, + } + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now + 600, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(527), + }, now) + + result := BuildStationResult(station) + if result.BattlePokemonId.ValueOrZero() != 527 { + t.Fatalf("expected future battle in compatibility fields, got %+v", result) + } + if len(result.Battles) != 1 { + t.Fatalf("expected 1 known battle, got %d", len(result.Battles)) + } + if !result.IsBattleAvailable { + t.Fatal("expected server is_battle_available flag to be preserved") + } +} + +func TestBuildFortLookupStationBattlesIncludesFutureBattle(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + station := &Station{StationData: StationData{Id: "station-1"}} + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now + 600, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(527), + }, now) + + battles := buildFortLookupStationBattles(station, now) + if len(battles) != 1 { + t.Fatalf("expected future battle in fort lookup, got %d", len(battles)) + } + if battles[0].BattlePokemonId != 527 { + t.Fatalf("expected battle pokemon 527, got %d", battles[0].BattlePokemonId) + } +} + +func TestCreateStationWebhooksSkipsEmptyExistingStation(t *testing.T) { + initStationBattleCache() + previousSender := webhooksSender + previousStats := statsCollector + sender := &recordingWebhooksSender{} + webhooksSender = sender + statsCollector = stats_collector.NewNoopStatsCollector() + defer func() { + webhooksSender = previousSender + statsCollector = previousStats + }() + + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + Name: "Station", + Lat: 1, + Lon: 2, + CellId: 123, + EndTime: now + 3600, + Updated: now, + }, + } + station.oldValues = StationOldValues{ + EndTime: now - 3600, + BattleListSignature: "", + } + + createStationWebhooks(station) + if len(sender.messages) != 0 { + t.Fatalf("expected no max_battle webhook, got %v", sender.messages) + } +} + +func TestCreateStationWebhooksEmitsFutureBattle(t *testing.T) { + initStationBattleCache() + previousSender := webhooksSender + previousStats := statsCollector + sender := &recordingWebhooksSender{} + webhooksSender = sender + statsCollector = &recordingStatsCollector{StatsCollector: stats_collector.NewNoopStatsCollector()} + defer func() { + webhooksSender = previousSender + statsCollector = previousStats + }() + + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + Name: "Station", + Lat: 1, + Lon: 2, + CellId: 123, + EndTime: now + 7200, + IsBattleAvailable: false, + Updated: now, + }, + } + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now + 600, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(527), + }, now) + station.oldValues = StationOldValues{ + EndTime: station.EndTime, + BattleListSignature: "", + } + + createStationWebhooks(station) + if len(sender.messages) != 1 || sender.messages[0] != webhooks.MaxBattle { + t.Fatalf("expected one max_battle webhook, got %v", sender.messages) + } +} + +func TestCreateStationWebhooksDoesNotRecountCanonicalBattleSeed(t *testing.T) { + initStationBattleCache() + previousSender := webhooksSender + previousStats := statsCollector + sender := &recordingWebhooksSender{} + collector := &recordingStatsCollector{StatsCollector: stats_collector.NewNoopStatsCollector()} + webhooksSender = sender + statsCollector = collector + defer func() { + webhooksSender = previousSender + statsCollector = previousStats + }() + + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + Name: "Station", + Lat: 1, + Lon: 2, + CellId: 123, + EndTime: now + 7200, + Updated: now, + }, + } + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now - 600, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(527), + }, now) + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 2, + StationId: station.Id, + BattleLevel: 2, + BattleStart: now + 600, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(133), + }, now) + station.oldValues = StationOldValues{ + CanonicalBattleSeed: 1, + EndTime: station.EndTime, + BattleListSignature: "old-signature", + } + + createStationWebhooks(station) + if len(sender.messages) != 1 || sender.messages[0] != webhooks.MaxBattle { + t.Fatalf("expected one max_battle webhook, got %v", sender.messages) + } + if len(collector.maxBattleLevels) != 0 { + t.Fatalf("expected no max battle metric increment, got %v", collector.maxBattleLevels) + } +} + +func TestSyncStationBattlesFromProtoKeepsFreshProjectionWhenSeedMissing(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + BattleLevel: null.IntFrom(2), + BattleStart: null.IntFrom(now - 60), + BattleEnd: null.IntFrom(now + 3600), + BattlePokemonId: null.IntFrom(133), + }, + } + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 99, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(527), + }, now) + + syncStationBattlesFromProto(context.Background(), db.DbDetails{}, station, &pogo.BreadBattleDetailProto{ + BreadBattleSeed: 0, + BattleWindowStartMs: (now - 60) * 1000, + BattleWindowEndMs: (now + 3600) * 1000, + BattleLevel: pogo.BreadBattleLevel_BREAD_BATTLE_LEVEL_2, + }) + + if station.BattlePokemonId.ValueOrZero() != 133 { + t.Fatalf("expected fresh station projection to win, got %d", station.BattlePokemonId.ValueOrZero()) + } + if _, ok := stationBattleCache.Load(station.Id); ok { + t.Fatal("expected stale station battle cache to be cleared") + } +} + +func TestSyncStationBattlesFromProtoKeepsFreshProjectionOnUpsertFailure(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + BattleLevel: null.IntFrom(2), + BattleStart: null.IntFrom(now - 60), + BattleEnd: null.IntFrom(now + 3600), + BattlePokemonId: null.IntFrom(133), + }, + } + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 99, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(527), + }, now) + + previousUpsert := upsertStationBattleRecordFunc + upsertStationBattleRecordFunc = func(context.Context, db.DbDetails, StationBattleData) error { + return errors.New("boom") + } + defer func() { + upsertStationBattleRecordFunc = previousUpsert + }() + + syncStationBattlesFromProto(context.Background(), db.DbDetails{}, station, &pogo.BreadBattleDetailProto{ + BreadBattleSeed: 7, + BattleWindowStartMs: (now - 60) * 1000, + BattleWindowEndMs: (now + 3600) * 1000, + BattleLevel: pogo.BreadBattleLevel_BREAD_BATTLE_LEVEL_2, + BattlePokemon: &pogo.PokemonProto{PokemonId: 133}, + }) + + if station.BattlePokemonId.ValueOrZero() != 133 { + t.Fatalf("expected fresh station projection to win, got %d", station.BattlePokemonId.ValueOrZero()) + } + if _, ok := stationBattleCache.Load(station.Id); ok { + t.Fatal("expected stale station battle cache to be cleared") + } +} diff --git a/decoder/station_decode.go b/decoder/station_decode.go index 6e9f54fa..f042c1e4 100644 --- a/decoder/station_decode.go +++ b/decoder/station_decode.go @@ -48,6 +48,8 @@ func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, log.Infof("[DYNAMAX] Pokemon reward differs from battle: Battle %v - Reward %v", pokemon, rewardPokemon) } } + } else { + clearStationBattleProjection(station) } station.SetCellId(int64(cellId)) return station diff --git a/decoder/station_state.go b/decoder/station_state.go index 2199bee0..3e955c79 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -25,27 +25,28 @@ const stationSelectColumns = `id, lat, lon, name, cell_id, start_time, end_time, stationed_pokemon` type StationWebhook struct { - Id string `json:"id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Name string `json:"name"` - StartTime int64 `json:"start_time"` - EndTime int64 `json:"end_time"` - IsBattleAvailable bool `json:"is_battle_available"` - BattleLevel null.Int `json:"battle_level"` - BattleStart null.Int `json:"battle_start"` - BattleEnd null.Int `json:"battle_end"` - BattlePokemonId null.Int `json:"battle_pokemon_id"` - BattlePokemonForm null.Int `json:"battle_pokemon_form"` - BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` - BattlePokemonGender null.Int `json:"battle_pokemon_gender"` - BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` - BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` - BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` - BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` - TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` - TotalStationedGmax null.Int `json:"total_stationed_gmax"` - Updated int64 `json:"updated"` + Id string `json:"id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + IsBattleAvailable bool `json:"is_battle_available"` + BattleLevel null.Int `json:"battle_level"` + BattleStart null.Int `json:"battle_start"` + BattleEnd null.Int `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` + TotalStationedGmax null.Int `json:"total_stationed_gmax"` + Battles []StationBattleWebhook `json:"battles,omitempty"` + Updated int64 `json:"updated"` } func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId string, station *Station) error { @@ -86,6 +87,9 @@ func GetStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId st return nil, nil, err } dbStation.ClearDirty() + if err := hydrateStationBattlesForStation(ctx, db, stationId, time.Now().Unix()); err != nil { + return nil, nil, err + } // Atomically cache the loaded Station - if another goroutine raced us, // we'll get their Station and use that instead (ensuring same mutex) @@ -135,6 +139,10 @@ func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId st // We loaded from DB station.newRecord = false station.ClearDirty() + if err := hydrateStationBattlesForStation(ctx, db, stationId, time.Now().Unix()); err != nil { + station.Unlock() + return nil, nil, err + } if config.Config.FortInMemory { fortRtreeUpdateStationOnGet(station) } @@ -149,7 +157,7 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { now := time.Now().Unix() // Skip save if not dirty and was updated recently (15-min debounce) - if !station.IsDirty() && !station.IsNewRecord() { + if !station.IsDirty() && !station.IsNewRecord() && !station.forceSave { if station.Updated > now-GetUpdateThreshold(900) { return } @@ -181,6 +189,7 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { station.changedFields = station.changedFields[:0] } station.ClearDirty() + station.forceSave = false createStationWebhooks(station) if isNewRecord { stationCache.Set(station.Id, station, ttlcache.DefaultTTL) @@ -255,39 +264,49 @@ func stationWriteDB(db db.DbDetails, station *Station, isNewRecord bool) error { func createStationWebhooks(station *Station) { old := &station.oldValues isNew := station.IsNewRecord() + now := time.Now().Unix() + currentSignature := stationBattleSignature(station, now) - if isNew || station.BattlePokemonId.Valid && (old.EndTime != station.EndTime || - old.BattleEnd != station.BattleEnd || - old.BattlePokemonId != station.BattlePokemonId || - old.BattlePokemonForm != station.BattlePokemonForm || - old.BattlePokemonCostume != station.BattlePokemonCostume || - old.BattlePokemonGender != station.BattlePokemonGender || - old.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { + if currentSignature == "" { + return + } + + if isNew || old.EndTime != station.EndTime || old.BattleListSignature != currentSignature { + battles := getKnownStationBattles(station.Id, station, now) + canonical := canonicalStationBattleFromSlice(battles, now) + if canonical == nil { + canonical = stationBattleFromStationProjection(station) + } stationHook := StationWebhook{ - Id: station.Id, - Latitude: station.Lat, - Longitude: station.Lon, - Name: station.Name, - StartTime: station.StartTime, - EndTime: station.EndTime, - IsBattleAvailable: station.IsBattleAvailable, - BattleLevel: station.BattleLevel, - BattleStart: station.BattleStart, - BattleEnd: station.BattleEnd, - BattlePokemonId: station.BattlePokemonId, - BattlePokemonForm: station.BattlePokemonForm, - BattlePokemonCostume: station.BattlePokemonCostume, - BattlePokemonGender: station.BattlePokemonGender, - BattlePokemonAlignment: station.BattlePokemonAlignment, - BattlePokemonBreadMode: station.BattlePokemonBreadMode, - BattlePokemonMove1: station.BattlePokemonMove1, - BattlePokemonMove2: station.BattlePokemonMove2, - TotalStationedPokemon: station.TotalStationedPokemon, - TotalStationedGmax: station.TotalStationedGmax, - Updated: station.Updated, + Id: station.Id, + Latitude: station.Lat, + Longitude: station.Lon, + Name: station.Name, + StartTime: station.StartTime, + EndTime: station.EndTime, + IsBattleAvailable: station.IsBattleAvailable, + TotalStationedPokemon: station.TotalStationedPokemon, + TotalStationedGmax: station.TotalStationedGmax, + Battles: buildStationWebhookBattles(station, now), + Updated: station.Updated, + } + if canonical != nil { + stationHook.BattleLevel = null.IntFrom(int64(canonical.BattleLevel)) + stationHook.BattleStart = null.IntFrom(canonical.BattleStart) + stationHook.BattleEnd = null.IntFrom(canonical.BattleEnd) + stationHook.BattlePokemonId = canonical.BattlePokemonId + stationHook.BattlePokemonForm = canonical.BattlePokemonForm + stationHook.BattlePokemonCostume = canonical.BattlePokemonCostume + stationHook.BattlePokemonGender = canonical.BattlePokemonGender + stationHook.BattlePokemonAlignment = canonical.BattlePokemonAlignment + stationHook.BattlePokemonBreadMode = canonical.BattlePokemonBreadMode + stationHook.BattlePokemonMove1 = canonical.BattlePokemonMove1 + stationHook.BattlePokemonMove2 = canonical.BattlePokemonMove2 } areas := MatchStatsGeofenceWithCell(station.Lat, station.Lon, uint64(station.CellId)) webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) - statsCollector.UpdateMaxBattleCount(areas, station.BattleLevel.ValueOrZero()) + if seed := canonicalBattleSeed(canonical); seed != 0 && (isNew || old.CanonicalBattleSeed != seed) { + statsCollector.UpdateMaxBattleCount(areas, int64(canonical.BattleLevel)) + } } } diff --git a/main.go b/main.go index 407da4ca..b54c696a 100644 --- a/main.go +++ b/main.go @@ -241,6 +241,10 @@ func main() { StartIncidentExpiry(db) } + if cfg.Cleanup.StationBattles == true { + StartStationBattleExpiry(db) + } + if cfg.Cleanup.Tappables == true { StartTappableExpiry(db) } diff --git a/sql/54_station_battle.up.sql b/sql/54_station_battle.up.sql new file mode 100644 index 00000000..73051d49 --- /dev/null +++ b/sql/54_station_battle.up.sql @@ -0,0 +1,23 @@ +CREATE TABLE `station_battle` ( + `bread_battle_seed` BIGINT NOT NULL, + `station_id` VARCHAR(35) NOT NULL, + `battle_level` TINYINT UNSIGNED NOT NULL, + `battle_start` INT UNSIGNED NOT NULL, + `battle_end` INT UNSIGNED NOT NULL, + `battle_pokemon_id` SMALLINT unsigned DEFAULT NULL, + `battle_pokemon_form` SMALLINT unsigned DEFAULT NULL, + `battle_pokemon_costume` SMALLINT unsigned DEFAULT NULL, + `battle_pokemon_gender` TINYINT unsigned DEFAULT NULL, + `battle_pokemon_alignment` SMALLINT unsigned DEFAULT NULL, + `battle_pokemon_bread_mode` SMALLINT unsigned DEFAULT NULL, + `battle_pokemon_move_1` SMALLINT unsigned DEFAULT NULL, + `battle_pokemon_move_2` SMALLINT unsigned DEFAULT NULL, + `battle_pokemon_stamina` INT unsigned DEFAULT NULL, + `battle_pokemon_cp_multiplier` FLOAT DEFAULT NULL, + `updated` INT UNSIGNED NOT NULL, + PRIMARY KEY(`bread_battle_seed`), + KEY `ix_station_battle_station_end` (`station_id`, `battle_end`), + KEY `ix_station_battle_end` (`battle_end`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COLLATE = utf8mb4_general_ci; diff --git a/stats.go b/stats.go index f3a2c85d..a569eac3 100644 --- a/stats.go +++ b/stats.go @@ -193,6 +193,30 @@ func StartIncidentExpiry(db *sqlx.DB) { }() } +func StartStationBattleExpiry(db *sqlx.DB) { + ticker := time.NewTicker(time.Hour + 13*time.Minute) + go func() { + for { + <-ticker.C + start := time.Now() + + var result sql.Result + var err error + + result, err = db.Exec("DELETE FROM station_battle WHERE battle_end < UNIX_TIMESTAMP();") + + elapsed := time.Since(start) + + if err != nil { + log.Errorf("DB - Cleanup of station_battle table error %s", err) + } else { + rows, _ := result.RowsAffected() + log.Infof("DB - Cleanup of station_battle table took %s (%d rows)", elapsed, rows) + } + } + }() +} + func StartTappableExpiry(db *sqlx.DB) { ticker := time.NewTicker(time.Hour + 16*time.Minute) go func() { From 1d11eb0ae2f133923390d6494c0cf10a4fe90376 Mon Sep 17 00:00:00 2001 From: Mygod Date: Tue, 31 Mar 2026 16:21:09 -0400 Subject: [PATCH 02/10] Prioritize new data over cached results --- decoder/station_battle.go | 65 ++++++++++++----- decoder/station_battle_test.go | 129 ++++++++++++++++++++++++++++----- 2 files changed, 156 insertions(+), 38 deletions(-) diff --git a/decoder/station_battle.go b/decoder/station_battle.go index 93d800d3..119d228c 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -81,7 +81,7 @@ const stationBattleSelectColumns = `bread_battle_seed, station_id, battle_level, battle_pokemon_stamina, battle_pokemon_cp_multiplier, updated` var stationBattleCache *xsync.MapOf[string, []StationBattleData] -var upsertStationBattleRecordFunc = upsertStationBattleRecord +var upsertStationBattleRecordFunc = storeStationBattleRecord func initStationBattleCache() { stationBattleCache = xsync.NewMapOf[string, []StationBattleData]() @@ -256,21 +256,7 @@ func stationBattlesEqual(a []StationBattleData, b []StationBattleData) bool { func upsertCachedStationBattle(battle StationBattleData, now int64) bool { existing, _ := stationBattleCache.Load(battle.StationId) - next := make([]StationBattleData, 0, len(existing)+1) - replaced := false - for _, cached := range existing { - if cached.BreadBattleSeed == battle.BreadBattleSeed { - next = append(next, battle) - replaced = true - continue - } - if stationBattleIsActive(cached, now) || cached.BattleStart > now { - next = append(next, cached) - } - } - if !replaced && (stationBattleIsActive(battle, now) || battle.BattleStart > now) { - next = append(next, battle) - } + next := pruneObsoleteStationBattles(existing, battle, now) sortStationBattlesByEnd(next) if stationBattlesEqual(existing, next) { return false @@ -283,6 +269,20 @@ func upsertCachedStationBattle(battle StationBattleData, now int64) bool { return true } +func pruneObsoleteStationBattles(existing []StationBattleData, battle StationBattleData, now int64) []StationBattleData { + next := make([]StationBattleData, 0, len(existing)+1) + if battle.BattleEnd > now { + next = append(next, battle) + } + for _, cached := range existing { + if cached.BreadBattleSeed == battle.BreadBattleSeed || cached.BattleEnd <= now || cached.BattleEnd <= battle.BattleEnd { + continue + } + next = append(next, cached) + } + return next +} + func getKnownStationBattles(stationId string, station *Station, now int64) []StationBattleData { if stationId != "" { if cached, ok := stationBattleCache.Load(stationId); ok { @@ -467,8 +467,14 @@ func buildFortLookupStationBattles(station *Station, now int64) []FortLookupStat return result } -func upsertStationBattleRecord(ctx context.Context, dbDetails db.DbDetails, battle StationBattleData) error { - _, err := dbDetails.GeneralDb.NamedExecContext(ctx, ` +func storeStationBattleRecord(ctx context.Context, dbDetails db.DbDetails, battle StationBattleData) error { + tx, err := dbDetails.GeneralDb.BeginTxx(ctx, nil) + statsCollector.IncDbQuery("begin station_battle", err) + if err != nil { + return err + } + + if _, err = tx.NamedExecContext(ctx, ` INSERT INTO station_battle ( bread_battle_seed, station_id, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, @@ -496,8 +502,27 @@ func upsertStationBattleRecord(ctx context.Context, dbDetails db.DbDetails, batt battle_pokemon_stamina = VALUES(battle_pokemon_stamina), battle_pokemon_cp_multiplier = VALUES(battle_pokemon_cp_multiplier), updated = VALUES(updated) - `, battle) - statsCollector.IncDbQuery("upsert station_battle", err) + `, battle); err != nil { + _ = tx.Rollback() + statsCollector.IncDbQuery("upsert station_battle", err) + return err + } + statsCollector.IncDbQuery("upsert station_battle", nil) + + if _, err = tx.ExecContext(ctx, ` + DELETE FROM station_battle + WHERE station_id = ? + AND bread_battle_seed <> ? + AND battle_end <= ? + `, battle.StationId, battle.BreadBattleSeed, battle.BattleEnd); err != nil { + _ = tx.Rollback() + statsCollector.IncDbQuery("delete obsolete station_battle", err) + return err + } + statsCollector.IncDbQuery("delete obsolete station_battle", nil) + + err = tx.Commit() + statsCollector.IncDbQuery("commit station_battle", err) return err } diff --git a/decoder/station_battle_test.go b/decoder/station_battle_test.go index 5f5ccbf1..dc4f2744 100644 --- a/decoder/station_battle_test.go +++ b/decoder/station_battle_test.go @@ -55,6 +55,99 @@ func TestUpsertCachedStationBattleIgnoresUpdatedOnlyChange(t *testing.T) { } } +func TestUpsertCachedStationBattleDropsEarlierEndAfterLaterObservation(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: "station-1", + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, now) + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 2, + StationId: "station-1", + BattleLevel: 2, + BattleStart: now - 60, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(133), + }, now) + + battles := getKnownStationBattles("station-1", nil, now) + if len(battles) != 1 { + t.Fatalf("expected 1 battle after later observation, got %d", len(battles)) + } + if battles[0].BreadBattleSeed != 2 { + t.Fatalf("expected seed 2 to replace earlier battle, got %d", battles[0].BreadBattleSeed) + } +} + +func TestUpsertCachedStationBattleReplacesEqualEndBattle(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: "station-1", + BattleLevel: 1, + BattleStart: now - 120, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(527), + }, now) + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 2, + StationId: "station-1", + BattleLevel: 2, + BattleStart: now - 60, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(133), + }, now) + + battles := getKnownStationBattles("station-1", nil, now) + if len(battles) != 1 { + t.Fatalf("expected 1 battle after equal-end replacement, got %d", len(battles)) + } + if battles[0].BreadBattleSeed != 2 { + t.Fatalf("expected latest equal-end seed 2, got %d", battles[0].BreadBattleSeed) + } +} + +func TestUpsertCachedStationBattleKeepsLongerBattleWhenShorterObserved(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: "station-1", + BattleLevel: 3, + BattleStart: now - 120, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(374), + }, now) + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 2, + StationId: "station-1", + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, now) + + battles := getKnownStationBattles("station-1", nil, now) + if len(battles) != 2 { + t.Fatalf("expected longer and shorter battles to coexist, got %d", len(battles)) + } + if battles[0].BreadBattleSeed != 1 || battles[1].BreadBattleSeed != 2 { + t.Fatalf("unexpected battle ordering after shorter observation: %+v", battles) + } +} + func TestCanonicalStationBattleUsesLatestEnd(t *testing.T) { initStationBattleCache() now := time.Now().Unix() @@ -77,8 +170,8 @@ func TestCanonicalStationBattleUsesLatestEnd(t *testing.T) { }, now) battles := getKnownStationBattles("station-1", nil, now) - if len(battles) != 2 { - t.Fatalf("expected 2 battles, got %d", len(battles)) + if len(battles) != 1 { + t.Fatalf("expected later-ending battle to replace earlier one, got %d battles", len(battles)) } if battles[0].BreadBattleSeed != 2 { t.Fatalf("expected latest-ending battle first, got seed %d", battles[0].BreadBattleSeed) @@ -131,8 +224,8 @@ func TestBuildStationResultUsesBattleCacheProjection(t *testing.T) { if result.BattlePokemonId.ValueOrZero() != 133 { t.Fatalf("expected canonical pokemon 133, got %d", result.BattlePokemonId.ValueOrZero()) } - if len(result.Battles) != 2 { - t.Fatalf("expected 2 battles, got %d", len(result.Battles)) + if len(result.Battles) != 1 { + t.Fatalf("expected 1 battle after later-ending replacement, got %d", len(result.Battles)) } } @@ -183,31 +276,31 @@ func TestGetActiveStationBattlesKeepsFutureBattleCached(t *testing.T) { } } -func TestCanonicalStationBattlePrefersActiveOverFuture(t *testing.T) { +func TestCanonicalStationBattleKeepsLongerBattleWhenShorterFutureObserved(t *testing.T) { initStationBattleCache() now := time.Now().Unix() upsertCachedStationBattle(StationBattleData{ BreadBattleSeed: 1, StationId: "station-1", - BattleLevel: 1, - BattleStart: now - 60, - BattleEnd: now + 1800, - BattlePokemonId: null.IntFrom(527), + BattleLevel: 3, + BattleStart: now - 120, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(374), }, now) upsertCachedStationBattle(StationBattleData{ BreadBattleSeed: 2, StationId: "station-1", BattleLevel: 2, BattleStart: now + 600, - BattleEnd: now + 7200, - BattlePokemonId: null.IntFrom(133), + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), }, now) battles := getKnownStationBattles("station-1", nil, now) canonical := canonicalStationBattleFromSlice(battles, now) if canonical == nil || canonical.BreadBattleSeed != 1 { - t.Fatalf("expected active battle seed 1 to override future battle, got %+v", canonical) + t.Fatalf("expected longer existing battle seed 1 to remain canonical, got %+v", canonical) } } @@ -382,18 +475,18 @@ func TestCreateStationWebhooksDoesNotRecountCanonicalBattleSeed(t *testing.T) { upsertCachedStationBattle(StationBattleData{ BreadBattleSeed: 1, StationId: station.Id, - BattleLevel: 1, + BattleLevel: 3, BattleStart: now - 600, - BattleEnd: now + 3600, - BattlePokemonId: null.IntFrom(527), + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(374), }, now) upsertCachedStationBattle(StationBattleData{ BreadBattleSeed: 2, StationId: station.Id, - BattleLevel: 2, + BattleLevel: 1, BattleStart: now + 600, - BattleEnd: now + 7200, - BattlePokemonId: null.IntFrom(133), + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(527), }, now) station.oldValues = StationOldValues{ CanonicalBattleSeed: 1, From 7c5b030e0270c658595c02c523bfc89f6b61d523 Mon Sep 17 00:00:00 2001 From: Mygod Date: Tue, 31 Mar 2026 17:04:51 -0400 Subject: [PATCH 03/10] Order preloaded station battles deterministically --- decoder/station_battle.go | 35 +++++++++++++++++++++++++++++----- decoder/station_battle_test.go | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/decoder/station_battle.go b/decoder/station_battle.go index 119d228c..a5edd57f 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -557,9 +557,19 @@ func hydrateStationBattlesForStation(ctx context.Context, dbDetails db.DbDetails return nil } +func cachePreloadedStationBattles(stationId string, battles []StationBattleData) bool { + if stationId == "" || len(battles) == 0 { + return false + } + sortStationBattlesByEnd(battles) + stationBattleCache.Store(stationId, battles) + return true +} + func preloadStationBattles(dbDetails db.DbDetails, populateRtree bool) int32 { now := time.Now().Unix() - query := "SELECT " + stationBattleSelectColumns + " FROM station_battle WHERE battle_end > ?" + query := "SELECT " + stationBattleSelectColumns + " FROM station_battle WHERE battle_end > ? " + + "ORDER BY station_id, battle_end DESC, battle_start DESC, bread_battle_seed DESC" rows, err := dbDetails.GeneralDb.Queryx(query, now) statsCollector.IncDbQuery("select station_battle active", err) if err != nil { @@ -569,20 +579,35 @@ func preloadStationBattles(dbDetails db.DbDetails, populateRtree bool) int32 { defer rows.Close() count := int32(0) - affected := make(map[string]struct{}) + affected := make([]string, 0) + currentStationId := "" + currentBattles := make([]StationBattleData, 0) + flushCurrent := func() { + if cachePreloadedStationBattles(currentStationId, currentBattles) { + affected = append(affected, currentStationId) + } + currentStationId = "" + currentBattles = nil + } for rows.Next() { var battle StationBattleData if err := rows.StructScan(&battle); err != nil { log.Errorf("Preload: station battle scan error - %s", err) continue } - upsertCachedStationBattle(battle, now) - affected[battle.StationId] = struct{}{} + if currentStationId != "" && battle.StationId != currentStationId { + flushCurrent() + } + if currentStationId == "" { + currentStationId = battle.StationId + } + currentBattles = append(currentBattles, battle) count++ } + flushCurrent() if populateRtree { - for stationId := range affected { + for _, stationId := range affected { station, unlock, _ := peekStationRecord(stationId, "preloadStationBattles") if station == nil { continue diff --git a/decoder/station_battle_test.go b/decoder/station_battle_test.go index dc4f2744..4eeba8bb 100644 --- a/decoder/station_battle_test.go +++ b/decoder/station_battle_test.go @@ -368,6 +368,40 @@ func TestBuildFortLookupStationBattlesIncludesFutureBattle(t *testing.T) { } } +func TestCachePreloadedStationBattlesPreservesPersistedSetRegardlessOfInputOrder(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + + if !cachePreloadedStationBattles("station-1", []StationBattleData{ + { + BreadBattleSeed: 2, + StationId: "station-1", + BattleLevel: 1, + BattleStart: now + 600, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, + { + BreadBattleSeed: 1, + StationId: "station-1", + BattleLevel: 3, + BattleStart: now - 120, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(374), + }, + }) { + t.Fatal("expected preloaded station battles to be cached") + } + + battles := getKnownStationBattles("station-1", nil, now) + if len(battles) != 2 { + t.Fatalf("expected both persisted battles after preload, got %d", len(battles)) + } + if battles[0].BreadBattleSeed != 1 || battles[1].BreadBattleSeed != 2 { + t.Fatalf("unexpected preloaded battle ordering: %+v", battles) + } +} + func TestCreateStationWebhooksSkipsEmptyExistingStation(t *testing.T) { initStationBattleCache() previousSender := webhooksSender From 69c05adb079813354add88374bf1791624163ec2 Mon Sep 17 00:00:00 2001 From: Mygod Date: Tue, 31 Mar 2026 19:41:12 -0400 Subject: [PATCH 04/10] Clean up --- decoder/station.go | 7 +++ decoder/station_battle.go | 26 ++++---- decoder/station_battle_test.go | 106 +++++++++++++++++++++++++-------- decoder/station_state.go | 7 ++- 4 files changed, 108 insertions(+), 38 deletions(-) diff --git a/decoder/station.go b/decoder/station.go index 691190d4..e5a4d2ae 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -52,6 +52,7 @@ type Station struct { newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) forceSave bool `db:"-" json:"-"` + skipWebhook bool `db:"-" json:"-"` oldValues StationOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -59,7 +60,10 @@ type Station struct { // StationOldValues holds old field values for webhook comparison type StationOldValues struct { EndTime int64 + IsBattleAvailable bool + HasCanonicalBattle bool CanonicalBattleSeed int64 + BattleProjection *StationBattleData BattleEnd null.Int BattlePokemonId null.Int BattlePokemonForm null.Int @@ -101,7 +105,10 @@ func (station *Station) snapshotOldValues() { battles := getKnownStationBattles(station.Id, station, now) canonical := canonicalStationBattleFromSlice(battles, now) station.oldValues = StationOldValues{ + IsBattleAvailable: station.IsBattleAvailable, + HasCanonicalBattle: canonical != nil, CanonicalBattleSeed: canonicalBattleSeed(canonical), + BattleProjection: stationBattleFromStationProjection(station), EndTime: station.EndTime, BattleEnd: station.BattleEnd, BattlePokemonId: station.BattlePokemonId, diff --git a/decoder/station_battle.go b/decoder/station_battle.go index a5edd57f..31eff10f 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -89,22 +89,15 @@ func initStationBattleCache() { func syncStationBattlesFromProto(ctx context.Context, dbDetails db.DbDetails, station *Station, battleDetail *pogo.BreadBattleDetailProto) { now := time.Now().Unix() - cacheFresh := battleDetail == nil if battle := stationBattleFromProto(station.Id, battleDetail, now); battle != nil { if err := upsertStationBattleRecordFunc(ctx, dbDetails, *battle); err != nil { log.Errorf("upsert station battle %s/%d: %v", station.Id, battle.BreadBattleSeed, err) + restoreStationBattleProjectionFromOldValues(station) + station.skipWebhook = true + return } else if upsertCachedStationBattle(*battle, now) { station.MarkBattleListChanged() - cacheFresh = true - } else { - cacheFresh = true } - } else if battleDetail == nil { - cacheFresh = true - } - - if !cacheFresh { - stationBattleCache.Delete(station.Id) } battles := getKnownStationBattles(station.Id, station, now) @@ -119,9 +112,6 @@ func stationBattleFromProto(stationId string, battleDetail *pogo.BreadBattleDeta return nil } seed := battleDetail.GetBreadBattleSeed() - if seed == 0 { - return nil - } battle := &StationBattleData{ BreadBattleSeed: seed, StationId: stationId, @@ -346,6 +336,16 @@ func clearStationBattleProjection(station *Station) { station.SetBattlePokemonCpMultiplier(null.Float{}) } +func restoreStationBattleProjectionFromOldValues(station *Station) { + station.SetIsBattleAvailable(station.oldValues.IsBattleAvailable) + if station.oldValues.BattleProjection == nil { + clearStationBattleProjection(station) + return + } + battle := *station.oldValues.BattleProjection + applyStationBattleProjection(station, &battle) +} + func applyStationBattleProjection(station *Station, battle *StationBattleData) { if battle == nil { clearStationBattleProjection(station) diff --git a/decoder/station_battle_test.go b/decoder/station_battle_test.go index 4eeba8bb..ef6ddb87 100644 --- a/decoder/station_battle_test.go +++ b/decoder/station_battle_test.go @@ -523,6 +523,7 @@ func TestCreateStationWebhooksDoesNotRecountCanonicalBattleSeed(t *testing.T) { BattlePokemonId: null.IntFrom(527), }, now) station.oldValues = StationOldValues{ + HasCanonicalBattle: true, CanonicalBattleSeed: 1, EndTime: station.EndTime, BattleListSignature: "old-signature", @@ -537,59 +538,116 @@ func TestCreateStationWebhooksDoesNotRecountCanonicalBattleSeed(t *testing.T) { } } -func TestSyncStationBattlesFromProtoKeepsFreshProjectionWhenSeedMissing(t *testing.T) { +func TestCreateStationWebhooksCountsZeroSeedCanonicalBattle(t *testing.T) { initStationBattleCache() + previousSender := webhooksSender + previousStats := statsCollector + sender := &recordingWebhooksSender{} + collector := &recordingStatsCollector{StatsCollector: stats_collector.NewNoopStatsCollector()} + webhooksSender = sender + statsCollector = collector + defer func() { + webhooksSender = previousSender + statsCollector = previousStats + }() + now := time.Now().Unix() station := &Station{ StationData: StationData{ - Id: "station-1", - BattleLevel: null.IntFrom(2), - BattleStart: null.IntFrom(now - 60), - BattleEnd: null.IntFrom(now + 3600), - BattlePokemonId: null.IntFrom(133), + Id: "station-1", + Name: "Station", + Lat: 1, + Lon: 2, + CellId: 123, + EndTime: now + 7200, + Updated: now, }, } upsertCachedStationBattle(StationBattleData{ - BreadBattleSeed: 99, + BreadBattleSeed: 0, StationId: station.Id, BattleLevel: 1, - BattleStart: now - 60, - BattleEnd: now + 7200, + BattleStart: now - 600, + BattleEnd: now + 3600, BattlePokemonId: null.IntFrom(527), }, now) + station.oldValues = StationOldValues{ + EndTime: station.EndTime, + BattleListSignature: "", + } + + createStationWebhooks(station) + if len(sender.messages) != 1 || sender.messages[0] != webhooks.MaxBattle { + t.Fatalf("expected one max_battle webhook, got %v", sender.messages) + } + if len(collector.maxBattleLevels) != 1 || collector.maxBattleLevels[0] != 1 { + t.Fatalf("expected one max battle metric increment, got %v", collector.maxBattleLevels) + } +} + +func TestSyncStationBattlesFromProtoAllowsZeroSeed(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + BattleLevel: null.IntFrom(2), + BattleStart: null.IntFrom(now - 60), + BattleEnd: null.IntFrom(now + 3600), + BattlePokemonId: null.IntFrom(133), + }, + } + + previousUpsert := upsertStationBattleRecordFunc + upsertStationBattleRecordFunc = func(context.Context, db.DbDetails, StationBattleData) error { + return nil + } + defer func() { + upsertStationBattleRecordFunc = previousUpsert + }() syncStationBattlesFromProto(context.Background(), db.DbDetails{}, station, &pogo.BreadBattleDetailProto{ BreadBattleSeed: 0, BattleWindowStartMs: (now - 60) * 1000, BattleWindowEndMs: (now + 3600) * 1000, BattleLevel: pogo.BreadBattleLevel_BREAD_BATTLE_LEVEL_2, + BattlePokemon: &pogo.PokemonProto{PokemonId: 133}, }) - if station.BattlePokemonId.ValueOrZero() != 133 { - t.Fatalf("expected fresh station projection to win, got %d", station.BattlePokemonId.ValueOrZero()) + battles := getKnownStationBattles(station.Id, station, now) + if len(battles) != 1 || battles[0].BreadBattleSeed != 0 { + t.Fatalf("expected zero-seed battle to be cached, got %+v", battles) } - if _, ok := stationBattleCache.Load(station.Id); ok { - t.Fatal("expected stale station battle cache to be cleared") + if station.BattlePokemonId.ValueOrZero() != 133 { + t.Fatalf("expected zero-seed battle projection, got %d", station.BattlePokemonId.ValueOrZero()) } } -func TestSyncStationBattlesFromProtoKeepsFreshProjectionOnUpsertFailure(t *testing.T) { +func TestSyncStationBattlesFromProtoRestoresOldProjectionOnUpsertFailure(t *testing.T) { initStationBattleCache() now := time.Now().Unix() station := &Station{ StationData: StationData{ - Id: "station-1", - BattleLevel: null.IntFrom(2), - BattleStart: null.IntFrom(now - 60), - BattleEnd: null.IntFrom(now + 3600), - BattlePokemonId: null.IntFrom(133), + Id: "station-1", + IsBattleAvailable: true, + BattleLevel: null.IntFrom(1), + BattleStart: null.IntFrom(now - 120), + BattleEnd: null.IntFrom(now + 7200), + BattlePokemonId: null.IntFrom(527), }, } + station.snapshotOldValues() + station.SetIsBattleAvailable(true) + station.SetBattleLevel(null.IntFrom(2)) + station.SetBattleStart(null.IntFrom(now - 60)) + station.SetBattleEnd(null.IntFrom(now + 3600)) + station.SetBattlePokemonId(null.IntFrom(133)) + upsertCachedStationBattle(StationBattleData{ BreadBattleSeed: 99, StationId: station.Id, BattleLevel: 1, - BattleStart: now - 60, + BattleStart: now - 120, BattleEnd: now + 7200, BattlePokemonId: null.IntFrom(527), }, now) @@ -610,10 +668,10 @@ func TestSyncStationBattlesFromProtoKeepsFreshProjectionOnUpsertFailure(t *testi BattlePokemon: &pogo.PokemonProto{PokemonId: 133}, }) - if station.BattlePokemonId.ValueOrZero() != 133 { - t.Fatalf("expected fresh station projection to win, got %d", station.BattlePokemonId.ValueOrZero()) + if station.BattlePokemonId.ValueOrZero() != 527 { + t.Fatalf("expected old battle projection to be restored, got %d", station.BattlePokemonId.ValueOrZero()) } - if _, ok := stationBattleCache.Load(station.Id); ok { - t.Fatal("expected stale station battle cache to be cleared") + if !station.skipWebhook { + t.Fatal("expected webhook suppression after failed station battle write") } } diff --git a/decoder/station_state.go b/decoder/station_state.go index 3e955c79..d05cdecf 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -262,6 +262,11 @@ func stationWriteDB(db db.DbDetails, station *Station, isNewRecord bool) error { } func createStationWebhooks(station *Station) { + if station.skipWebhook { + station.skipWebhook = false + return + } + old := &station.oldValues isNew := station.IsNewRecord() now := time.Now().Unix() @@ -305,7 +310,7 @@ func createStationWebhooks(station *Station) { } areas := MatchStatsGeofenceWithCell(station.Lat, station.Lon, uint64(station.CellId)) webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) - if seed := canonicalBattleSeed(canonical); seed != 0 && (isNew || old.CanonicalBattleSeed != seed) { + if seed := canonicalBattleSeed(canonical); canonical != nil && (isNew || !old.HasCanonicalBattle || old.CanonicalBattleSeed != seed) { statsCollector.UpdateMaxBattleCount(areas, int64(canonical.BattleLevel)) } } From e057654add696f261e46ba32bd96a211245adcba Mon Sep 17 00:00:00 2001 From: Mygod Date: Tue, 31 Mar 2026 20:04:10 -0400 Subject: [PATCH 05/10] Handle deadlocks --- decoder/station_battle.go | 25 +++++++++++++++++++- decoder/station_battle_test.go | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/decoder/station_battle.go b/decoder/station_battle.go index 31eff10f..7a885b46 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/go-sql-driver/mysql" "github.com/guregu/null/v6" "github.com/puzpuzpuz/xsync/v3" log "github.com/sirupsen/logrus" @@ -80,6 +81,11 @@ const stationBattleSelectColumns = `bread_battle_seed, station_id, battle_level, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, updated` +const ( + mysqlDeadlockCode = 1213 + stationBattleDeadlockRetries = 3 +) + var stationBattleCache *xsync.MapOf[string, []StationBattleData] var upsertStationBattleRecordFunc = storeStationBattleRecord @@ -90,7 +96,7 @@ func initStationBattleCache() { func syncStationBattlesFromProto(ctx context.Context, dbDetails db.DbDetails, station *Station, battleDetail *pogo.BreadBattleDetailProto) { now := time.Now().Unix() if battle := stationBattleFromProto(station.Id, battleDetail, now); battle != nil { - if err := upsertStationBattleRecordFunc(ctx, dbDetails, *battle); err != nil { + if err := upsertStationBattleRecordWithRetry(ctx, dbDetails, *battle); err != nil { log.Errorf("upsert station battle %s/%d: %v", station.Id, battle.BreadBattleSeed, err) restoreStationBattleProjectionFromOldValues(station) station.skipWebhook = true @@ -107,6 +113,23 @@ func syncStationBattlesFromProto(ctx context.Context, dbDetails db.DbDetails, st } } +func upsertStationBattleRecordWithRetry(ctx context.Context, dbDetails db.DbDetails, battle StationBattleData) error { + var err error + for attempt := 0; attempt <= stationBattleDeadlockRetries; attempt++ { + err = upsertStationBattleRecordFunc(ctx, dbDetails, battle) + if err == nil { + return nil + } + if mysqlErr, ok := err.(*mysql.MySQLError); ok && mysqlErr.Number == mysqlDeadlockCode && attempt < stationBattleDeadlockRetries { + log.Warnf("station_battle deadlock on attempt %d/%d for %s/%d, retrying...", attempt+1, stationBattleDeadlockRetries, battle.StationId, battle.BreadBattleSeed) + time.Sleep(time.Duration(50*(attempt+1)) * time.Millisecond) + continue + } + return err + } + return err +} + func stationBattleFromProto(stationId string, battleDetail *pogo.BreadBattleDetailProto, updated int64) *StationBattleData { if stationId == "" || battleDetail == nil { return nil diff --git a/decoder/station_battle_test.go b/decoder/station_battle_test.go index ef6ddb87..420335d0 100644 --- a/decoder/station_battle_test.go +++ b/decoder/station_battle_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/go-sql-driver/mysql" "github.com/guregu/null/v6" "golbat/db" @@ -675,3 +676,44 @@ func TestSyncStationBattlesFromProtoRestoresOldProjectionOnUpsertFailure(t *test t.Fatal("expected webhook suppression after failed station battle write") } } + +func TestSyncStationBattlesFromProtoRetriesDeadlock(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + }, + } + + attempts := 0 + previousUpsert := upsertStationBattleRecordFunc + upsertStationBattleRecordFunc = func(context.Context, db.DbDetails, StationBattleData) error { + attempts++ + if attempts == 1 { + return &mysql.MySQLError{Number: 1213, Message: "deadlock"} + } + return nil + } + defer func() { + upsertStationBattleRecordFunc = previousUpsert + }() + + syncStationBattlesFromProto(context.Background(), db.DbDetails{}, station, &pogo.BreadBattleDetailProto{ + BreadBattleSeed: 7, + BattleWindowStartMs: (now - 60) * 1000, + BattleWindowEndMs: (now + 3600) * 1000, + BattleLevel: pogo.BreadBattleLevel_BREAD_BATTLE_LEVEL_2, + BattlePokemon: &pogo.PokemonProto{PokemonId: 133}, + }) + + if attempts != 2 { + t.Fatalf("expected one deadlock retry, got %d attempts", attempts) + } + if station.skipWebhook { + t.Fatal("expected retry to succeed without suppressing webhook") + } + if station.BattlePokemonId.ValueOrZero() != 133 { + t.Fatalf("expected battle projection after retry success, got %d", station.BattlePokemonId.ValueOrZero()) + } +} From f0661b6d0f040ffc2ad61e15a4a71497b722d69c Mon Sep 17 00:00:00 2001 From: Mygod Date: Tue, 31 Mar 2026 20:53:21 -0400 Subject: [PATCH 06/10] Less noisy please --- decoder/station_battle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decoder/station_battle.go b/decoder/station_battle.go index 7a885b46..967dfaff 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -121,7 +121,7 @@ func upsertStationBattleRecordWithRetry(ctx context.Context, dbDetails db.DbDeta return nil } if mysqlErr, ok := err.(*mysql.MySQLError); ok && mysqlErr.Number == mysqlDeadlockCode && attempt < stationBattleDeadlockRetries { - log.Warnf("station_battle deadlock on attempt %d/%d for %s/%d, retrying...", attempt+1, stationBattleDeadlockRetries, battle.StationId, battle.BreadBattleSeed) + log.Debugf("station_battle deadlock on attempt %d/%d for %s/%d, retrying...", attempt+1, stationBattleDeadlockRetries, battle.StationId, battle.BreadBattleSeed) time.Sleep(time.Duration(50*(attempt+1)) * time.Millisecond) continue } From a3072da6b9e59fa4bd8d825c19718a77ee08fae4 Mon Sep 17 00:00:00 2001 From: Mygod Date: Wed, 1 Apr 2026 10:12:16 -0400 Subject: [PATCH 07/10] Add AGENTS.md as a symlink --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) create mode 120000 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 120000 index 00000000..681311eb --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file From c46059b12e19d575b9c735a99b73fbc6eec457f7 Mon Sep 17 00:00:00 2001 From: Mygod Date: Wed, 1 Apr 2026 11:08:10 -0400 Subject: [PATCH 08/10] Address Claude review comments --- decoder/api_station.go | 29 ++-- decoder/fortRtree.go | 9 +- decoder/gmo_decode.go | 2 +- decoder/station.go | 15 +- decoder/station_battle.go | 264 ++++++++++++++++++--------------- decoder/station_battle_test.go | 116 +++------------ decoder/station_state.go | 53 ++++--- decoder/writebehind_batch.go | 32 ++-- 8 files changed, 248 insertions(+), 272 deletions(-) diff --git a/decoder/api_station.go b/decoder/api_station.go index 04142221..9a119109 100644 --- a/decoder/api_station.go +++ b/decoder/api_station.go @@ -34,8 +34,7 @@ type ApiStationResult struct { func BuildStationResult(station *Station) ApiStationResult { now := time.Now().Unix() - battles := getKnownStationBattles(station.Id, station, now) - canonical := canonicalStationBattleFromSlice(battles, now) + snapshot := collectStationBattleSnapshot(station, now) _, hasBattleCache := stationBattleCache.Load(station.Id) result := ApiStationResult{ @@ -50,20 +49,20 @@ func BuildStationResult(station *Station) ApiStationResult { TotalStationedPokemon: station.TotalStationedPokemon, TotalStationedGmax: station.TotalStationedGmax, StationedPokemon: station.StationedPokemon, - Battles: buildApiStationBattles(station, now), + Battles: buildApiStationBattlesFromSlice(snapshot.Battles), } - if canonical != nil { - result.BattleLevel = null.IntFrom(int64(canonical.BattleLevel)) - result.BattleStart = null.IntFrom(canonical.BattleStart) - result.BattleEnd = null.IntFrom(canonical.BattleEnd) - result.BattlePokemonId = canonical.BattlePokemonId - result.BattlePokemonForm = canonical.BattlePokemonForm - result.BattlePokemonCostume = canonical.BattlePokemonCostume - result.BattlePokemonGender = canonical.BattlePokemonGender - result.BattlePokemonAlignment = canonical.BattlePokemonAlignment - result.BattlePokemonBreadMode = canonical.BattlePokemonBreadMode - result.BattlePokemonMove1 = canonical.BattlePokemonMove1 - result.BattlePokemonMove2 = canonical.BattlePokemonMove2 + if snapshot.Canonical != nil { + result.BattleLevel = null.IntFrom(int64(snapshot.Canonical.BattleLevel)) + result.BattleStart = null.IntFrom(snapshot.Canonical.BattleStart) + result.BattleEnd = null.IntFrom(snapshot.Canonical.BattleEnd) + result.BattlePokemonId = snapshot.Canonical.BattlePokemonId + result.BattlePokemonForm = snapshot.Canonical.BattlePokemonForm + result.BattlePokemonCostume = snapshot.Canonical.BattlePokemonCostume + result.BattlePokemonGender = snapshot.Canonical.BattlePokemonGender + result.BattlePokemonAlignment = snapshot.Canonical.BattlePokemonAlignment + result.BattlePokemonBreadMode = snapshot.Canonical.BattlePokemonBreadMode + result.BattlePokemonMove1 = snapshot.Canonical.BattlePokemonMove1 + result.BattlePokemonMove2 = snapshot.Canonical.BattlePokemonMove2 } else if !hasBattleCache { result.BattleLevel = station.BattleLevel result.BattleStart = station.BattleStart diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 39377a83..dc6fe54d 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -204,9 +204,12 @@ func updateGymLookup(gym *Gym) { } func updateStationLookup(station *Station) { - now := time.Now().Unix() - battles := buildFortLookupStationBattles(station, now) - canonical := canonicalStationBattleFromSlice(getKnownStationBattles(station.Id, station, now), now) + updateStationLookupFromSnapshot(station, collectStationBattleSnapshot(station, time.Now().Unix())) +} + +func updateStationLookupFromSnapshot(station *Station, snapshot stationBattleSnapshot) { + battles := buildFortLookupStationBattlesFromSlice(snapshot.Battles) + canonical := snapshot.Canonical battleEndTimestamp := int64(0) battleLevel := int8(0) battlePokemonId := int16(0) diff --git a/decoder/gmo_decode.go b/decoder/gmo_decode.go index 706a9a35..3fe8f9b9 100644 --- a/decoder/gmo_decode.go +++ b/decoder/gmo_decode.go @@ -122,7 +122,7 @@ func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters Sca continue } station.updateFromStationProto(stationProto.Data, stationProto.Cell) - syncStationBattlesFromProto(ctx, db, station, stationProto.Data.BattleDetails) + syncStationBattlesFromProto(station, stationProto.Data.BattleDetails) saveStationRecord(ctx, db, station) unlock() } diff --git a/decoder/station.go b/decoder/station.go index e5a4d2ae..00119c6f 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -51,8 +51,6 @@ type Station struct { dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) - forceSave bool `db:"-" json:"-"` - skipWebhook bool `db:"-" json:"-"` oldValues StationOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -102,12 +100,11 @@ func (station *Station) Unlock() { // Call this after loading from cache/DB but before modifications func (station *Station) snapshotOldValues() { now := time.Now().Unix() - battles := getKnownStationBattles(station.Id, station, now) - canonical := canonicalStationBattleFromSlice(battles, now) + snapshot := collectStationBattleSnapshot(station, now) station.oldValues = StationOldValues{ IsBattleAvailable: station.IsBattleAvailable, - HasCanonicalBattle: canonical != nil, - CanonicalBattleSeed: canonicalBattleSeed(canonical), + HasCanonicalBattle: snapshot.Canonical != nil, + CanonicalBattleSeed: canonicalBattleSeed(snapshot.Canonical), BattleProjection: stationBattleFromStationProjection(station), EndTime: station.EndTime, BattleEnd: station.BattleEnd, @@ -116,14 +113,10 @@ func (station *Station) snapshotOldValues() { BattlePokemonCostume: station.BattlePokemonCostume, BattlePokemonGender: station.BattlePokemonGender, BattlePokemonBreadMode: station.BattlePokemonBreadMode, - BattleListSignature: stationBattleSignatureFromSlice(battles), + BattleListSignature: snapshot.Signature, } } -func (station *Station) MarkBattleListChanged() { - station.forceSave = true -} - // --- Set methods with dirty tracking --- func (station *Station) SetId(v string) { diff --git a/decoder/station_battle.go b/decoder/station_battle.go index 967dfaff..e7cc2421 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -2,12 +2,11 @@ package decoder import ( "context" - "fmt" "slices" + "strconv" "strings" "time" - "github.com/go-sql-driver/mysql" "github.com/guregu/null/v6" "github.com/puzpuzpuz/xsync/v3" log "github.com/sirupsen/logrus" @@ -76,58 +75,36 @@ type FortLookupStationBattle struct { BattlePokemonForm int16 } +type StationBattleWrite struct { + StationId string + Battles []StationBattleData +} + +type stationBattleSnapshot struct { + Battles []StationBattleData + Canonical *StationBattleData + Signature string +} + const stationBattleSelectColumns = `bread_battle_seed, station_id, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, updated` -const ( - mysqlDeadlockCode = 1213 - stationBattleDeadlockRetries = 3 -) - var stationBattleCache *xsync.MapOf[string, []StationBattleData] -var upsertStationBattleRecordFunc = storeStationBattleRecord func initStationBattleCache() { stationBattleCache = xsync.NewMapOf[string, []StationBattleData]() } -func syncStationBattlesFromProto(ctx context.Context, dbDetails db.DbDetails, station *Station, battleDetail *pogo.BreadBattleDetailProto) { +func syncStationBattlesFromProto(station *Station, battleDetail *pogo.BreadBattleDetailProto) { now := time.Now().Unix() if battle := stationBattleFromProto(station.Id, battleDetail, now); battle != nil { - if err := upsertStationBattleRecordWithRetry(ctx, dbDetails, *battle); err != nil { - log.Errorf("upsert station battle %s/%d: %v", station.Id, battle.BreadBattleSeed, err) - restoreStationBattleProjectionFromOldValues(station) - station.skipWebhook = true - return - } else if upsertCachedStationBattle(*battle, now) { - station.MarkBattleListChanged() - } + upsertCachedStationBattle(*battle, now) } - battles := getKnownStationBattles(station.Id, station, now) - applyStationBattleProjection(station, canonicalStationBattleFromSlice(battles, now)) - if station.oldValues.BattleListSignature != stationBattleSignatureFromSlice(battles) { - station.MarkBattleListChanged() - } -} - -func upsertStationBattleRecordWithRetry(ctx context.Context, dbDetails db.DbDetails, battle StationBattleData) error { - var err error - for attempt := 0; attempt <= stationBattleDeadlockRetries; attempt++ { - err = upsertStationBattleRecordFunc(ctx, dbDetails, battle) - if err == nil { - return nil - } - if mysqlErr, ok := err.(*mysql.MySQLError); ok && mysqlErr.Number == mysqlDeadlockCode && attempt < stationBattleDeadlockRetries { - log.Debugf("station_battle deadlock on attempt %d/%d for %s/%d, retrying...", attempt+1, stationBattleDeadlockRetries, battle.StationId, battle.BreadBattleSeed) - time.Sleep(time.Duration(50*(attempt+1)) * time.Millisecond) - continue - } - return err - } - return err + snapshot := collectStationBattleSnapshot(station, now) + applyStationBattleProjection(station, snapshot.Canonical) } func stationBattleFromProto(stationId string, battleDetail *pogo.BreadBattleDetailProto, updated int64) *StationBattleData { @@ -233,7 +210,6 @@ func activeStationBattlesFromSlice(battles []StationBattleData, now int64) []Sta active = append(active, battle) } } - sortStationBattlesByEnd(active) return active } @@ -247,7 +223,6 @@ func nonExpiredStationBattlesFromSlice(battles []StationBattleData, now int64) [ current = append(current, battle) } } - sortStationBattlesByEnd(current) return current } @@ -300,15 +275,8 @@ func getKnownStationBattles(stationId string, station *Station, now int64) []Sta if stationId != "" { if cached, ok := stationBattleCache.Load(stationId); ok { current := nonExpiredStationBattlesFromSlice(cached, now) - if !stationBattlesEqual(cached, current) { - if len(current) == 0 { - stationBattleCache.Delete(stationId) - } else { - stationBattleCache.Store(stationId, current) - } - } if len(current) > 0 { - return cloneStationBattles(current) + return current } } } @@ -318,6 +286,15 @@ func getKnownStationBattles(stationId string, station *Station, now int64) []Sta return nil } +func collectStationBattleSnapshot(station *Station, now int64) stationBattleSnapshot { + battles := getKnownStationBattles(station.Id, station, now) + return stationBattleSnapshot{ + Battles: battles, + Canonical: canonicalStationBattleFromSlice(battles, now), + Signature: stationBattleSignatureFromSlice(battles), + } +} + func getActiveStationBattles(stationId string, station *Station, now int64) []StationBattleData { return activeStationBattlesFromSlice(getKnownStationBattles(stationId, station, now), now) } @@ -390,7 +367,7 @@ func applyStationBattleProjection(station *Station, battle *StationBattleData) { } func stationBattleSignature(station *Station, now int64) string { - return stationBattleSignatureFromSlice(getKnownStationBattles(station.Id, station, now)) + return collectStationBattleSnapshot(station, now).Signature } func stationBattleSignatureFromSlice(battles []StationBattleData) string { @@ -399,28 +376,41 @@ func stationBattleSignatureFromSlice(battles []StationBattleData) string { } var builder strings.Builder for _, battle := range battles { - builder.WriteString(fmt.Sprintf("%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%t:%t;", - battle.BreadBattleSeed, - battle.BattleLevel, - battle.BattleStart, - battle.BattleEnd, - battle.BattlePokemonId.ValueOrZero(), - battle.BattlePokemonForm.ValueOrZero(), - battle.BattlePokemonCostume.ValueOrZero(), - battle.BattlePokemonGender.ValueOrZero(), - battle.BattlePokemonAlignment.ValueOrZero(), - battle.BattlePokemonBreadMode.ValueOrZero(), - battle.BattlePokemonMove1.ValueOrZero(), - battle.BattlePokemonMove2.Valid, - battle.BattlePokemonCpMultiplier.Valid, - )) - builder.WriteString(fmt.Sprintf("%d:%g;", battle.BattlePokemonMove2.ValueOrZero(), battle.BattlePokemonCpMultiplier.ValueOrZero())) + builder.WriteString(strconv.FormatInt(battle.BreadBattleSeed, 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(int64(battle.BattleLevel), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattleStart, 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattleEnd, 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattlePokemonId.ValueOrZero(), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattlePokemonForm.ValueOrZero(), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattlePokemonCostume.ValueOrZero(), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattlePokemonGender.ValueOrZero(), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattlePokemonAlignment.ValueOrZero(), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattlePokemonBreadMode.ValueOrZero(), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattlePokemonMove1.ValueOrZero(), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatBool(battle.BattlePokemonMove2.Valid)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatBool(battle.BattlePokemonCpMultiplier.Valid)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatInt(battle.BattlePokemonMove2.ValueOrZero(), 10)) + builder.WriteByte(':') + builder.WriteString(strconv.FormatFloat(battle.BattlePokemonCpMultiplier.ValueOrZero(), 'g', -1, 64)) + builder.WriteByte(';') } return builder.String() } -func buildApiStationBattles(station *Station, now int64) []ApiStationBattle { - battles := getKnownStationBattles(station.Id, station, now) +func buildApiStationBattlesFromSlice(battles []StationBattleData) []ApiStationBattle { if len(battles) == 0 { return nil } @@ -446,8 +436,11 @@ func buildApiStationBattles(station *Station, now int64) []ApiStationBattle { return result } -func buildStationWebhookBattles(station *Station, now int64) []StationBattleWebhook { - battles := getKnownStationBattles(station.Id, station, now) +func buildApiStationBattles(station *Station, now int64) []ApiStationBattle { + return buildApiStationBattlesFromSlice(getKnownStationBattles(station.Id, station, now)) +} + +func buildStationWebhookBattlesFromSlice(battles []StationBattleData) []StationBattleWebhook { if len(battles) == 0 { return nil } @@ -473,8 +466,11 @@ func buildStationWebhookBattles(station *Station, now int64) []StationBattleWebh return result } -func buildFortLookupStationBattles(station *Station, now int64) []FortLookupStationBattle { - battles := getKnownStationBattles(station.Id, station, now) +func buildStationWebhookBattles(station *Station, now int64) []StationBattleWebhook { + return buildStationWebhookBattlesFromSlice(getKnownStationBattles(station.Id, station, now)) +} + +func buildFortLookupStationBattlesFromSlice(battles []StationBattleData) []FortLookupStationBattle { if len(battles) == 0 { return nil } @@ -490,54 +486,75 @@ func buildFortLookupStationBattles(station *Station, now int64) []FortLookupStat return result } -func storeStationBattleRecord(ctx context.Context, dbDetails db.DbDetails, battle StationBattleData) error { +func buildFortLookupStationBattles(station *Station, now int64) []FortLookupStationBattle { + return buildFortLookupStationBattlesFromSlice(getKnownStationBattles(station.Id, station, now)) +} + +func stationBattleWriteFromSlice(stationId string, battles []StationBattleData) StationBattleWrite { + return StationBattleWrite{ + StationId: stationId, + Battles: cloneStationBattles(battles), + } +} + +func storeStationBattleSnapshot(ctx context.Context, dbDetails db.DbDetails, snapshot StationBattleWrite) error { tx, err := dbDetails.GeneralDb.BeginTxx(ctx, nil) statsCollector.IncDbQuery("begin station_battle", err) if err != nil { return err } - if _, err = tx.NamedExecContext(ctx, ` - INSERT INTO station_battle ( - bread_battle_seed, station_id, battle_level, battle_start, battle_end, - battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, - battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, - battle_pokemon_stamina, battle_pokemon_cp_multiplier, updated - ) VALUES ( - :bread_battle_seed, :station_id, :battle_level, :battle_start, :battle_end, - :battle_pokemon_id, :battle_pokemon_form, :battle_pokemon_costume, :battle_pokemon_gender, - :battle_pokemon_alignment, :battle_pokemon_bread_mode, :battle_pokemon_move_1, :battle_pokemon_move_2, - :battle_pokemon_stamina, :battle_pokemon_cp_multiplier, :updated - ) - ON DUPLICATE KEY UPDATE - station_id = VALUES(station_id), - battle_level = VALUES(battle_level), - battle_start = VALUES(battle_start), - battle_end = VALUES(battle_end), - battle_pokemon_id = VALUES(battle_pokemon_id), - battle_pokemon_form = VALUES(battle_pokemon_form), - battle_pokemon_costume = VALUES(battle_pokemon_costume), - battle_pokemon_gender = VALUES(battle_pokemon_gender), - battle_pokemon_alignment = VALUES(battle_pokemon_alignment), - battle_pokemon_bread_mode = VALUES(battle_pokemon_bread_mode), - battle_pokemon_move_1 = VALUES(battle_pokemon_move_1), - battle_pokemon_move_2 = VALUES(battle_pokemon_move_2), - battle_pokemon_stamina = VALUES(battle_pokemon_stamina), - battle_pokemon_cp_multiplier = VALUES(battle_pokemon_cp_multiplier), - updated = VALUES(updated) - `, battle); err != nil { - _ = tx.Rollback() - statsCollector.IncDbQuery("upsert station_battle", err) - return err + if len(snapshot.Battles) > 0 { + if _, err = tx.NamedExecContext(ctx, ` + INSERT INTO station_battle ( + bread_battle_seed, station_id, battle_level, battle_start, battle_end, + battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, + battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, + battle_pokemon_stamina, battle_pokemon_cp_multiplier, updated + ) VALUES ( + :bread_battle_seed, :station_id, :battle_level, :battle_start, :battle_end, + :battle_pokemon_id, :battle_pokemon_form, :battle_pokemon_costume, :battle_pokemon_gender, + :battle_pokemon_alignment, :battle_pokemon_bread_mode, :battle_pokemon_move_1, :battle_pokemon_move_2, + :battle_pokemon_stamina, :battle_pokemon_cp_multiplier, :updated + ) + ON DUPLICATE KEY UPDATE + station_id = VALUES(station_id), + battle_level = VALUES(battle_level), + battle_start = VALUES(battle_start), + battle_end = VALUES(battle_end), + battle_pokemon_id = VALUES(battle_pokemon_id), + battle_pokemon_form = VALUES(battle_pokemon_form), + battle_pokemon_costume = VALUES(battle_pokemon_costume), + battle_pokemon_gender = VALUES(battle_pokemon_gender), + battle_pokemon_alignment = VALUES(battle_pokemon_alignment), + battle_pokemon_bread_mode = VALUES(battle_pokemon_bread_mode), + battle_pokemon_move_1 = VALUES(battle_pokemon_move_1), + battle_pokemon_move_2 = VALUES(battle_pokemon_move_2), + battle_pokemon_stamina = VALUES(battle_pokemon_stamina), + battle_pokemon_cp_multiplier = VALUES(battle_pokemon_cp_multiplier), + updated = VALUES(updated) + `, snapshot.Battles); err != nil { + _ = tx.Rollback() + statsCollector.IncDbQuery("upsert station_battle", err) + return err + } + statsCollector.IncDbQuery("upsert station_battle", nil) } - statsCollector.IncDbQuery("upsert station_battle", nil) - if _, err = tx.ExecContext(ctx, ` - DELETE FROM station_battle - WHERE station_id = ? - AND bread_battle_seed <> ? - AND battle_end <= ? - `, battle.StationId, battle.BreadBattleSeed, battle.BattleEnd); err != nil { + deleteQuery := "DELETE FROM station_battle WHERE station_id = ?" + deleteArgs := []any{snapshot.StationId} + if len(snapshot.Battles) > 0 { + deleteQuery += " AND bread_battle_seed NOT IN (" + for i, battle := range snapshot.Battles { + if i > 0 { + deleteQuery += "," + } + deleteQuery += "?" + deleteArgs = append(deleteArgs, battle.BreadBattleSeed) + } + deleteQuery += ")" + } + if _, err = tx.ExecContext(ctx, deleteQuery, deleteArgs...); err != nil { _ = tx.Rollback() statsCollector.IncDbQuery("delete obsolete station_battle", err) return err @@ -549,6 +566,19 @@ func storeStationBattleRecord(ctx context.Context, dbDetails db.DbDetails, battl return err } +func flushStationBattleBatch(ctx context.Context, dbDetails db.DbDetails, snapshots []StationBattleWrite) error { + var firstErr error + for _, snapshot := range snapshots { + if err := storeStationBattleSnapshot(ctx, dbDetails, snapshot); err != nil { + log.Errorf("flush station_battle %s: %v", snapshot.StationId, err) + if firstErr == nil { + firstErr = err + } + } + } + return firstErr +} + func loadStationBattlesForStation(ctx context.Context, dbDetails db.DbDetails, stationId string, now int64) ([]StationBattleData, error) { var battles []StationBattleData err := dbDetails.GeneralDb.SelectContext(ctx, &battles, ` @@ -564,19 +594,19 @@ func loadStationBattlesForStation(ctx context.Context, dbDetails db.DbDetails, s return battles, nil } -func hydrateStationBattlesForStation(ctx context.Context, dbDetails db.DbDetails, stationId string, now int64) error { - if stationId == "" { +func hydrateStationBattlesForStation(ctx context.Context, dbDetails db.DbDetails, station *Station, now int64) error { + if station == nil || station.Id == "" { return nil } - battles, err := loadStationBattlesForStation(ctx, dbDetails, stationId, now) + battles, err := loadStationBattlesForStation(ctx, dbDetails, station.Id, now) if err != nil { return err } if len(battles) == 0 { - stationBattleCache.Delete(stationId) + stationBattleCache.Delete(station.Id) return nil } - stationBattleCache.Store(stationId, battles) + stationBattleCache.Store(station.Id, battles) return nil } diff --git a/decoder/station_battle_test.go b/decoder/station_battle_test.go index 420335d0..58e04628 100644 --- a/decoder/station_battle_test.go +++ b/decoder/station_battle_test.go @@ -1,15 +1,11 @@ package decoder import ( - "context" - "errors" "testing" "time" - "github.com/go-sql-driver/mysql" "github.com/guregu/null/v6" - "golbat/db" "golbat/geo" "golbat/pogo" "golbat/stats_collector" @@ -599,15 +595,7 @@ func TestSyncStationBattlesFromProtoAllowsZeroSeed(t *testing.T) { }, } - previousUpsert := upsertStationBattleRecordFunc - upsertStationBattleRecordFunc = func(context.Context, db.DbDetails, StationBattleData) error { - return nil - } - defer func() { - upsertStationBattleRecordFunc = previousUpsert - }() - - syncStationBattlesFromProto(context.Background(), db.DbDetails{}, station, &pogo.BreadBattleDetailProto{ + syncStationBattlesFromProto(station, &pogo.BreadBattleDetailProto{ BreadBattleSeed: 0, BattleWindowStartMs: (now - 60) * 1000, BattleWindowEndMs: (now + 3600) * 1000, @@ -624,96 +612,34 @@ func TestSyncStationBattlesFromProtoAllowsZeroSeed(t *testing.T) { } } -func TestSyncStationBattlesFromProtoRestoresOldProjectionOnUpsertFailure(t *testing.T) { +func TestGetKnownStationBattlesDoesNotMutateCacheOnRead(t *testing.T) { initStationBattleCache() now := time.Now().Unix() - station := &Station{ - StationData: StationData{ - Id: "station-1", - IsBattleAvailable: true, - BattleLevel: null.IntFrom(1), - BattleStart: null.IntFrom(now - 120), - BattleEnd: null.IntFrom(now + 7200), - BattlePokemonId: null.IntFrom(527), - }, - } - station.snapshotOldValues() - station.SetIsBattleAvailable(true) - station.SetBattleLevel(null.IntFrom(2)) - station.SetBattleStart(null.IntFrom(now - 60)) - station.SetBattleEnd(null.IntFrom(now + 3600)) - station.SetBattlePokemonId(null.IntFrom(133)) - - upsertCachedStationBattle(StationBattleData{ - BreadBattleSeed: 99, - StationId: station.Id, + expired := StationBattleData{ + BreadBattleSeed: 1, + StationId: "station-1", BattleLevel: 1, - BattleStart: now - 120, - BattleEnd: now + 7200, + BattleStart: now - 7200, + BattleEnd: now - 60, BattlePokemonId: null.IntFrom(527), - }, now) - - previousUpsert := upsertStationBattleRecordFunc - upsertStationBattleRecordFunc = func(context.Context, db.DbDetails, StationBattleData) error { - return errors.New("boom") - } - defer func() { - upsertStationBattleRecordFunc = previousUpsert - }() - - syncStationBattlesFromProto(context.Background(), db.DbDetails{}, station, &pogo.BreadBattleDetailProto{ - BreadBattleSeed: 7, - BattleWindowStartMs: (now - 60) * 1000, - BattleWindowEndMs: (now + 3600) * 1000, - BattleLevel: pogo.BreadBattleLevel_BREAD_BATTLE_LEVEL_2, - BattlePokemon: &pogo.PokemonProto{PokemonId: 133}, - }) - - if station.BattlePokemonId.ValueOrZero() != 527 { - t.Fatalf("expected old battle projection to be restored, got %d", station.BattlePokemonId.ValueOrZero()) } - if !station.skipWebhook { - t.Fatal("expected webhook suppression after failed station battle write") - } -} - -func TestSyncStationBattlesFromProtoRetriesDeadlock(t *testing.T) { - initStationBattleCache() - now := time.Now().Unix() - station := &Station{ - StationData: StationData{ - Id: "station-1", - }, + current := StationBattleData{ + BreadBattleSeed: 2, + StationId: "station-1", + BattleLevel: 2, + BattleStart: now - 60, + BattleEnd: now + 3600, + BattlePokemonId: null.IntFrom(133), } + stationBattleCache.Store("station-1", []StationBattleData{current, expired}) - attempts := 0 - previousUpsert := upsertStationBattleRecordFunc - upsertStationBattleRecordFunc = func(context.Context, db.DbDetails, StationBattleData) error { - attempts++ - if attempts == 1 { - return &mysql.MySQLError{Number: 1213, Message: "deadlock"} - } - return nil + battles := getKnownStationBattles("station-1", nil, now) + if len(battles) != 1 || battles[0].BreadBattleSeed != 2 { + t.Fatalf("expected only current battle from read, got %+v", battles) } - defer func() { - upsertStationBattleRecordFunc = previousUpsert - }() - - syncStationBattlesFromProto(context.Background(), db.DbDetails{}, station, &pogo.BreadBattleDetailProto{ - BreadBattleSeed: 7, - BattleWindowStartMs: (now - 60) * 1000, - BattleWindowEndMs: (now + 3600) * 1000, - BattleLevel: pogo.BreadBattleLevel_BREAD_BATTLE_LEVEL_2, - BattlePokemon: &pogo.PokemonProto{PokemonId: 133}, - }) - if attempts != 2 { - t.Fatalf("expected one deadlock retry, got %d attempts", attempts) - } - if station.skipWebhook { - t.Fatal("expected retry to succeed without suppressing webhook") - } - if station.BattlePokemonId.ValueOrZero() != 133 { - t.Fatalf("expected battle projection after retry success, got %d", station.BattlePokemonId.ValueOrZero()) + cached, ok := stationBattleCache.Load("station-1") + if !ok || len(cached) != 2 { + t.Fatalf("expected cached slice to remain unchanged, got %+v", cached) } } diff --git a/decoder/station_state.go b/decoder/station_state.go index d05cdecf..b52e541e 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -87,21 +87,27 @@ func GetStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId st return nil, nil, err } dbStation.ClearDirty() - if err := hydrateStationBattlesForStation(ctx, db, stationId, time.Now().Unix()); err != nil { - return nil, nil, err - } // Atomically cache the loaded Station - if another goroutine raced us, // we'll get their Station and use that instead (ensuring same mutex) existingStation, _ := stationCache.GetOrSetFunc(stationId, func() *Station { - if config.Config.FortInMemory { - fortRtreeUpdateStationOnGet(&dbStation) - } return &dbStation }) station := existingStation.Value() station.Lock(caller) + loadedFromDb := station == &dbStation + hydratedBattles := false + if _, ok := stationBattleCache.Load(stationId); !ok { + if err := hydrateStationBattlesForStation(ctx, db, station, time.Now().Unix()); err != nil { + station.Unlock() + return nil, nil, err + } + hydratedBattles = true + } + if config.Config.FortInMemory && (loadedFromDb || hydratedBattles) { + fortRtreeUpdateStationOnGet(station) + } return station, func() { station.Unlock() }, nil } @@ -139,7 +145,7 @@ func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId st // We loaded from DB station.newRecord = false station.ClearDirty() - if err := hydrateStationBattlesForStation(ctx, db, stationId, time.Now().Unix()); err != nil { + if err := hydrateStationBattlesForStation(ctx, db, station, time.Now().Unix()); err != nil { station.Unlock() return nil, nil, err } @@ -155,9 +161,11 @@ func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId st func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { now := time.Now().Unix() + snapshot := collectStationBattleSnapshot(station, now) + battleListChanged := station.oldValues.BattleListSignature != snapshot.Signature // Skip save if not dirty and was updated recently (15-min debounce) - if !station.IsDirty() && !station.IsNewRecord() && !station.forceSave { + if !station.IsDirty() && !station.IsNewRecord() && !battleListChanged { if station.Updated > now-GetUpdateThreshold(900) { return } @@ -184,19 +192,26 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { // Fallback to direct write if queue not initialized _ = stationWriteDB(db, station, isNewRecord) } + if battleListChanged { + if stationBattleQueue != nil { + stationBattleQueue.Enqueue(stationBattleWriteFromSlice(station.Id, snapshot.Battles), false, 0) + } else { + _ = storeStationBattleSnapshot(ctx, db, stationBattleWriteFromSlice(station.Id, snapshot.Battles)) + } + } if dbDebugEnabled { station.changedFields = station.changedFields[:0] } station.ClearDirty() - station.forceSave = false - createStationWebhooks(station) + createStationWebhooksWithSnapshot(station, snapshot, isNewRecord) if isNewRecord { stationCache.Set(station.Id, station, ttlcache.DefaultTTL) station.newRecord = false } if config.Config.FortInMemory { - fortRtreeUpdateStationOnSave(station) + genericUpdateFort(station.Id, station.Lat, station.Lon, false) + updateStationLookupFromSnapshot(station, snapshot) } } @@ -262,23 +277,19 @@ func stationWriteDB(db db.DbDetails, station *Station, isNewRecord bool) error { } func createStationWebhooks(station *Station) { - if station.skipWebhook { - station.skipWebhook = false - return - } + createStationWebhooksWithSnapshot(station, collectStationBattleSnapshot(station, time.Now().Unix()), station.IsNewRecord()) +} +func createStationWebhooksWithSnapshot(station *Station, snapshot stationBattleSnapshot, isNew bool) { old := &station.oldValues - isNew := station.IsNewRecord() - now := time.Now().Unix() - currentSignature := stationBattleSignature(station, now) + currentSignature := snapshot.Signature if currentSignature == "" { return } if isNew || old.EndTime != station.EndTime || old.BattleListSignature != currentSignature { - battles := getKnownStationBattles(station.Id, station, now) - canonical := canonicalStationBattleFromSlice(battles, now) + canonical := snapshot.Canonical if canonical == nil { canonical = stationBattleFromStationProjection(station) } @@ -292,7 +303,7 @@ func createStationWebhooks(station *Station) { IsBattleAvailable: station.IsBattleAvailable, TotalStationedPokemon: station.TotalStationedPokemon, TotalStationedGmax: station.TotalStationedGmax, - Battles: buildStationWebhookBattles(station, now), + Battles: buildStationWebhookBattlesFromSlice(snapshot.Battles), Updated: station.Updated, } if canonical != nil { diff --git a/decoder/writebehind_batch.go b/decoder/writebehind_batch.go index 9afac25c..f0c1bb9e 100644 --- a/decoder/writebehind_batch.go +++ b/decoder/writebehind_batch.go @@ -23,15 +23,16 @@ type S2CellData struct { // Typed queues for each entity type - using native key types for efficiency var ( - pokestopQueue *writebehind.TypedQueue[string, PokestopData] - gymQueue *writebehind.TypedQueue[string, GymData] - pokemonQueue *writebehind.TypedQueue[uint64, PokemonData] - spawnpointQueue *writebehind.TypedQueue[int64, SpawnpointData] - routeQueue *writebehind.TypedQueue[string, RouteData] - tappableQueue *writebehind.TypedQueue[uint64, TappableData] - stationQueue *writebehind.TypedQueue[string, StationData] - incidentQueue *writebehind.TypedQueue[string, IncidentData] - s2cellQueue *writebehind.TypedQueue[uint64, S2CellData] + pokestopQueue *writebehind.TypedQueue[string, PokestopData] + gymQueue *writebehind.TypedQueue[string, GymData] + pokemonQueue *writebehind.TypedQueue[uint64, PokemonData] + spawnpointQueue *writebehind.TypedQueue[int64, SpawnpointData] + routeQueue *writebehind.TypedQueue[string, RouteData] + tappableQueue *writebehind.TypedQueue[uint64, TappableData] + stationQueue *writebehind.TypedQueue[string, StationData] + stationBattleQueue *writebehind.TypedQueue[string, StationBattleWrite] + incidentQueue *writebehind.TypedQueue[string, IncidentData] + s2cellQueue *writebehind.TypedQueue[uint64, S2CellData] // QueueManager coordinates all queues queueManager *writebehind.QueueManager @@ -152,6 +153,19 @@ func InitTypedQueues(ctx context.Context, dbDetails db.DbDetails, stats stats_co }) queueManager.Register(stationQueue) + stationBattleQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[string, StationBattleWrite]{ + Name: "station_battle", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushStationBattleBatch, + KeyFunc: func(d StationBattleWrite) string { return d.StationId }, + }) + queueManager.Register(stationBattleQueue) + incidentQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[string, IncidentData]{ Name: "incident", BatchSize: batchSize, From cf516cffe381a03023320d915f3f6722774be770 Mon Sep 17 00:00:00 2001 From: Mygod Date: Wed, 1 Apr 2026 12:27:58 -0400 Subject: [PATCH 09/10] Fixes some issues --- decoder/api_station.go | 4 +- decoder/main.go | 9 +- decoder/station_battle.go | 63 ++++++-- decoder/station_battle_test.go | 277 +++++++++++++++++++++++++++++++++ decoder/station_state.go | 20 ++- 5 files changed, 352 insertions(+), 21 deletions(-) diff --git a/decoder/api_station.go b/decoder/api_station.go index 9a119109..cab8f128 100644 --- a/decoder/api_station.go +++ b/decoder/api_station.go @@ -35,7 +35,7 @@ type ApiStationResult struct { func BuildStationResult(station *Station) ApiStationResult { now := time.Now().Unix() snapshot := collectStationBattleSnapshot(station, now) - _, hasBattleCache := stationBattleCache.Load(station.Id) + hasBattleState := hasHydratedStationBattles(station.Id) result := ApiStationResult{ Id: station.Id, @@ -63,7 +63,7 @@ func BuildStationResult(station *Station) ApiStationResult { result.BattlePokemonBreadMode = snapshot.Canonical.BattlePokemonBreadMode result.BattlePokemonMove1 = snapshot.Canonical.BattlePokemonMove1 result.BattlePokemonMove2 = snapshot.Canonical.BattlePokemonMove2 - } else if !hasBattleCache { + } else if !hasBattleState { result.BattleLevel = station.BattleLevel result.BattleStart = station.BattleStart result.BattleEnd = station.BattleEnd diff --git a/decoder/main.go b/decoder/main.go index 14a4ab66..393e3b04 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -127,12 +127,13 @@ func initDataCache() { TTL: fortCacheTTL, KeyToShard: StringKeyToShard, }) - if config.Config.FortInMemory { - stationCache.OnEviction(func(ctx context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *Station]) { + stationCache.OnEviction(func(ctx context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *Station]) { + clearStationBattleCaches(item.Key()) + if config.Config.FortInMemory { s := item.Value() evictFortFromTree(s.Id, s.Lat, s.Lon) - }) - } + } + }) tappableCache = ttlcache.New[uint64, *Tappable]( ttlcache.WithTTL[uint64, *Tappable](60 * time.Minute), diff --git a/decoder/station_battle.go b/decoder/station_battle.go index e7cc2421..09522a34 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -8,6 +8,7 @@ import ( "time" "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" "github.com/puzpuzpuz/xsync/v3" log "github.com/sirupsen/logrus" @@ -92,9 +93,34 @@ const stationBattleSelectColumns = `bread_battle_seed, station_id, battle_level, battle_pokemon_stamina, battle_pokemon_cp_multiplier, updated` var stationBattleCache *xsync.MapOf[string, []StationBattleData] +var stationBattleHydratedCache *xsync.MapOf[string, struct{}] func initStationBattleCache() { stationBattleCache = xsync.NewMapOf[string, []StationBattleData]() + stationBattleHydratedCache = xsync.NewMapOf[string, struct{}]() +} + +func markStationBattlesHydrated(stationId string) { + if stationId == "" { + return + } + stationBattleHydratedCache.Store(stationId, struct{}{}) +} + +func clearStationBattleCaches(stationId string) { + if stationId == "" { + return + } + stationBattleCache.Delete(stationId) + stationBattleHydratedCache.Delete(stationId) +} + +func hasHydratedStationBattles(stationId string) bool { + if stationId == "" { + return false + } + _, ok := stationBattleHydratedCache.Load(stationId) + return ok } func syncStationBattlesFromProto(station *Station, battleDetail *pogo.BreadBattleDetailProto) { @@ -102,6 +128,7 @@ func syncStationBattlesFromProto(station *Station, battleDetail *pogo.BreadBattl if battle := stationBattleFromProto(station.Id, battleDetail, now); battle != nil { upsertCachedStationBattle(*battle, now) } + markStationBattlesHydrated(station.Id) snapshot := collectStationBattleSnapshot(station, now) applyStationBattleProjection(station, snapshot.Canonical) @@ -278,6 +305,13 @@ func getKnownStationBattles(stationId string, station *Station, now int64) []Sta if len(current) > 0 { return current } + stationBattleCache.Delete(stationId) + if hasHydratedStationBattles(stationId) { + return nil + } + } + if hasHydratedStationBattles(stationId) { + return nil } } if fallback := stationBattleFromStationProjection(station); fallback != nil && fallback.BattleEnd > now { @@ -604,9 +638,11 @@ func hydrateStationBattlesForStation(ctx context.Context, dbDetails db.DbDetails } if len(battles) == 0 { stationBattleCache.Delete(station.Id) + markStationBattlesHydrated(station.Id) return nil } stationBattleCache.Store(station.Id, battles) + markStationBattlesHydrated(station.Id) return nil } @@ -616,9 +652,24 @@ func cachePreloadedStationBattles(stationId string, battles []StationBattleData) } sortStationBattlesByEnd(battles) stationBattleCache.Store(stationId, battles) + markStationBattlesHydrated(stationId) return true } +func markPreloadedStationsHydrated(populateRtree bool) { + stationCache.Range(func(item *ttlcache.Item[string, *Station]) bool { + stationId := item.Key() + markStationBattlesHydrated(stationId) + if populateRtree { + station := item.Value() + station.Lock("preloadStationBattles") + fortRtreeUpdateStationOnSave(station) + station.Unlock() + } + return true + }) +} + func preloadStationBattles(dbDetails db.DbDetails, populateRtree bool) int32 { now := time.Now().Unix() query := "SELECT " + stationBattleSelectColumns + " FROM station_battle WHERE battle_end > ? " + @@ -659,15 +710,7 @@ func preloadStationBattles(dbDetails db.DbDetails, populateRtree bool) int32 { } flushCurrent() - if populateRtree { - for _, stationId := range affected { - station, unlock, _ := peekStationRecord(stationId, "preloadStationBattles") - if station == nil { - continue - } - updateStationLookup(station) - unlock() - } - } + markPreloadedStationsHydrated(populateRtree) + return count } diff --git a/decoder/station_battle_test.go b/decoder/station_battle_test.go index 58e04628..6bb696dc 100644 --- a/decoder/station_battle_test.go +++ b/decoder/station_battle_test.go @@ -1,11 +1,16 @@ package decoder import ( + "context" + "errors" "testing" "time" "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" + "golbat/config" + "golbat/db" "golbat/geo" "golbat/pogo" "golbat/stats_collector" @@ -643,3 +648,275 @@ func TestGetKnownStationBattlesDoesNotMutateCacheOnRead(t *testing.T) { t.Fatalf("expected cached slice to remain unchanged, got %+v", cached) } } + +func TestBuildStationResultSuppressesStaleProjectionAfterExpiredHydratedCache(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + Name: "Station", + Lat: 1, + Lon: 2, + StartTime: now - 3600, + EndTime: now + 3600, + Updated: now, + BattleLevel: null.IntFrom(1), + BattleStart: null.IntFrom(now - 600), + BattleEnd: null.IntFrom(now + 600), + BattlePokemonId: null.IntFrom(527), + }, + } + markStationBattlesHydrated(station.Id) + + stationBattleCache.Store("station-1", []StationBattleData{{ + BreadBattleSeed: 1, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now - 7200, + BattleEnd: now - 60, + BattlePokemonId: null.IntFrom(527), + }}) + + result := BuildStationResult(station) + if result.BattleEnd.Valid || result.BattlePokemonId.Valid { + t.Fatalf("expected expired hydrated cache to suppress stale projection, got %+v", result) + } + if _, ok := stationBattleCache.Load(station.Id); ok { + t.Fatal("expected expired hydrated cache entry to be cleaned up") + } +} + +func TestGetStationRecordReadOnlyRetriesHydrationOnCachedStation(t *testing.T) { + initStationBattleCache() + stationId := "station-hydration-retry" + station := &Station{StationData: StationData{Id: stationId}} + stationCache.Set(stationId, station, ttlcache.DefaultTTL) + defer stationCache.Delete(stationId) + defer clearStationBattleCaches(stationId) + + attempts := 0 + previousHydrate := hydrateStationBattlesForStationFunc + hydrateStationBattlesForStationFunc = func(_ context.Context, _ db.DbDetails, station *Station, _ int64) error { + attempts++ + if attempts == 1 { + return errors.New("boom") + } + markStationBattlesHydrated(station.Id) + return nil + } + defer func() { + hydrateStationBattlesForStationFunc = previousHydrate + }() + + record, unlock, err := GetStationRecordReadOnly(context.Background(), db.DbDetails{}, stationId, "test") + if err != nil { + t.Fatalf("expected cached station to be served even when hydration fails, got %v", err) + } + if record == nil || unlock == nil { + t.Fatal("expected cached station record on hydration failure") + } + unlock() + + record, unlock, err = GetStationRecordReadOnly(context.Background(), db.DbDetails{}, stationId, "test") + if err != nil { + t.Fatalf("expected second hydration attempt to succeed, got %v", err) + } + if record == nil || unlock == nil { + t.Fatal("expected cached station record after retry") + } + unlock() + if attempts != 2 { + t.Fatalf("expected hydration retry on cached station, got %d attempts", attempts) + } +} + +func TestGetStationRecordReadOnlyKeepsSingletonAfterHydrationFailureOnCacheMiss(t *testing.T) { + initStationBattleCache() + stationId := "station-hydration-miss-retry" + defer stationCache.Delete(stationId) + defer clearStationBattleCaches(stationId) + + loadCalls := 0 + previousLoad := loadStationFromDatabaseFunc + loadStationFromDatabaseFunc = func(_ context.Context, _ db.DbDetails, id string, station *Station) error { + loadCalls++ + station.Id = id + station.Name = "Station" + return nil + } + defer func() { + loadStationFromDatabaseFunc = previousLoad + }() + + hydrateCalls := 0 + previousHydrate := hydrateStationBattlesForStationFunc + hydrateStationBattlesForStationFunc = func(_ context.Context, _ db.DbDetails, station *Station, _ int64) error { + hydrateCalls++ + if hydrateCalls == 1 { + return errors.New("boom") + } + markStationBattlesHydrated(station.Id) + return nil + } + defer func() { + hydrateStationBattlesForStationFunc = previousHydrate + }() + + record, unlock, err := GetStationRecordReadOnly(context.Background(), db.DbDetails{}, stationId, "test") + if err == nil { + if unlock != nil { + unlock() + } + t.Fatal("expected first cache-miss hydration to fail") + } + if record != nil || unlock != nil { + t.Fatal("expected no station return on failed cache-miss hydration") + } + + cachedItem := stationCache.Get(stationId) + if cachedItem == nil { + t.Fatal("expected failed hydration to keep cached station instance") + } + cachedStation := cachedItem.Value() + + record, unlock, err = GetStationRecordReadOnly(context.Background(), db.DbDetails{}, stationId, "test") + if err != nil { + t.Fatalf("expected retry on cached singleton to succeed, got %v", err) + } + if record == nil || unlock == nil { + t.Fatal("expected cached station record after retry") + } + if record != cachedStation { + unlock() + t.Fatal("expected retry to reuse cached station singleton") + } + unlock() + + if loadCalls != 1 { + t.Fatalf("expected one DB load across retry, got %d", loadCalls) + } + if hydrateCalls != 2 { + t.Fatalf("expected two hydration attempts across retry, got %d", hydrateCalls) + } +} + +func TestGetStationRecordReadOnlySkipsHydrationAfterProtoSync(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + stationId := "station-hydration-skip" + station := &Station{StationData: StationData{Id: stationId}} + stationCache.Set(stationId, station, ttlcache.DefaultTTL) + defer stationCache.Delete(stationId) + defer clearStationBattleCaches(stationId) + + syncStationBattlesFromProto(station, &pogo.BreadBattleDetailProto{ + BreadBattleSeed: 7, + BattleWindowStartMs: (now - 60) * 1000, + BattleWindowEndMs: (now + 3600) * 1000, + BattleLevel: pogo.BreadBattleLevel_BREAD_BATTLE_LEVEL_2, + BattlePokemon: &pogo.PokemonProto{PokemonId: 133}, + }) + + attempts := 0 + previousHydrate := hydrateStationBattlesForStationFunc + hydrateStationBattlesForStationFunc = func(_ context.Context, _ db.DbDetails, _ *Station, _ int64) error { + attempts++ + return nil + } + defer func() { + hydrateStationBattlesForStationFunc = previousHydrate + }() + + record, unlock, err := GetStationRecordReadOnly(context.Background(), db.DbDetails{}, stationId, "test") + if err != nil { + t.Fatalf("expected cached station read to succeed, got %v", err) + } + if record == nil || unlock == nil { + t.Fatal("expected cached station record") + } + unlock() + if attempts != 0 { + t.Fatalf("expected no DB hydration after proto sync, got %d attempts", attempts) + } +} + +func TestMarkPreloadedStationsHydratedMarksEmptyStations(t *testing.T) { + initStationBattleCache() + stationId := "station-preload-empty" + station := &Station{StationData: StationData{Id: stationId}} + stationCache.Set(stationId, station, ttlcache.DefaultTTL) + defer stationCache.Delete(stationId) + defer clearStationBattleCaches(stationId) + + if hasHydratedStationBattles(stationId) { + t.Fatal("expected station to start unhydrated") + } + + markPreloadedStationsHydrated(false) + + if !hasHydratedStationBattles(stationId) { + t.Fatal("expected empty preloaded station to be marked hydrated") + } +} + +func TestGetStationRecordReadOnlyHydrationRefreshesFortLookup(t *testing.T) { + initStationBattleCache() + previousFortInMemory := config.Config.FortInMemory + config.Config.FortInMemory = true + defer func() { + config.Config.FortInMemory = previousFortInMemory + }() + + now := time.Now().Unix() + stationId := "station-hydration-lookup" + station := &Station{ + StationData: StationData{ + Id: stationId, + Lat: 1, + Lon: 2, + BattleLevel: null.IntFrom(1), + BattleStart: null.IntFrom(now - 600), + BattleEnd: null.IntFrom(now + 600), + BattlePokemonId: null.IntFrom(527), + }, + } + stationCache.Set(stationId, station, ttlcache.DefaultTTL) + defer stationCache.Delete(stationId) + defer clearStationBattleCaches(stationId) + fortLookupCache.Store(stationId, FortLookup{ + FortType: STATION, + Lat: station.Lat, + Lon: station.Lon, + BattleEndTimestamp: station.BattleEnd.ValueOrZero(), + BattleLevel: int8(station.BattleLevel.ValueOrZero()), + BattlePokemonId: int16(station.BattlePokemonId.ValueOrZero()), + }) + + previousHydrate := hydrateStationBattlesForStationFunc + hydrateStationBattlesForStationFunc = func(_ context.Context, _ db.DbDetails, station *Station, _ int64) error { + markStationBattlesHydrated(station.Id) + stationBattleCache.Delete(station.Id) + return nil + } + defer func() { + hydrateStationBattlesForStationFunc = previousHydrate + }() + + record, unlock, err := GetStationRecordReadOnly(context.Background(), db.DbDetails{}, stationId, "test") + if err != nil { + t.Fatalf("expected hydration to succeed, got %v", err) + } + if record == nil || unlock == nil { + t.Fatal("expected cached station") + } + unlock() + + lookup, ok := fortLookupCache.Load(stationId) + if !ok { + t.Fatal("expected fort lookup entry") + } + if lookup.BattleEndTimestamp != 0 || lookup.BattleLevel != 0 || lookup.BattlePokemonId != 0 { + t.Fatalf("expected fort lookup to be cleared after hydration, got %+v", lookup) + } +} diff --git a/decoder/station_state.go b/decoder/station_state.go index b52e541e..2c10c1a5 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -56,6 +56,9 @@ func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId str return err } +var loadStationFromDatabaseFunc = loadStationFromDatabase +var hydrateStationBattlesForStationFunc = hydrateStationBattlesForStation + // peekStationRecord - cache-only lookup, no DB fallback, returns locked. // Caller MUST call returned unlock function if non-nil. func peekStationRecord(stationId string, caller string) (*Station, func(), error) { @@ -75,11 +78,18 @@ func GetStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId st if item := stationCache.Get(stationId); item != nil { station := item.Value() station.Lock(caller) + if !hasHydratedStationBattles(stationId) { + if err := hydrateStationBattlesForStationFunc(ctx, db, station, time.Now().Unix()); err != nil { + log.Debugf("GetStationRecordReadOnly: station battle hydration failed for %s: %v", stationId, err) + } else if config.Config.FortInMemory { + fortRtreeUpdateStationOnSave(station) + } + } return station, func() { station.Unlock() }, nil } dbStation := Station{} - err := loadStationFromDatabase(ctx, db, stationId, &dbStation) + err := loadStationFromDatabaseFunc(ctx, db, stationId, &dbStation) if errors.Is(err, sql.ErrNoRows) { return nil, nil, nil } @@ -98,8 +108,8 @@ func GetStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId st station.Lock(caller) loadedFromDb := station == &dbStation hydratedBattles := false - if _, ok := stationBattleCache.Load(stationId); !ok { - if err := hydrateStationBattlesForStation(ctx, db, station, time.Now().Unix()); err != nil { + if !hasHydratedStationBattles(stationId) { + if err := hydrateStationBattlesForStationFunc(ctx, db, station, time.Now().Unix()); err != nil { station.Unlock() return nil, nil, err } @@ -135,7 +145,7 @@ func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId st if station.newRecord { // We should attempt to load from database - err := loadStationFromDatabase(ctx, db, stationId, station) + err := loadStationFromDatabaseFunc(ctx, db, stationId, station) if err != nil { if !errors.Is(err, sql.ErrNoRows) { station.Unlock() @@ -145,7 +155,7 @@ func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId st // We loaded from DB station.newRecord = false station.ClearDirty() - if err := hydrateStationBattlesForStation(ctx, db, station, time.Now().Unix()); err != nil { + if err := hydrateStationBattlesForStationFunc(ctx, db, station, time.Now().Unix()); err != nil { station.Unlock() return nil, nil, err } From ec613e46765aeb939b72d0b21a9e060d20d9c082 Mon Sep 17 00:00:00 2001 From: Mygod Date: Mon, 6 Apr 2026 01:04:38 -0400 Subject: [PATCH 10/10] Pick currently active battle for canonical battle in the station row --- decoder/station_battle.go | 32 ++++++++++++++- decoder/station_battle_test.go | 71 +++++++++++++++++++++++++++------- 2 files changed, 86 insertions(+), 17 deletions(-) diff --git a/decoder/station_battle.go b/decoder/station_battle.go index 09522a34..16a67a67 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -324,7 +324,7 @@ func collectStationBattleSnapshot(station *Station, now int64) stationBattleSnap battles := getKnownStationBattles(station.Id, station, now) return stationBattleSnapshot{ Battles: battles, - Canonical: canonicalStationBattleFromSlice(battles, now), + Canonical: canonicalStationBattleFromSlice(station, battles, now), Signature: stationBattleSignatureFromSlice(battles), } } @@ -333,16 +333,44 @@ func getActiveStationBattles(stationId string, station *Station, now int64) []St return activeStationBattlesFromSlice(getKnownStationBattles(stationId, station, now), now) } -func canonicalStationBattleFromSlice(battles []StationBattleData, now int64) *StationBattleData { +func stationBattleMatchesProjection(battle StationBattleData, projection *StationBattleData) bool { + if projection == nil { + return false + } + return battle.BattleLevel == projection.BattleLevel && + battle.BattleStart == projection.BattleStart && + battle.BattleEnd == projection.BattleEnd && + battle.BattlePokemonId == projection.BattlePokemonId && + battle.BattlePokemonForm == projection.BattlePokemonForm +} + +func canonicalStationBattleFromSlice(station *Station, battles []StationBattleData, now int64) *StationBattleData { if len(battles) == 0 { return nil } + projection := stationBattleFromStationProjection(station) + if projection != nil && stationBattleIsActive(*projection, now) { + for _, battle := range battles { + if stationBattleIsActive(battle, now) && stationBattleMatchesProjection(battle, projection) { + current := battle + return ¤t + } + } + } for _, battle := range battles { if stationBattleIsActive(battle, now) { current := battle return ¤t } } + if projection != nil { + for _, battle := range battles { + if stationBattleMatchesProjection(battle, projection) { + current := battle + return ¤t + } + } + } battle := battles[0] return &battle } diff --git a/decoder/station_battle_test.go b/decoder/station_battle_test.go index 6bb696dc..9efaf67a 100644 --- a/decoder/station_battle_test.go +++ b/decoder/station_battle_test.go @@ -179,13 +179,13 @@ func TestCanonicalStationBattleUsesLatestEnd(t *testing.T) { t.Fatalf("expected latest-ending battle first, got seed %d", battles[0].BreadBattleSeed) } - canonical := canonicalStationBattleFromSlice(battles, now) + canonical := canonicalStationBattleFromSlice(nil, battles, now) if canonical == nil || canonical.BreadBattleSeed != 2 { t.Fatalf("expected canonical seed 2, got %+v", canonical) } } -func TestBuildStationResultUsesBattleCacheProjection(t *testing.T) { +func TestBuildStationResultPrefersCurrentActiveProjection(t *testing.T) { initStationBattleCache() now := time.Now().Unix() @@ -205,14 +205,6 @@ func TestBuildStationResultUsesBattleCacheProjection(t *testing.T) { }, } - upsertCachedStationBattle(StationBattleData{ - BreadBattleSeed: 1, - StationId: station.Id, - BattleLevel: 1, - BattleStart: now - 60, - BattleEnd: now + 1800, - BattlePokemonId: null.IntFrom(527), - }, now) upsertCachedStationBattle(StationBattleData{ BreadBattleSeed: 2, StationId: station.Id, @@ -221,13 +213,21 @@ func TestBuildStationResultUsesBattleCacheProjection(t *testing.T) { BattleEnd: now + 7200, BattlePokemonId: null.IntFrom(133), }, now) + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, now) result := BuildStationResult(station) - if result.BattlePokemonId.ValueOrZero() != 133 { - t.Fatalf("expected canonical pokemon 133, got %d", result.BattlePokemonId.ValueOrZero()) + if result.BattlePokemonId.ValueOrZero() != 527 { + t.Fatalf("expected current active pokemon 527, got %d", result.BattlePokemonId.ValueOrZero()) } - if len(result.Battles) != 1 { - t.Fatalf("expected 1 battle after later-ending replacement, got %d", len(result.Battles)) + if len(result.Battles) != 2 { + t.Fatalf("expected both battles to remain known, got %d", len(result.Battles)) } } @@ -300,12 +300,53 @@ func TestCanonicalStationBattleKeepsLongerBattleWhenShorterFutureObserved(t *tes }, now) battles := getKnownStationBattles("station-1", nil, now) - canonical := canonicalStationBattleFromSlice(battles, now) + canonical := canonicalStationBattleFromSlice(nil, battles, now) if canonical == nil || canonical.BreadBattleSeed != 1 { t.Fatalf("expected longer existing battle seed 1 to remain canonical, got %+v", canonical) } } +func TestCanonicalStationBattlePrefersCurrentActiveProjection(t *testing.T) { + initStationBattleCache() + now := time.Now().Unix() + station := &Station{ + StationData: StationData{ + Id: "station-1", + BattleLevel: null.IntFrom(1), + BattleStart: null.IntFrom(now - 60), + BattleEnd: null.IntFrom(now + 1800), + BattlePokemonId: null.IntFrom(527), + }, + } + + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 2, + StationId: station.Id, + BattleLevel: 2, + BattleStart: now - 120, + BattleEnd: now + 7200, + BattlePokemonId: null.IntFrom(133), + }, now) + upsertCachedStationBattle(StationBattleData{ + BreadBattleSeed: 1, + StationId: station.Id, + BattleLevel: 1, + BattleStart: now - 60, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, now) + + battles := getKnownStationBattles(station.Id, station, now) + if len(battles) != 2 { + t.Fatalf("expected both active battles to remain known, got %d", len(battles)) + } + + canonical := canonicalStationBattleFromSlice(station, battles, now) + if canonical == nil || canonical.BreadBattleSeed != 1 { + t.Fatalf("expected current station projection seed 1 to be canonical, got %+v", canonical) + } +} + func TestBuildStationResultProjectsFutureBattleFromCache(t *testing.T) { initStationBattleCache() now := time.Now().Unix()