From 215fcb2d1dff59d3cac40a6cf65938c40b61f235 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 25 Jan 2026 16:15:11 +0000 Subject: [PATCH 01/78] Webhooks to structures --- decoder/fort.go | 29 ++++--- decoder/gym.go | 126 +++++++++++++++++-------------- decoder/incident.go | 71 +++++++++++------- decoder/pokemon.go | 168 ++++++++++++++++++++++++----------------- decoder/pokestop.go | 179 +++++++++++++++++++++++++------------------- decoder/station.go | 68 +++++++++++------ decoder/weather.go | 51 +++++++++---- routes.go | 12 +-- 8 files changed, 421 insertions(+), 283 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index ed28030f..cb17aa73 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -27,6 +27,13 @@ type FortWebhook struct { Location Location `json:"location"` } +type FortChangeWebhook struct { + ChangeType string `json:"change_type"` + EditTypes []string `json:"edit_types,omitempty"` + Old *FortWebhook `json:"old,omitempty"` + New *FortWebhook `json:"new,omitempty"` +} + type FortChange string type FortType string @@ -129,17 +136,17 @@ func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []strin func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { if change == NEW { areas := MatchStatsGeofence(new.Location.Latitude, new.Location.Longitude) - hook := map[string]interface{}{ - "change_type": change.String(), - "new": new, + hook := FortChangeWebhook{ + ChangeType: change.String(), + New: new, } webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) statsCollector.UpdateFortCount(areas, new.Type, "addition") } else if change == REMOVAL { areas := MatchStatsGeofence(old.Location.Latitude, old.Location.Longitude) - hook := map[string]interface{}{ - "change_type": change.String(), - "old": old, + hook := FortChangeWebhook{ + ChangeType: change.String(), + Old: old, } webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) statsCollector.UpdateFortCount(areas, new.Type, "removal") @@ -181,11 +188,11 @@ func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { editTypes = append(editTypes, "location") } if len(editTypes) > 0 { - hook := map[string]interface{}{ - "change_type": change.String(), - "edit_types": editTypes, - "old": old, - "new": new, + hook := FortChangeWebhook{ + ChangeType: change.String(), + EditTypes: editTypes, + Old: old, + New: new, } webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) statsCollector.UpdateFortCount(areas, new.Type, "edit") diff --git a/decoder/gym.go b/decoder/gym.go index 4f1784fe..a1f3b06b 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -505,23 +505,37 @@ type GymDetailsWebhook struct { PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` ArScanEligible int64 `json:"ar_scan_eligible"` Defenders any `json:"defenders"` +} - //"id": id, - //"name": name ?? "Unknown", - //"url": url ?? "", - //"latitude": lat, - //"longitude": lon, - //"team": teamId ?? 0, - //"guard_pokemon_id": guardPokemonId ?? 0, - //"slots_available": availableSlots ?? 6, - //"ex_raid_eligible": exRaidEligible ?? 0, - //"in_battle": inBattle ?? false, - //"sponsor_id": sponsorId ?? 0, - //"partner_id": partnerId ?? 0, - //"power_up_points": powerUpPoints ?? 0, - //"power_up_level": powerUpLevel ?? 0, - //"power_up_end_timestamp": powerUpEndTimestamp ?? 0, - //"ar_scan_eligible": arScanEligible ?? 0 +type RaidWebhook struct { + GymId string `json:"gym_id"` + GymName string `json:"gym_name"` + GymUrl string `json:"gym_url"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + TeamId int64 `json:"team_id"` + Spawn int64 `json:"spawn"` + Start int64 `json:"start"` + End int64 `json:"end"` + Level int64 `json:"level"` + PokemonId int64 `json:"pokemon_id"` + Cp int64 `json:"cp"` + Gender int64 `json:"gender"` + Form int64 `json:"form"` + Alignment int64 `json:"alignment"` + Costume int64 `json:"costume"` + Evolution int64 `json:"evolution"` + Move1 int64 `json:"move_1"` + Move2 int64 `json:"move_2"` + ExRaidEligible int64 `json:"ex_raid_eligible"` + IsExclusive int64 `json:"is_exclusive"` + SponsorId int64 `json:"sponsor_id"` + PartnerId string `json:"partner_id"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + ArScanEligible int64 `json:"ar_scan_eligible"` + Rsvps json.RawMessage `json:"rsvps"` } func createGymFortWebhooks(oldGym *Gym, gym *Gym) { @@ -576,47 +590,45 @@ func createGymWebhooks(oldGym *Gym, gym *Gym, areas []geo.AreaName) { if (raidBattleTime > now && gym.RaidLevel.ValueOrZero() > 0) || (raidEndTime > now && gym.RaidPokemonId.ValueOrZero() > 0) { - raidHook := map[string]any{ - "gym_id": gym.Id, - "gym_name": func() string { - if !gym.Name.Valid { - return "Unknown" - } else { - return gym.Name.String - } - }(), - "gym_url": gym.Url.ValueOrZero(), - "latitude": gym.Lat, - "longitude": gym.Lon, - "team_id": gym.TeamId.ValueOrZero(), - "spawn": gym.RaidSpawnTimestamp.ValueOrZero(), - "start": gym.RaidBattleTimestamp.ValueOrZero(), - "end": gym.RaidEndTimestamp.ValueOrZero(), - "level": gym.RaidLevel.ValueOrZero(), - "pokemon_id": gym.RaidPokemonId.ValueOrZero(), - "cp": gym.RaidPokemonCp.ValueOrZero(), - "gender": gym.RaidPokemonGender.ValueOrZero(), - "form": gym.RaidPokemonForm.ValueOrZero(), - "alignment": gym.RaidPokemonAlignment.ValueOrZero(), - "costume": gym.RaidPokemonCostume.ValueOrZero(), - "evolution": gym.RaidPokemonEvolution.ValueOrZero(), - "move_1": gym.RaidPokemonMove1.ValueOrZero(), - "move_2": gym.RaidPokemonMove2.ValueOrZero(), - "ex_raid_eligible": gym.ExRaidEligible.ValueOrZero(), - "is_exclusive": gym.RaidIsExclusive.ValueOrZero(), - "sponsor_id": gym.SponsorId.ValueOrZero(), - "partner_id": gym.PartnerId.ValueOrZero(), - "power_up_points": gym.PowerUpPoints.ValueOrZero(), - "power_up_level": gym.PowerUpLevel.ValueOrZero(), - "power_up_end_timestamp": gym.PowerUpEndTimestamp.ValueOrZero(), - "ar_scan_eligible": gym.ArScanEligible.ValueOrZero(), - "rsvps": func() any { - if !gym.Rsvps.Valid { - return nil - } else { - return json.RawMessage(gym.Rsvps.ValueOrZero()) - } - }(), + gymName := "Unknown" + if gym.Name.Valid { + gymName = gym.Name.String + } + + var rsvps json.RawMessage + if gym.Rsvps.Valid { + rsvps = json.RawMessage(gym.Rsvps.ValueOrZero()) + } + + raidHook := RaidWebhook{ + GymId: gym.Id, + GymName: gymName, + GymUrl: gym.Url.ValueOrZero(), + Latitude: gym.Lat, + Longitude: gym.Lon, + TeamId: gym.TeamId.ValueOrZero(), + Spawn: gym.RaidSpawnTimestamp.ValueOrZero(), + Start: gym.RaidBattleTimestamp.ValueOrZero(), + End: gym.RaidEndTimestamp.ValueOrZero(), + Level: gym.RaidLevel.ValueOrZero(), + PokemonId: gym.RaidPokemonId.ValueOrZero(), + Cp: gym.RaidPokemonCp.ValueOrZero(), + Gender: gym.RaidPokemonGender.ValueOrZero(), + Form: gym.RaidPokemonForm.ValueOrZero(), + Alignment: gym.RaidPokemonAlignment.ValueOrZero(), + Costume: gym.RaidPokemonCostume.ValueOrZero(), + Evolution: gym.RaidPokemonEvolution.ValueOrZero(), + Move1: gym.RaidPokemonMove1.ValueOrZero(), + Move2: gym.RaidPokemonMove2.ValueOrZero(), + ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), + IsExclusive: gym.RaidIsExclusive.ValueOrZero(), + SponsorId: gym.SponsorId.ValueOrZero(), + PartnerId: gym.PartnerId.ValueOrZero(), + PowerUpPoints: gym.PowerUpPoints.ValueOrZero(), + PowerUpLevel: gym.PowerUpLevel.ValueOrZero(), + PowerUpEndTimestamp: gym.PowerUpEndTimestamp.ValueOrZero(), + ArScanEligible: gym.ArScanEligible.ValueOrZero(), + Rsvps: rsvps, } webhooksSender.AddMessage(webhooks.Raid, raidHook, areas) diff --git a/decoder/incident.go b/decoder/incident.go index 6d2a7dce..7e5fbde5 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -40,6 +40,26 @@ type webhookLineup struct { Form null.Int `json:"form"` } +type IncidentWebhook struct { + Id string `json:"id"` + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + PokestopName string `json:"pokestop_name"` + Url string `json:"url"` + Enabled bool `json:"enabled"` + Start int64 `json:"start"` + IncidentExpireTimestamp int64 `json:"incident_expire_timestamp"` + Expiration int64 `json:"expiration"` + DisplayType int16 `json:"display_type"` + Style int16 `json:"style"` + GruntType int16 `json:"grunt_type"` + Character int16 `json:"character"` + Updated int64 `json:"updated"` + Confirmed bool `json:"confirmed"` + Lineup []webhookLineup `json:"lineup"` +} + //-> `id` varchar(35) NOT NULL, //-> `pokestop_id` varchar(35) NOT NULL, //-> `start` int unsigned NOT NULL, @@ -160,34 +180,14 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *I stop = &Pokestop{} } - incidentHook := map[string]interface{}{ - "id": incident.Id, - "pokestop_id": incident.PokestopId, - "latitude": stop.Lat, - "longitude": stop.Lon, - "pokestop_name": func() string { - if stop.Name.Valid { - return stop.Name.String - } else { - return "Unknown" - } - }(), - "url": stop.Url.ValueOrZero(), - "enabled": stop.Enabled.ValueOrZero(), - "start": incident.StartTime, - "incident_expire_timestamp": incident.ExpirationTime, // deprecated, remove old key in the future - "expiration": incident.ExpirationTime, - "display_type": incident.DisplayType, - "style": incident.Style, - "grunt_type": incident.Character, // deprecated, remove old key in the future - "character": incident.Character, - "updated": incident.Updated, - "confirmed": incident.Confirmed, - "lineup": nil, + pokestopName := "Unknown" + if stop.Name.Valid { + pokestopName = stop.Name.String } + var lineup []webhookLineup if incident.Slot1PokemonId.Valid { - incidentHook["lineup"] = []webhookLineup{ + lineup = []webhookLineup{ { Slot: 1, PokemonId: incident.Slot1PokemonId, @@ -205,6 +205,27 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *I }, } } + + incidentHook := IncidentWebhook{ + Id: incident.Id, + PokestopId: incident.PokestopId, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Url: stop.Url.ValueOrZero(), + Enabled: stop.Enabled.ValueOrZero(), + Start: incident.StartTime, + IncidentExpireTimestamp: incident.ExpirationTime, + Expiration: incident.ExpirationTime, + DisplayType: incident.DisplayType, + Style: incident.Style, + GruntType: incident.Character, + Character: incident.Character, + Updated: incident.Updated, + Confirmed: incident.Confirmed, + Lineup: lineup, + } + areas := MatchStatsGeofence(stop.Lat, stop.Lon) webhooksSender.AddMessage(webhooks.Invasion, incidentHook, areas) statsCollector.UpdateIncidentCount(areas) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9244cc2b..27416d1c 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -412,81 +412,109 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } -func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, new *Pokemon, areas []geo.AreaName) { - //nullString := func (v null.Int) interface{} { - // if !v.Valid { - // return "null" - // } - // return v.ValueOrZero() - //} +type PokemonWebhook struct { + SpawnpointId string `json:"spawnpoint_id"` + PokestopId string `json:"pokestop_id"` + PokestopName *string `json:"pokestop_name"` + EncounterId string `json:"encounter_id"` + PokemonId int16 `json:"pokemon_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + DisappearTime int64 `json:"disappear_time"` + DisappearTimeVerified bool `json:"disappear_time_verified"` + FirstSeen int64 `json:"first_seen"` + LastModifiedTime null.Int `json:"last_modified_time"` + Gender null.Int `json:"gender"` + Cp null.Int `json:"cp"` + Form null.Int `json:"form"` + Costume null.Int `json:"costume"` + IndividualAttack null.Int `json:"individual_attack"` + IndividualDefense null.Int `json:"individual_defense"` + IndividualStamina null.Int `json:"individual_stamina"` + PokemonLevel null.Int `json:"pokemon_level"` + Move1 null.Int `json:"move_1"` + Move2 null.Int `json:"move_2"` + Weight null.Float `json:"weight"` + Size null.Int `json:"size"` + Height null.Float `json:"height"` + Weather null.Int `json:"weather"` + Capture1 float64 `json:"capture_1"` + Capture2 float64 `json:"capture_2"` + Capture3 float64 `json:"capture_3"` + Shiny null.Bool `json:"shiny"` + Username null.String `json:"username"` + DisplayPokemonId null.Int `json:"display_pokemon_id"` + IsEvent int8 `json:"is_event"` + SeenType null.String `json:"seen_type"` + Pvp json.RawMessage `json:"pvp"` +} +func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, new *Pokemon, areas []geo.AreaName) { if old == nil || old.PokemonId != new.PokemonId || old.Weather != new.Weather || old.Cp != new.Cp { - pokemonHook := map[string]interface{}{ - "spawnpoint_id": func() string { - if !new.SpawnId.Valid { - return "None" - } - return strconv.FormatInt(new.SpawnId.ValueOrZero(), 16) - }(), - "pokestop_id": func() string { - if !new.PokestopId.Valid { - return "None" - } else { - return new.PokestopId.ValueOrZero() - } - }(), - "pokestop_name": func() *string { - if !new.PokestopId.Valid { - return nil - } else { - pokestop, _ := GetPokestopRecord(ctx, db, new.PokestopId.String) - name := "Unknown" - if pokestop != nil { - name = pokestop.Name.ValueOrZero() - } - return &name - } - }(), - "encounter_id": strconv.FormatUint(new.Id, 10), - "pokemon_id": new.PokemonId, - "latitude": new.Lat, - "longitude": new.Lon, - "disappear_time": new.ExpireTimestamp.ValueOrZero(), - "disappear_time_verified": new.ExpireTimestampVerified, - "first_seen": new.FirstSeenTimestamp, - "last_modified_time": new.Updated, - "gender": new.Gender, - "cp": new.Cp, - "form": new.Form, - "costume": new.Costume, - "individual_attack": new.AtkIv, - "individual_defense": new.DefIv, - "individual_stamina": new.StaIv, - "pokemon_level": new.Level, - "move_1": new.Move1, - "move_2": new.Move2, - "weight": new.Weight, - "size": new.Size, - "height": new.Height, - "weather": new.Weather, - "capture_1": new.Capture1.ValueOrZero(), - "capture_2": new.Capture2.ValueOrZero(), - "capture_3": new.Capture3.ValueOrZero(), - "shiny": new.Shiny, - "username": new.Username, - "display_pokemon_id": new.DisplayPokemonId, - "is_event": new.IsEvent, - "seen_type": new.SeenType, - "pvp": func() interface{} { - if !new.Pvp.Valid { - return nil - } else { - return json.RawMessage(new.Pvp.ValueOrZero()) - } - }(), + + spawnpointId := "None" + if new.SpawnId.Valid { + spawnpointId = strconv.FormatInt(new.SpawnId.ValueOrZero(), 16) + } + + pokestopId := "None" + if new.PokestopId.Valid { + pokestopId = new.PokestopId.ValueOrZero() + } + + var pokestopName *string + if new.PokestopId.Valid { + pokestop, _ := GetPokestopRecord(ctx, db, new.PokestopId.String) + name := "Unknown" + if pokestop != nil { + name = pokestop.Name.ValueOrZero() + } + pokestopName = &name + } + + var pvp json.RawMessage + if new.Pvp.Valid { + pvp = json.RawMessage(new.Pvp.ValueOrZero()) + } + + pokemonHook := PokemonWebhook{ + SpawnpointId: spawnpointId, + PokestopId: pokestopId, + PokestopName: pokestopName, + EncounterId: strconv.FormatUint(new.Id, 10), + PokemonId: new.PokemonId, + Latitude: new.Lat, + Longitude: new.Lon, + DisappearTime: new.ExpireTimestamp.ValueOrZero(), + DisappearTimeVerified: new.ExpireTimestampVerified, + FirstSeen: new.FirstSeenTimestamp, + LastModifiedTime: new.Updated, + Gender: new.Gender, + Cp: new.Cp, + Form: new.Form, + Costume: new.Costume, + IndividualAttack: new.AtkIv, + IndividualDefense: new.DefIv, + IndividualStamina: new.StaIv, + PokemonLevel: new.Level, + Move1: new.Move1, + Move2: new.Move2, + Weight: new.Weight, + Size: new.Size, + Height: new.Height, + Weather: new.Weather, + Capture1: new.Capture1.ValueOrZero(), + Capture2: new.Capture2.ValueOrZero(), + Capture3: new.Capture3.ValueOrZero(), + Shiny: new.Shiny, + Username: new.Username, + DisplayPokemonId: new.DisplayPokemonId, + IsEvent: new.IsEvent, + SeenType: new.SeenType, + Pvp: pvp, } if new.AtkIv.Valid && new.DefIv.Valid && new.StaIv.Valid { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index ee7e966c..d6dbba86 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -108,6 +108,47 @@ type Pokestop struct { } +type QuestWebhook struct { + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + PokestopName string `json:"pokestop_name"` + Type null.Int `json:"type"` + Target null.Int `json:"target"` + Template null.String `json:"template"` + Title null.String `json:"title"` + Conditions json.RawMessage `json:"conditions"` + Rewards json.RawMessage `json:"rewards"` + Updated int64 `json:"updated"` + ArScanEligible int64 `json:"ar_scan_eligible"` + PokestopUrl string `json:"pokestop_url"` + WithAr bool `json:"with_ar"` +} + +type PokestopWebhook struct { + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + Url string `json:"url"` + LureExpiration int64 `json:"lure_expiration"` + LastModified int64 `json:"last_modified"` + Enabled bool `json:"enabled"` + LureId int16 `json:"lure_id"` + ArScanEligible int64 `json:"ar_scan_eligible"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + Updated int64 `json:"updated"` + ShowcaseFocus null.String `json:"showcase_focus"` + ShowcasePokemonId null.Int `json:"showcase_pokemon_id"` + ShowcasePokemonFormId null.Int `json:"showcase_pokemon_form_id"` + ShowcasePokemonTypeId null.Int `json:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `json:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `json:"showcase_expiry"` + ShowcaseRankings json.RawMessage `json:"showcase_rankings"` +} + func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, error) { stop := pokestopCache.Get(fortId) if stop != nil { @@ -663,92 +704,78 @@ func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { areas := MatchStatsGeofence(stop.Lat, stop.Lon) + pokestopName := "Unknown" + if stop.Name.Valid { + pokestopName = stop.Name.String + } + if stop.AlternativeQuestType.Valid && (oldStop == nil || stop.AlternativeQuestType != oldStop.AlternativeQuestType) { - questHook := map[string]any{ - "pokestop_id": stop.Id, - "latitude": stop.Lat, - "longitude": stop.Lon, - "pokestop_name": func() string { - if stop.Name.Valid { - return stop.Name.String - } else { - return "Unknown" - } - }(), - "type": stop.AlternativeQuestType, - "target": stop.AlternativeQuestTarget, - "template": stop.AlternativeQuestTemplate, - "title": stop.AlternativeQuestTitle, - "conditions": json.RawMessage(stop.AlternativeQuestConditions.ValueOrZero()), - "rewards": json.RawMessage(stop.AlternativeQuestRewards.ValueOrZero()), - "updated": stop.Updated, - "ar_scan_eligible": stop.ArScanEligible.ValueOrZero(), - "pokestop_url": stop.Url.ValueOrZero(), - "with_ar": false, + questHook := QuestWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Type: stop.AlternativeQuestType, + Target: stop.AlternativeQuestTarget, + Template: stop.AlternativeQuestTemplate, + Title: stop.AlternativeQuestTitle, + Conditions: json.RawMessage(stop.AlternativeQuestConditions.ValueOrZero()), + Rewards: json.RawMessage(stop.AlternativeQuestRewards.ValueOrZero()), + Updated: stop.Updated, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PokestopUrl: stop.Url.ValueOrZero(), + WithAr: false, } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } if stop.QuestType.Valid && (oldStop == nil || stop.QuestType != oldStop.QuestType) { - questHook := map[string]any{ - "pokestop_id": stop.Id, - "latitude": stop.Lat, - "longitude": stop.Lon, - "pokestop_name": func() string { - if stop.Name.Valid { - return stop.Name.String - } else { - return "Unknown" - } - }(), - "type": stop.QuestType, - "target": stop.QuestTarget, - "template": stop.QuestTemplate, - "title": stop.QuestTitle, - "conditions": json.RawMessage(stop.QuestConditions.ValueOrZero()), - "rewards": json.RawMessage(stop.QuestRewards.ValueOrZero()), - "updated": stop.Updated, - "ar_scan_eligible": stop.ArScanEligible.ValueOrZero(), - "pokestop_url": stop.Url.ValueOrZero(), - "with_ar": true, + questHook := QuestWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Type: stop.QuestType, + Target: stop.QuestTarget, + Template: stop.QuestTemplate, + Title: stop.QuestTitle, + Conditions: json.RawMessage(stop.QuestConditions.ValueOrZero()), + Rewards: json.RawMessage(stop.QuestRewards.ValueOrZero()), + Updated: stop.Updated, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PokestopUrl: stop.Url.ValueOrZero(), + WithAr: true, } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } if (oldStop == nil && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (oldStop != nil && ((stop.LureExpireTimestamp != oldStop.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != oldStop.PowerUpEndTimestamp)) { - pokestopHook := map[string]any{ - "pokestop_id": stop.Id, - "latitude": stop.Lat, - "longitude": stop.Lon, - "name": func() string { - if stop.Name.Valid { - return stop.Name.String - } else { - return "Unknown" - } - }(), - "url": stop.Url.ValueOrZero(), - "lure_expiration": stop.LureExpireTimestamp.ValueOrZero(), - "last_modified": stop.LastModifiedTimestamp.ValueOrZero(), - "enabled": stop.Enabled.ValueOrZero(), - "lure_id": stop.LureId, - "ar_scan_eligible": stop.ArScanEligible.ValueOrZero(), - "power_up_level": stop.PowerUpLevel.ValueOrZero(), - "power_up_points": stop.PowerUpPoints.ValueOrZero(), - "power_up_end_timestamp": stop.PowerUpPoints.ValueOrZero(), - "updated": stop.Updated, - "showcase_focus": stop.ShowcaseFocus, - "showcase_pokemon_id": stop.ShowcasePokemon, - "showcase_pokemon_form_id": stop.ShowcasePokemonForm, - "showcase_pokemon_type_id": stop.ShowcasePokemonType, - "showcase_ranking_standard": stop.ShowcaseRankingStandard, - "showcase_expiry": stop.ShowcaseExpiry, - "showcase_rankings": func() any { - if !stop.ShowcaseRankings.Valid { - return nil - } else { - return json.RawMessage(stop.ShowcaseRankings.ValueOrZero()) - } - }(), + var showcaseRankings json.RawMessage + if stop.ShowcaseRankings.Valid { + showcaseRankings = json.RawMessage(stop.ShowcaseRankings.ValueOrZero()) + } + + pokestopHook := PokestopWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + Name: pokestopName, + Url: stop.Url.ValueOrZero(), + LureExpiration: stop.LureExpireTimestamp.ValueOrZero(), + LastModified: stop.LastModifiedTimestamp.ValueOrZero(), + Enabled: stop.Enabled.ValueOrZero(), + LureId: stop.LureId, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PowerUpLevel: stop.PowerUpLevel.ValueOrZero(), + PowerUpPoints: stop.PowerUpPoints.ValueOrZero(), + PowerUpEndTimestamp: stop.PowerUpPoints.ValueOrZero(), + Updated: stop.Updated, + ShowcaseFocus: stop.ShowcaseFocus, + ShowcasePokemonId: stop.ShowcasePokemon, + ShowcasePokemonFormId: stop.ShowcasePokemonForm, + ShowcasePokemonTypeId: stop.ShowcasePokemonType, + ShowcaseRankingStandard: stop.ShowcaseRankingStandard, + ShowcaseExpiry: stop.ShowcaseExpiry, + ShowcaseRankings: showcaseRankings, } webhooksSender.AddMessage(webhooks.Pokestop, pokestopHook, areas) diff --git a/decoder/station.go b/decoder/station.go index 34b0de81..f02b46f1 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -49,6 +49,30 @@ type Station struct { StationedPokemon null.String `db:"stationed_pokemon"` } +type StationWebhook struct { + Id string `json:"id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + IsBattleAvailable bool `json:"is_battle_available"` + BattleLevel null.Int `json:"battle_level"` + BattleStart null.Int `json:"battle_start"` + BattleEnd null.Int `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` + TotalStationedGmax null.Int `json:"total_stationed_gmax"` + Updated int64 `json:"updated"` +} + func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, error) { inMemoryStation := stationCache.Get(stationId) if inMemoryStation != nil { @@ -317,28 +341,28 @@ func createStationWebhooks(oldStation *Station, station *Station) { oldStation.BattlePokemonCostume != station.BattlePokemonCostume || oldStation.BattlePokemonGender != station.BattlePokemonGender || oldStation.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { - stationHook := map[string]any{ - "id": station.Id, - "latitude": station.Lat, - "longitude": station.Lon, - "name": station.Name, - "start_time": station.StartTime, - "end_time": station.EndTime, - "is_battle_available": station.IsBattleAvailable, - "battle_level": station.BattleLevel, - "battle_start": station.BattleStart, - "battle_end": station.BattleEnd, - "battle_pokemon_id": station.BattlePokemonId, - "battle_pokemon_form": station.BattlePokemonForm, - "battle_pokemon_costume": station.BattlePokemonCostume, - "battle_pokemon_gender": station.BattlePokemonGender, - "battle_pokemon_alignment": station.BattlePokemonAlignment, - "battle_pokemon_bread_mode": station.BattlePokemonBreadMode, - "battle_pokemon_move_1": station.BattlePokemonMove1, - "battle_pokemon_move_2": station.BattlePokemonMove2, - "total_stationed_pokemon": station.TotalStationedPokemon, - "total_stationed_gmax": station.TotalStationedGmax, - "updated": station.Updated, + stationHook := StationWebhook{ + Id: station.Id, + Latitude: station.Lat, + Longitude: station.Lon, + Name: station.Name, + StartTime: station.StartTime, + EndTime: station.EndTime, + IsBattleAvailable: station.IsBattleAvailable, + BattleLevel: station.BattleLevel, + BattleStart: station.BattleStart, + BattleEnd: station.BattleEnd, + BattlePokemonId: station.BattlePokemonId, + BattlePokemonForm: station.BattlePokemonForm, + BattlePokemonCostume: station.BattlePokemonCostume, + BattlePokemonGender: station.BattlePokemonGender, + BattlePokemonAlignment: station.BattlePokemonAlignment, + BattlePokemonBreadMode: station.BattlePokemonBreadMode, + BattlePokemonMove1: station.BattlePokemonMove1, + BattlePokemonMove2: station.BattlePokemonMove2, + TotalStationedPokemon: station.TotalStationedPokemon, + TotalStationedGmax: station.TotalStationedGmax, + Updated: station.Updated, } areas := MatchStatsGeofence(station.Lat, station.Lon) webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) diff --git a/decoder/weather.go b/decoder/weather.go index ae5aa573..3e29aa24 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -122,6 +122,24 @@ func hasChangesWeather(old *Weather, new *Weather) bool { !floatAlmostEqual(old.Longitude, new.Longitude, floatTolerance) } +type WeatherWebhook struct { + S2CellId int64 `json:"s2_cell_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Polygon [4][2]float64 `json:"polygon"` + GameplayCondition int64 `json:"gameplay_condition"` + WindDirection int64 `json:"wind_direction"` + CloudLevel int64 `json:"cloud_level"` + RainLevel int64 `json:"rain_level"` + WindLevel int64 `json:"wind_level"` + SnowLevel int64 `json:"snow_level"` + FogLevel int64 `json:"fog_level"` + SpecialEffectLevel int64 `json:"special_effect_level"` + Severity int64 `json:"severity"` + WarnWeather bool `json:"warn_weather"` + Updated int64 `json:"updated"` +} + func createWeatherWebhooks(oldWeather *Weather, weather *Weather) { if oldWeather == nil || oldWeather.GameplayCondition.ValueOrZero() != weather.GameplayCondition.ValueOrZero() || oldWeather.WarnWeather.ValueOrZero() != weather.WarnWeather.ValueOrZero() { @@ -133,22 +151,23 @@ func createWeatherWebhooks(oldWeather *Weather, weather *Weather) { latLng := s2.LatLngFromPoint(vertex) polygon[i] = [...]float64{latLng.Lat.Degrees(), latLng.Lng.Degrees()} } - weatherHook := map[string]interface{}{ - "s2_cell_id": weather.Id, - "latitude": weather.Latitude, - "longitude": weather.Longitude, - "polygon": polygon, - "gameplay_condition": weather.GameplayCondition.ValueOrZero(), - "wind_direction": weather.WindDirection.ValueOrZero(), - "cloud_level": weather.CloudLevel.ValueOrZero(), - "rain_level": weather.RainLevel.ValueOrZero(), - "wind_level": weather.WindLevel.ValueOrZero(), - "snow_level": weather.SnowLevel.ValueOrZero(), - "fog_level": weather.FogLevel.ValueOrZero(), - "special_effect_level": weather.SpecialEffectLevel.ValueOrZero(), - "severity": weather.Severity.ValueOrZero(), - "warn_weather": weather.WarnWeather.ValueOrZero(), - "updated": weather.UpdatedMs / 1000, + + weatherHook := WeatherWebhook{ + S2CellId: weather.Id, + Latitude: weather.Latitude, + Longitude: weather.Longitude, + Polygon: polygon, + GameplayCondition: weather.GameplayCondition.ValueOrZero(), + WindDirection: weather.WindDirection.ValueOrZero(), + CloudLevel: weather.CloudLevel.ValueOrZero(), + RainLevel: weather.RainLevel.ValueOrZero(), + WindLevel: weather.WindLevel.ValueOrZero(), + SnowLevel: weather.SnowLevel.ValueOrZero(), + FogLevel: weather.FogLevel.ValueOrZero(), + SpecialEffectLevel: weather.SpecialEffectLevel.ValueOrZero(), + Severity: weather.Severity.ValueOrZero(), + WarnWeather: weather.WarnWeather.ValueOrZero(), + Updated: weather.UpdatedMs / 1000, } areas := MatchStatsGeofence(weather.Latitude, weather.Longitude) webhooksSender.AddMessage(webhooks.Weather, weatherHook, areas) diff --git a/routes.go b/routes.go index e68c16a6..ca840480 100644 --- a/routes.go +++ b/routes.go @@ -40,6 +40,10 @@ type InboundRawData struct { HaveAr *bool } +type StatusResponse struct { + Status string `json:"status"` +} + func questsHeldHasARTask(quests_held any) *bool { const ar_quest_id = int64(pogo.QuestType_QUEST_GEOTARGETED_AR_SCAN) @@ -346,17 +350,13 @@ func ClearQuests(c *gin.Context) { decoder.ClearQuestsWithinGeofence(ctx, dbDetails, fence) log.Infof("Clear quest took %s", time.Since(startTime)) - c.JSON(http.StatusAccepted, map[string]interface{}{ - "status": "ok", - }) + c.JSON(http.StatusAccepted, StatusResponse{Status: "ok"}) } func ReloadGeojson(c *gin.Context) { decoder.ReloadGeofenceAndClearStats() - c.JSON(http.StatusAccepted, map[string]interface{}{ - "status": "ok", - }) + c.JSON(http.StatusAccepted, StatusResponse{Status: "ok"}) } func PokemonScan(c *gin.Context) { From f8b2eb151320ede613d51feef25530010f4cbfcc Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 11 Jan 2026 16:04:25 +0000 Subject: [PATCH 02/78] Replace per-pokemon goroutines with pending queue Instead of spawning a goroutine for each wild pokemon that sleeps waiting for an encounter, use a single pending queue with a background sweeper. This reduces memory pressure and goroutine overhead under high load. Co-Authored-By: Claude Opus 4.5 --- decoder/main.go | 33 +++----- decoder/pending_pokemon.go | 166 +++++++++++++++++++++++++++++++++++++ decoder/pokemon.go | 5 ++ main.go | 1 + 4 files changed, 184 insertions(+), 21 deletions(-) create mode 100644 decoder/pending_pokemon.go diff --git a/decoder/main.go b/decoder/main.go index 42ec4c1b..e8804edf 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -415,27 +415,18 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca } else { updateTime := wild.Timestamp / 1000 if pokemon.isNewRecord() || pokemon.wildSignificantUpdate(wild.Data, updateTime) { - go func(wildPokemon *pogo.WildPokemonProto, cellId int64, timestampMs int64) { - time.Sleep(15 * time.Second) - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - defer pokemonMutex.Unlock() - - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - - if pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId); err != nil { - log.Errorf("getOrCreatePokemonRecord: %s", err) - } else { - // Update if there is still a change required & this update is the most recent - if pokemon.wildSignificantUpdate(wildPokemon, updateTime) && pokemon.Updated.ValueOrZero() < updateTime { - log.Debugf("DELAYED UPDATE: Updating pokemon %d from wild", encounterId) - - pokemon.updateFromWild(ctx, db, wildPokemon, cellId, weatherLookup, timestampMs, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, updateTime) - } - } - }(wild.Data, int64(wild.Cell), wild.Timestamp) + // Add to pending queue instead of spawning a goroutine + // The sweeper will process it after timeout if no encounter arrives + pending := &PendingPokemon{ + EncounterId: encounterId, + WildPokemon: wild.Data, + CellId: int64(wild.Cell), + TimestampMs: wild.Timestamp, + UpdateTime: updateTime, + WeatherLookup: weatherLookup, + Username: username, + } + pokemonPendingQueue.AddPending(pending) } } } diff --git a/decoder/pending_pokemon.go b/decoder/pending_pokemon.go new file mode 100644 index 00000000..4a22f070 --- /dev/null +++ b/decoder/pending_pokemon.go @@ -0,0 +1,166 @@ +package decoder + +import ( + "context" + "sync" + "time" + + "golbat/db" + "golbat/pogo" + + log "github.com/sirupsen/logrus" +) + +// PendingPokemon stores wild pokemon data awaiting a potential encounter +type PendingPokemon struct { + EncounterId uint64 + WildPokemon *pogo.WildPokemonProto + CellId int64 + TimestampMs int64 + UpdateTime int64 + WeatherLookup map[int64]pogo.GameplayWeatherProto_WeatherCondition + Username string + ReceivedAt time.Time +} + +// PokemonPendingQueue manages pokemon awaiting encounter data +type PokemonPendingQueue struct { + mu sync.RWMutex + pending map[uint64]*PendingPokemon + timeout time.Duration +} + +// NewPokemonPendingQueue creates a new pending queue with the specified timeout +func NewPokemonPendingQueue(timeout time.Duration) *PokemonPendingQueue { + return &PokemonPendingQueue{ + pending: make(map[uint64]*PendingPokemon), + timeout: timeout, + } +} + +// AddPending stores a wild pokemon awaiting encounter data. +// Returns true if the pokemon was added, false if it already exists. +func (q *PokemonPendingQueue) AddPending(p *PendingPokemon) bool { + q.mu.Lock() + defer q.mu.Unlock() + + // Only add if not already present (first sighting wins) + if _, exists := q.pending[p.EncounterId]; exists { + return false + } + + p.ReceivedAt = time.Now() + q.pending[p.EncounterId] = p + return true +} + +// TryComplete attempts to retrieve and remove a pending pokemon for an encounter. +// Returns the pending pokemon and true if found, nil and false otherwise. +func (q *PokemonPendingQueue) TryComplete(encounterId uint64) (*PendingPokemon, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + p, exists := q.pending[encounterId] + if exists { + delete(q.pending, encounterId) + } + return p, exists +} + +// Remove removes a pending pokemon without processing it. +func (q *PokemonPendingQueue) Remove(encounterId uint64) { + q.mu.Lock() + delete(q.pending, encounterId) + q.mu.Unlock() +} + +// Size returns the current number of pending pokemon +func (q *PokemonPendingQueue) Size() int { + q.mu.RLock() + defer q.mu.RUnlock() + return len(q.pending) +} + +// collectExpired removes and returns all entries older than timeout +func (q *PokemonPendingQueue) collectExpired() []*PendingPokemon { + cutoff := time.Now().Add(-q.timeout) + + q.mu.Lock() + defer q.mu.Unlock() + + var expired []*PendingPokemon + for id, p := range q.pending { + if p.ReceivedAt.Before(cutoff) { + expired = append(expired, p) + delete(q.pending, id) + } + } + + return expired +} + +// StartSweeper starts a background goroutine that processes expired entries +func (q *PokemonPendingQueue) StartSweeper(ctx context.Context, interval time.Duration, dbDetails db.DbDetails) { + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Info("Pokemon pending queue sweeper stopped") + return + case <-ticker.C: + expired := q.collectExpired() + if len(expired) > 0 { + log.Debugf("Processing %d expired pending pokemon", len(expired)) + q.processExpired(ctx, dbDetails, expired) + } + } + } + }() +} + +// processExpired handles pokemon that didn't receive an encounter within the timeout +func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.DbDetails, expired []*PendingPokemon) { + for _, p := range expired { + pokemonMutex, _ := pokemonStripedMutex.GetLock(p.EncounterId) + pokemonMutex.Lock() + + processCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + + pokemon, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) + if err != nil { + log.Errorf("getOrCreatePokemonRecord in sweeper: %s", err) + cancel() + pokemonMutex.Unlock() + continue + } + + // Update if there is still a change required & this update is the most recent + if pokemon.wildSignificantUpdate(p.WildPokemon, p.UpdateTime) && pokemon.Updated.ValueOrZero() < p.UpdateTime { + log.Debugf("DELAYED UPDATE: Updating pokemon %d from wild (sweeper)", p.EncounterId) + + pokemon.updateFromWild(processCtx, dbDetails, p.WildPokemon, p.CellId, p.WeatherLookup, p.TimestampMs, p.Username) + savePokemonRecordAsAtTime(processCtx, dbDetails, pokemon, false, true, true, p.UpdateTime) + } + + cancel() + pokemonMutex.Unlock() + } +} + +// Global pending queue instance +var pokemonPendingQueue *PokemonPendingQueue + +// InitPokemonPendingQueue initializes the global pending queue +func InitPokemonPendingQueue(ctx context.Context, dbDetails db.DbDetails, timeout time.Duration, sweepInterval time.Duration) { + pokemonPendingQueue = NewPokemonPendingQueue(timeout) + pokemonPendingQueue.StartSweeper(ctx, sweepInterval, dbDetails) + log.Infof("Pokemon pending queue started with %v timeout and %v sweep interval", timeout, sweepInterval) +} + +// GetPokemonPendingQueue returns the global pending queue instance +func GetPokemonPendingQueue() *PokemonPendingQueue { + return pokemonPendingQueue +} diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9244cc2b..33da3fa6 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -1400,6 +1400,11 @@ func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, encounterId := encounter.Pokemon.EncounterId + // Remove from pending queue - encounter arrived so no need for delayed wild update + if pokemonPendingQueue != nil { + pokemonPendingQueue.Remove(encounterId) + } + pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) pokemonMutex.Lock() defer pokemonMutex.Unlock() diff --git a/main.go b/main.go index 02cae765..209599b8 100644 --- a/main.go +++ b/main.go @@ -210,6 +210,7 @@ func main() { _ = decoder.WatchMasterFileData() } decoder.LoadStatsGeofences() + decoder.InitPokemonPendingQueue(ctx, dbDetails, 30*time.Second, 5*time.Second) InitDeviceCache() wg.Add(1) From 22afd74fe3e6488f555ebe56fb3bcc30855b3178 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 11:39:38 +0000 Subject: [PATCH 03/78] Remove comment --- decoder/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/decoder/main.go b/decoder/main.go index e8804edf..4f7b9c38 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -415,7 +415,6 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca } else { updateTime := wild.Timestamp / 1000 if pokemon.isNewRecord() || pokemon.wildSignificantUpdate(wild.Data, updateTime) { - // Add to pending queue instead of spawning a goroutine // The sweeper will process it after timeout if no encounter arrives pending := &PendingPokemon{ EncounterId: encounterId, From 3de2ece4921b28498bbb454ef58e165124398c23 Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 26 Jan 2026 18:06:02 +0000 Subject: [PATCH 04/78] Example dirty flag implementation --- decoder/fort.go | 16 +- decoder/main.go | 13 +- decoder/pokestop.go | 564 +++++++++++++++++++++++++++-------- decoder/pokestop_showcase.go | 18 +- 4 files changed, 459 insertions(+), 152 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index bc308a01..b4b8a08c 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -245,27 +245,27 @@ func (gym *Gym) copySharedFieldsFrom(pokestop *Pokestop) { // copySharedFieldsFrom copies shared fields from a gym to a pokestop during conversion func (stop *Pokestop) copySharedFieldsFrom(gym *Gym) { if gym.Name.Valid && !stop.Name.Valid { - stop.Name = gym.Name + stop.SetName(gym.Name) } if gym.Url.Valid && !stop.Url.Valid { - stop.Url = gym.Url + stop.SetUrl(gym.Url) } if gym.Description.Valid && !stop.Description.Valid { - stop.Description = gym.Description + stop.SetDescription(gym.Description) } if gym.PartnerId.Valid && !stop.PartnerId.Valid { - stop.PartnerId = gym.PartnerId + stop.SetPartnerId(gym.PartnerId) } if gym.ArScanEligible.Valid && !stop.ArScanEligible.Valid { - stop.ArScanEligible = gym.ArScanEligible + stop.SetArScanEligible(gym.ArScanEligible) } if gym.PowerUpLevel.Valid && !stop.PowerUpLevel.Valid { - stop.PowerUpLevel = gym.PowerUpLevel + stop.SetPowerUpLevel(gym.PowerUpLevel) } if gym.PowerUpPoints.Valid && !stop.PowerUpPoints.Valid { - stop.PowerUpPoints = gym.PowerUpPoints + stop.SetPowerUpPoints(gym.PowerUpPoints) } if gym.PowerUpEndTimestamp.Valid && !stop.PowerUpEndTimestamp.Valid { - stop.PowerUpEndTimestamp = gym.PowerUpEndTimestamp + stop.SetPowerUpEndTimestamp(gym.PowerUpEndTimestamp) } } diff --git a/decoder/main.go b/decoder/main.go index 42ec4c1b..9173ad5d 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -58,7 +58,7 @@ type webhooksSenderInterface interface { var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector -var pokestopCache *ttlcache.Cache[string, Pokestop] +var pokestopCache *ttlcache.Cache[string, *Pokestop] var gymCache *ttlcache.Cache[string, Gym] var stationCache *ttlcache.Cache[string, Station] var tappableCache *ttlcache.Cache[uint64, Tappable] @@ -118,8 +118,8 @@ func deletePokemonFromCache(key uint64) { } func initDataCache() { - pokestopCache = ttlcache.New[string, Pokestop]( - ttlcache.WithTTL[string, Pokestop](60 * time.Minute), + pokestopCache = ttlcache.New[string, *Pokestop]( + ttlcache.WithTTL[string, *Pokestop](60 * time.Minute), ) go pokestopCache.Start() @@ -297,14 +297,13 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa continue } - isNewPokestop := pokestop == nil - if isNewPokestop { - pokestop = &Pokestop{} + if pokestop == nil { + pokestop = &Pokestop{newRecord: true} } pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) // If this is a new pokestop, check if it was converted from a gym and copy shared fields - if isNewPokestop { + if pokestop.IsNewRecord() { gym, _ := GetGymRecord(ctx, db, fortId) if gym != nil { pokestop.copySharedFieldsFrom(gym) diff --git a/decoder/pokestop.go b/decoder/pokestop.go index ee7e966c..7be500ae 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -23,7 +23,6 @@ import ( ) // Pokestop struct. -// REMINDER! Keep hasChangesPokestop updated after making changes type Pokestop struct { Id string `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` @@ -68,6 +67,22 @@ type Pokestop struct { ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard" json:"showcase_ranking_standard"` ShowcaseExpiry null.Int `db:"showcase_expiry" json:"showcase_expiry"` ShowcaseRankings null.String `db:"showcase_rankings" json:"showcase_rankings"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + // Old values for webhook comparison (populated when loading from cache/DB) + oldQuestType null.Int `db:"-" json:"-"` + oldAlternativeQuestType null.Int `db:"-" json:"-"` + oldLureExpireTimestamp null.Int `db:"-" json:"-"` + oldLureId int16 `db:"-" json:"-"` + oldPowerUpEndTimestamp null.Int `db:"-" json:"-"` + oldName null.String `db:"-" json:"-"` + oldUrl null.String `db:"-" json:"-"` + oldDescription null.String `db:"-" json:"-"` + oldLat float64 `db:"-" json:"-"` + oldLon float64 `db:"-" json:"-"` + //`id` varchar(35) NOT NULL, //`lat` double(18,14) NOT NULL, //`lon` double(18,14) NOT NULL, @@ -108,12 +123,339 @@ type Pokestop struct { } +// IsDirty returns true if any field has been modified +func (p *Pokestop) IsDirty() bool { + return p.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (p *Pokestop) ClearDirty() { + p.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (p *Pokestop) IsNewRecord() bool { + return p.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (p *Pokestop) snapshotOldValues() { + p.oldQuestType = p.QuestType + p.oldAlternativeQuestType = p.AlternativeQuestType + p.oldLureExpireTimestamp = p.LureExpireTimestamp + p.oldLureId = p.LureId + p.oldPowerUpEndTimestamp = p.PowerUpEndTimestamp + p.oldName = p.Name + p.oldUrl = p.Url + p.oldDescription = p.Description + p.oldLat = p.Lat + p.oldLon = p.Lon +} + +// --- Set methods with dirty tracking --- + +func (p *Pokestop) SetId(v string) { + if p.Id != v { + p.Id = v + p.dirty = true + } +} + +func (p *Pokestop) SetLat(v float64) { + if !floatAlmostEqual(p.Lat, v, floatTolerance) { + p.Lat = v + p.dirty = true + } +} + +func (p *Pokestop) SetLon(v float64) { + if !floatAlmostEqual(p.Lon, v, floatTolerance) { + p.Lon = v + p.dirty = true + } +} + +func (p *Pokestop) SetName(v null.String) { + if p.Name != v { + p.Name = v + p.dirty = true + } +} + +func (p *Pokestop) SetUrl(v null.String) { + if p.Url != v { + p.Url = v + p.dirty = true + } +} + +func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { + if p.LureExpireTimestamp != v { + p.LureExpireTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { + if p.LastModifiedTimestamp != v { + p.LastModifiedTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetEnabled(v null.Bool) { + if p.Enabled != v { + p.Enabled = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestType(v null.Int) { + if p.QuestType != v { + p.QuestType = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestTimestamp(v null.Int) { + if p.QuestTimestamp != v { + p.QuestTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestTarget(v null.Int) { + if p.QuestTarget != v { + p.QuestTarget = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestConditions(v null.String) { + if p.QuestConditions != v { + p.QuestConditions = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestRewards(v null.String) { + if p.QuestRewards != v { + p.QuestRewards = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestTemplate(v null.String) { + if p.QuestTemplate != v { + p.QuestTemplate = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestTitle(v null.String) { + if p.QuestTitle != v { + p.QuestTitle = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestExpiry(v null.Int) { + if p.QuestExpiry != v { + p.QuestExpiry = v + p.dirty = true + } +} + +func (p *Pokestop) SetCellId(v null.Int) { + if p.CellId != v { + p.CellId = v + p.dirty = true + } +} + +func (p *Pokestop) SetDeleted(v bool) { + if p.Deleted != v { + p.Deleted = v + p.dirty = true + } +} + +func (p *Pokestop) SetLureId(v int16) { + if p.LureId != v { + p.LureId = v + p.dirty = true + } +} + +func (p *Pokestop) SetFirstSeenTimestamp(v int16) { + if p.FirstSeenTimestamp != v { + p.FirstSeenTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetSponsorId(v null.Int) { + if p.SponsorId != v { + p.SponsorId = v + p.dirty = true + } +} + +func (p *Pokestop) SetPartnerId(v null.String) { + if p.PartnerId != v { + p.PartnerId = v + p.dirty = true + } +} + +func (p *Pokestop) SetArScanEligible(v null.Int) { + if p.ArScanEligible != v { + p.ArScanEligible = v + p.dirty = true + } +} + +func (p *Pokestop) SetPowerUpLevel(v null.Int) { + if p.PowerUpLevel != v { + p.PowerUpLevel = v + p.dirty = true + } +} + +func (p *Pokestop) SetPowerUpPoints(v null.Int) { + if p.PowerUpPoints != v { + p.PowerUpPoints = v + p.dirty = true + } +} + +func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { + if p.PowerUpEndTimestamp != v { + p.PowerUpEndTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestType(v null.Int) { + if p.AlternativeQuestType != v { + p.AlternativeQuestType = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { + if p.AlternativeQuestTimestamp != v { + p.AlternativeQuestTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { + if p.AlternativeQuestTarget != v { + p.AlternativeQuestTarget = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { + if p.AlternativeQuestConditions != v { + p.AlternativeQuestConditions = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { + if p.AlternativeQuestRewards != v { + p.AlternativeQuestRewards = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { + if p.AlternativeQuestTemplate != v { + p.AlternativeQuestTemplate = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { + if p.AlternativeQuestTitle != v { + p.AlternativeQuestTitle = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { + if p.AlternativeQuestExpiry != v { + p.AlternativeQuestExpiry = v + p.dirty = true + } +} + +func (p *Pokestop) SetDescription(v null.String) { + if p.Description != v { + p.Description = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcaseFocus(v null.String) { + if p.ShowcaseFocus != v { + p.ShowcaseFocus = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcasePokemon(v null.Int) { + if p.ShowcasePokemon != v { + p.ShowcasePokemon = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { + if p.ShowcasePokemonForm != v { + p.ShowcasePokemonForm = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcasePokemonType(v null.Int) { + if p.ShowcasePokemonType != v { + p.ShowcasePokemonType = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { + if p.ShowcaseRankingStandard != v { + p.ShowcaseRankingStandard = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcaseExpiry(v null.Int) { + if p.ShowcaseExpiry != v { + p.ShowcaseExpiry = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcaseRankings(v null.String) { + if p.ShowcaseRankings != v { + p.ShowcaseRankings = v + p.dirty = true + } +} + func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, error) { stop := pokestopCache.Get(fortId) if stop != nil { - pokestop := stop.Value() //log.Debugf("GetPokestopRecord %s (from cache)", fortId) - return &pokestop, nil + pokestop := stop.Value() + pokestop.snapshotOldValues() // Snapshot for webhook comparison + return pokestop, nil } pokestop := Pokestop{} err := db.GeneralDb.GetContext(ctx, &pokestop, @@ -138,75 +480,33 @@ func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Po return nil, err } - pokestopCache.Set(fortId, pokestop, ttlcache.DefaultTTL) + pokestop.snapshotOldValues() // Snapshot for webhook comparison + pokestopCache.Set(fortId, &pokestop, ttlcache.DefaultTTL) if config.Config.TestFortInMemory { fortRtreeUpdatePokestopOnGet(&pokestop) } return &pokestop, nil } -// hasChangesPokestop compares two Pokestop structs -// Float tolerance: Lat, Lon -func hasChangesPokestop(old *Pokestop, new *Pokestop) bool { - return old.Id != new.Id || - old.Name != new.Name || - old.Url != new.Url || - old.LureExpireTimestamp != new.LureExpireTimestamp || - old.LastModifiedTimestamp != new.LastModifiedTimestamp || - old.Updated != new.Updated || - old.Enabled != new.Enabled || - old.QuestType != new.QuestType || - old.QuestTimestamp != new.QuestTimestamp || - old.QuestTarget != new.QuestTarget || - old.QuestConditions != new.QuestConditions || - old.QuestRewards != new.QuestRewards || - old.QuestTemplate != new.QuestTemplate || - old.QuestTitle != new.QuestTitle || - old.QuestExpiry != new.QuestExpiry || - old.CellId != new.CellId || - old.Deleted != new.Deleted || - old.LureId != new.LureId || - old.FirstSeenTimestamp != new.FirstSeenTimestamp || - old.SponsorId != new.SponsorId || - old.PartnerId != new.PartnerId || - old.ArScanEligible != new.ArScanEligible || - old.PowerUpLevel != new.PowerUpLevel || - old.PowerUpPoints != new.PowerUpPoints || - old.PowerUpEndTimestamp != new.PowerUpEndTimestamp || - old.AlternativeQuestType != new.AlternativeQuestType || - old.AlternativeQuestTimestamp != new.AlternativeQuestTimestamp || - old.AlternativeQuestTarget != new.AlternativeQuestTarget || - old.AlternativeQuestConditions != new.AlternativeQuestConditions || - old.AlternativeQuestRewards != new.AlternativeQuestRewards || - old.AlternativeQuestTemplate != new.AlternativeQuestTemplate || - old.AlternativeQuestTitle != new.AlternativeQuestTitle || - old.AlternativeQuestExpiry != new.AlternativeQuestExpiry || - old.Description != new.Description || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) || - old.ShowcaseRankingStandard != new.ShowcaseRankingStandard || - old.ShowcaseFocus != new.ShowcaseFocus || - old.ShowcaseRankings != new.ShowcaseRankings || - old.ShowcaseExpiry != new.ShowcaseExpiry -} - var LureTime int64 = 1800 func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, cellId uint64, now int64) *Pokestop { - stop.Id = fortData.FortId - stop.Lat = fortData.Latitude - stop.Lon = fortData.Longitude - - stop.PartnerId = null.NewString(fortData.PartnerId, fortData.PartnerId != "") - stop.SponsorId = null.IntFrom(int64(fortData.Sponsor)) - stop.Enabled = null.BoolFrom(fortData.Enabled) - stop.ArScanEligible = null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible)) - stop.PowerUpPoints = null.IntFrom(int64(fortData.PowerUpProgressPoints)) - stop.PowerUpLevel, stop.PowerUpEndTimestamp = calculatePowerUpPoints(fortData) + stop.SetId(fortData.FortId) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) + + stop.SetPartnerId(null.NewString(fortData.PartnerId, fortData.PartnerId != "")) + stop.SetSponsorId(null.IntFrom(int64(fortData.Sponsor))) + stop.SetEnabled(null.BoolFrom(fortData.Enabled)) + stop.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) + stop.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) + powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) + stop.SetPowerUpLevel(powerUpLevel) + stop.SetPowerUpEndTimestamp(powerUpEndTimestamp) // lasModifiedMs is also modified when incident happens lastModifiedTimestamp := fortData.LastModifiedMs / 1000 - stop.LastModifiedTimestamp = null.IntFrom(lastModifiedTimestamp) + stop.SetLastModifiedTimestamp(null.IntFrom(lastModifiedTimestamp)) if len(fortData.ActiveFortModifier) > 0 { lureId := int16(fortData.ActiveFortModifier[0]) @@ -214,8 +514,8 @@ func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, ce lureEnd := lastModifiedTimestamp + LureTime oldLureEnd := stop.LureExpireTimestamp.ValueOrZero() if stop.LureId != lureId { - stop.LureExpireTimestamp = null.IntFrom(lureEnd) - stop.LureId = lureId + stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) + stop.SetLureId(lureId) } else { // wait some time after lure end before a restart in case of timing issue if now > oldLureEnd+30 { @@ -223,19 +523,19 @@ func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, ce lureEnd += LureTime } // lure needs to be restarted - stop.LureExpireTimestamp = null.IntFrom(lureEnd) + stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) } } } } if fortData.ImageUrl != "" { - stop.Url = null.StringFrom(fortData.ImageUrl) + stop.SetUrl(null.StringFrom(fortData.ImageUrl)) } - stop.CellId = null.IntFrom(int64(cellId)) + stop.SetCellId(null.IntFrom(int64(cellId))) if stop.Deleted { - stop.Deleted = false + stop.SetDeleted(false) log.Warnf("Cleared Stop with id '%s' is found again in GMO, therefore un-deleted", stop.Id) // Restore in fort tracker if enabled if fortTracker != nil { @@ -514,41 +814,41 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu } if !haveAr { - stop.AlternativeQuestType = null.IntFrom(questType) - stop.AlternativeQuestTarget = null.IntFrom(questTarget) - stop.AlternativeQuestTemplate = null.StringFrom(questTemplate) - stop.AlternativeQuestTitle = null.StringFrom(questTitle) - stop.AlternativeQuestConditions = null.StringFrom(string(questConditions)) - stop.AlternativeQuestRewards = null.StringFrom(string(questRewards)) - stop.AlternativeQuestTimestamp = null.IntFrom(questTimestamp) - stop.AlternativeQuestExpiry = questExpiry + stop.SetAlternativeQuestType(null.IntFrom(questType)) + stop.SetAlternativeQuestTarget(null.IntFrom(questTarget)) + stop.SetAlternativeQuestTemplate(null.StringFrom(questTemplate)) + stop.SetAlternativeQuestTitle(null.StringFrom(questTitle)) + stop.SetAlternativeQuestConditions(null.StringFrom(string(questConditions))) + stop.SetAlternativeQuestRewards(null.StringFrom(string(questRewards))) + stop.SetAlternativeQuestTimestamp(null.IntFrom(questTimestamp)) + stop.SetAlternativeQuestExpiry(questExpiry) } else { - stop.QuestType = null.IntFrom(questType) - stop.QuestTarget = null.IntFrom(questTarget) - stop.QuestTemplate = null.StringFrom(questTemplate) - stop.QuestTitle = null.StringFrom(questTitle) - stop.QuestConditions = null.StringFrom(string(questConditions)) - stop.QuestRewards = null.StringFrom(string(questRewards)) - stop.QuestTimestamp = null.IntFrom(questTimestamp) - stop.QuestExpiry = questExpiry + stop.SetQuestType(null.IntFrom(questType)) + stop.SetQuestTarget(null.IntFrom(questTarget)) + stop.SetQuestTemplate(null.StringFrom(questTemplate)) + stop.SetQuestTitle(null.StringFrom(questTitle)) + stop.SetQuestConditions(null.StringFrom(string(questConditions))) + stop.SetQuestRewards(null.StringFrom(string(questRewards))) + stop.SetQuestTimestamp(null.IntFrom(questTimestamp)) + stop.SetQuestExpiry(questExpiry) } return questTitle } func (stop *Pokestop) updatePokestopFromFortDetailsProto(fortData *pogo.FortDetailsOutProto) *Pokestop { - stop.Id = fortData.Id - stop.Lat = fortData.Latitude - stop.Lon = fortData.Longitude + stop.SetId(fortData.Id) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) if len(fortData.ImageUrl) > 0 { - stop.Url = null.StringFrom(fortData.ImageUrl[0]) + stop.SetUrl(null.StringFrom(fortData.ImageUrl[0])) } - stop.Name = null.StringFrom(fortData.Name) + stop.SetName(null.StringFrom(fortData.Name)) if fortData.Description == "" { - stop.Description = null.NewString("", false) + stop.SetDescription(null.NewString("", false)) } else { - stop.Description = null.StringFrom(fortData.Description) + stop.SetDescription(null.StringFrom(fortData.Description)) } if fortData.Modifier != nil && len(fortData.Modifier) > 0 { @@ -556,22 +856,22 @@ func (stop *Pokestop) updatePokestopFromFortDetailsProto(fortData *pogo.FortDeta lureId := int16(fortData.Modifier[0].ModifierType) lureExpiry := fortData.Modifier[0].ExpirationTimeMs / 1000 - stop.LureId = lureId - stop.LureExpireTimestamp = null.IntFrom(lureExpiry) + stop.SetLureId(lureId) + stop.SetLureExpireTimestamp(null.IntFrom(lureExpiry)) } return stop } func (stop *Pokestop) updatePokestopFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto) *Pokestop { - stop.Id = fortData.Id - stop.Lat = fortData.Latitude - stop.Lon = fortData.Longitude + stop.SetId(fortData.Id) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) if len(fortData.Image) > 0 { - stop.Url = null.StringFrom(fortData.Image[0].Url) + stop.SetUrl(null.StringFrom(fortData.Image[0].Url)) } - stop.Name = null.StringFrom(fortData.Name) + stop.SetName(null.StringFrom(fortData.Name)) if stop.Deleted { log.Debugf("Cleared Stop with id '%s' is found again in GMF, therefore kept deleted", stop.Id) } @@ -579,8 +879,8 @@ func (stop *Pokestop) updatePokestopFromGetMapFortsOutProto(fortData *pogo.GetMa } func (stop *Pokestop) updatePokestopFromGetContestDataOutProto(contest *pogo.ContestProto) { - stop.ShowcaseRankingStandard = null.IntFrom(int64(contest.GetMetric().GetRankingStandard())) - stop.ShowcaseExpiry = null.IntFrom(contest.GetSchedule().GetContestCycle().GetEndTimeMs() / 1000) + stop.SetShowcaseRankingStandard(null.IntFrom(int64(contest.GetMetric().GetRankingStandard()))) + stop.SetShowcaseExpiry(null.IntFrom(contest.GetSchedule().GetContestCycle().GetEndTimeMs() / 1000)) focusStore := createFocusStoreFromContestProto(contest) @@ -594,7 +894,7 @@ func (stop *Pokestop) updatePokestopFromGetContestDataOutProto(contest *pogo.Con if err != nil { log.Errorf("SHOWCASE: Stop '%s' - Focus '%v' marshalling failed: %s", stop.Id, focus, err) } - stop.ShowcaseFocus = null.StringFrom(string(jsonBytes)) + stop.SetShowcaseFocus(null.StringFrom(string(jsonBytes))) // still support old format - probably still required to filter in external tools stop.extractShowcasePokemonInfoDeprecated(key, focus) } @@ -646,24 +946,32 @@ func (stop *Pokestop) updatePokestopFromGetPokemonSizeContestEntryOutProto(conte } jsonString, _ := json.Marshal(j) - stop.ShowcaseRankings = null.StringFrom(string(jsonString)) + stop.SetShowcaseRankings(null.StringFrom(string(jsonString))) } -func createPokestopFortWebhooks(oldStop *Pokestop, stop *Pokestop) { +func createPokestopFortWebhooks(stop *Pokestop) { fort := InitWebHookFortFromPokestop(stop) - oldFort := InitWebHookFortFromPokestop(oldStop) - if oldStop == nil { - CreateFortWebHooks(oldFort, fort, NEW) + if stop.newRecord { + CreateFortWebHooks(nil, fort, NEW) } else { + // Build old fort from saved old values + oldFort := &FortWebhook{ + Type: POKESTOP.String(), + Id: stop.Id, + Name: stop.oldName.Ptr(), + ImageUrl: stop.oldUrl.Ptr(), + Description: stop.oldDescription.Ptr(), + Location: Location{Latitude: stop.oldLat, Longitude: stop.oldLon}, + } CreateFortWebHooks(oldFort, fort, EDIT) } } -func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { +func createPokestopWebhooks(stop *Pokestop) { areas := MatchStatsGeofence(stop.Lat, stop.Lon) - if stop.AlternativeQuestType.Valid && (oldStop == nil || stop.AlternativeQuestType != oldStop.AlternativeQuestType) { + if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldAlternativeQuestType) { questHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -689,7 +997,7 @@ func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } - if stop.QuestType.Valid && (oldStop == nil || stop.QuestType != oldStop.QuestType) { + if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldQuestType) { questHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -714,7 +1022,7 @@ func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } - if (oldStop == nil && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (oldStop != nil && ((stop.LureExpireTimestamp != oldStop.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != oldStop.PowerUpEndTimestamp)) { + if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldLureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldPowerUpEndTimestamp)) { pokestopHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -756,19 +1064,16 @@ func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { } func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { - oldPokestop, _ := GetPokestopRecord(ctx, db, pokestop.Id) now := time.Now().Unix() - if oldPokestop != nil && !hasChangesPokestop(oldPokestop, pokestop) { - if oldPokestop.Updated > now-900 { + if !pokestop.IsNewRecord() && !pokestop.IsDirty() { + if pokestop.Updated > now-900 { // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again return } } pokestop.Updated = now - //log.Traceln(cmp.Diff(oldPokestop, pokestop)) - - if oldPokestop == nil { + if pokestop.IsNewRecord() { res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO pokestop ( id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, @@ -800,6 +1105,7 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } _ = res } else { + // Existing record - UPDATE res, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE pokestop SET lat = :lat, @@ -810,17 +1116,17 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop lure_expire_timestamp = :lure_expire_timestamp, last_modified_timestamp = :last_modified_timestamp, updated = :updated, - quest_type = :quest_type, - quest_timestamp = :quest_timestamp, - quest_target = :quest_target, - quest_conditions = :quest_conditions, - quest_rewards = :quest_rewards, - quest_template = :quest_template, + quest_type = :quest_type, + quest_timestamp = :quest_timestamp, + quest_target = :quest_target, + quest_conditions = :quest_conditions, + quest_rewards = :quest_rewards, + quest_template = :quest_template, quest_title = :quest_title, - alternative_quest_type = :alternative_quest_type, + alternative_quest_type = :alternative_quest_type, alternative_quest_timestamp = :alternative_quest_timestamp, - alternative_quest_target = :alternative_quest_target, - alternative_quest_conditions = :alternative_quest_conditions, + alternative_quest_target = :alternative_quest_target, + alternative_quest_conditions = :alternative_quest_conditions, alternative_quest_rewards = :alternative_quest_rewards, alternative_quest_template = :alternative_quest_template, alternative_quest_title = :alternative_quest_title, @@ -854,9 +1160,11 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } _ = res } - pokestopCache.Set(pokestop.Id, *pokestop, ttlcache.DefaultTTL) - createPokestopWebhooks(oldPokestop, pokestop) - createPokestopFortWebhooks(oldPokestop, pokestop) + pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + pokestop.newRecord = false // After saving, it's no longer a new record + pokestop.ClearDirty() + createPokestopWebhooks(pokestop) + createPokestopFortWebhooks(pokestop) } func updatePokestopGetMapFortCache(pokestop *Pokestop) { @@ -881,7 +1189,7 @@ func UpdatePokestopRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDe } if pokestop == nil { - pokestop = &Pokestop{} + pokestop = &Pokestop{newRecord: true} } pokestop.updatePokestopFromFortDetailsProto(fort) @@ -913,7 +1221,7 @@ func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.F } if pokestop == nil { - pokestop = &Pokestop{} + pokestop = &Pokestop{newRecord: true} } questTitle := pokestop.updatePokestopFromQuestProto(quest, haveAr) diff --git a/decoder/pokestop_showcase.go b/decoder/pokestop_showcase.go index bf9de8a2..42876295 100644 --- a/decoder/pokestop_showcase.go +++ b/decoder/pokestop_showcase.go @@ -95,28 +95,28 @@ func createFocusStoreFromContestProto(contest *pogo.ContestProto) map[contestFoc func (stop *Pokestop) extractShowcasePokemonInfoDeprecated(key contestFocusType, focus map[string]any) { if key == focusPokemon { if pokemonID, ok := focus["pokemon_id"].(int32); ok { - stop.ShowcasePokemon = null.IntFrom(int64(pokemonID)) + stop.SetShowcasePokemon(null.IntFrom(int64(pokemonID))) } else { log.Warnf("SHOWCASE: Stop '%s' - Missing or invalid 'pokemon_id'", stop.Id) - stop.ShowcasePokemon = null.IntFromPtr(nil) + stop.SetShowcasePokemon(null.IntFromPtr(nil)) } if form, ok := focus["pokemon_form"].(int32); ok { - stop.ShowcasePokemonForm = null.IntFrom(int64(form)) + stop.SetShowcasePokemonForm(null.IntFrom(int64(form))) } else { - stop.ShowcasePokemonForm = null.IntFromPtr(nil) + stop.SetShowcasePokemonForm(null.IntFromPtr(nil)) } } else { - stop.ShowcasePokemon = null.IntFromPtr(nil) - stop.ShowcasePokemonForm = null.IntFromPtr(nil) + stop.SetShowcasePokemon(null.IntFromPtr(nil)) + stop.SetShowcasePokemonForm(null.IntFromPtr(nil)) } if key == focusPokemonType { if type1, ok := focus["pokemon_type_1"].(int32); ok { - stop.ShowcasePokemonType = null.IntFrom(int64(type1)) + stop.SetShowcasePokemonType(null.IntFrom(int64(type1))) } else { log.Warnf("SHOWCASE: Stop '%s' - Missing or invalid 'pokemon_type_1'", stop.Id) - stop.ShowcasePokemonType = null.IntFromPtr(nil) + stop.SetShowcasePokemonType(null.IntFromPtr(nil)) } if type2, ok := focus["pokemon_type_2"].(int32); ok { @@ -125,6 +125,6 @@ func (stop *Pokestop) extractShowcasePokemonInfoDeprecated(key contestFocusType, } } } else { - stop.ShowcasePokemonType = null.IntFromPtr(nil) + stop.SetShowcasePokemonType(null.IntFromPtr(nil)) } } From d23858962e3637146ca3edbd4046a53589012340 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 11:24:40 +0000 Subject: [PATCH 05/78] Update pokemon and gym to use new pattern --- decoder/api_pokemon.go | 4 +- decoder/api_pokemon_scan_v1.go | 2 +- decoder/api_pokemon_scan_v2.go | 2 +- decoder/api_pokemon_scan_v3.go | 2 +- decoder/fort.go | 16 +- decoder/gym.go | 620 +++++++++++++++++++++++++-------- decoder/main.go | 29 +- decoder/pokemon.go | 598 +++++++++++++++++++++---------- decoder/pokemonRtree.go | 13 +- decoder/pokestop.go | 141 ++++---- decoder/stats.go | 61 ++-- decoder/weather_iv.go | 4 +- 12 files changed, 1036 insertions(+), 456 deletions(-) diff --git a/decoder/api_pokemon.go b/decoder/api_pokemon.go index 0126cad4..c41393af 100644 --- a/decoder/api_pokemon.go +++ b/decoder/api_pokemon.go @@ -130,7 +130,7 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { if found { if pokemonCacheEntry := getPokemonFromCache(pokemonId); pokemonCacheEntry != nil { pokemon := pokemonCacheEntry.Value() - results = append(results, &pokemon) + results = append(results, pokemon) pokemonMatched++ if pokemonMatched > maxPokemon { @@ -153,7 +153,7 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { func GetOnePokemon(pokemonId uint64) *ApiPokemonResult { if item := getPokemonFromCache(pokemonId); item != nil { pokemon := item.Value() - apiPokemon := buildApiPokemonResult(&pokemon) + apiPokemon := buildApiPokemonResult(pokemon) return &apiPokemon } return nil diff --git a/decoder/api_pokemon_scan_v1.go b/decoder/api_pokemon_scan_v1.go index 737393ad..c4baf317 100644 --- a/decoder/api_pokemon_scan_v1.go +++ b/decoder/api_pokemon_scan_v1.go @@ -230,7 +230,7 @@ func GetPokemonInArea(retrieveParameters ApiPokemonScan) []*ApiPokemonResult { if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { pokemon := pokemonCacheEntry.Value() - apiPokemon := buildApiPokemonResult(&pokemon) + apiPokemon := buildApiPokemonResult(pokemon) results = append(results, &apiPokemon) } } diff --git a/decoder/api_pokemon_scan_v2.go b/decoder/api_pokemon_scan_v2.go index 878f3ab5..726d0f3f 100644 --- a/decoder/api_pokemon_scan_v2.go +++ b/decoder/api_pokemon_scan_v2.go @@ -110,7 +110,7 @@ func GetPokemonInArea2(retrieveParameters ApiPokemonScan2) []*ApiPokemonResult { continue } - apiPokemon := buildApiPokemonResult(&pokemon) + apiPokemon := buildApiPokemonResult(pokemon) results = append(results, &apiPokemon) } diff --git a/decoder/api_pokemon_scan_v3.go b/decoder/api_pokemon_scan_v3.go index 8c5ef4ad..a7c3b14d 100644 --- a/decoder/api_pokemon_scan_v3.go +++ b/decoder/api_pokemon_scan_v3.go @@ -118,7 +118,7 @@ func GetPokemonInArea3(retrieveParameters ApiPokemonScan3) *PokemonScan3Result { continue } - apiPokemon := buildApiPokemonResult(&pokemon) + apiPokemon := buildApiPokemonResult(pokemon) results = append(results, &apiPokemon) } diff --git a/decoder/fort.go b/decoder/fort.go index b4b8a08c..fe008724 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -217,28 +217,28 @@ func UpdateFortRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetail // copySharedFieldsFrom copies shared fields from a pokestop to a gym during conversion func (gym *Gym) copySharedFieldsFrom(pokestop *Pokestop) { if pokestop.Name.Valid && !gym.Name.Valid { - gym.Name = pokestop.Name + gym.SetName(pokestop.Name) } if pokestop.Url.Valid && !gym.Url.Valid { - gym.Url = pokestop.Url + gym.SetUrl(pokestop.Url) } if pokestop.Description.Valid && !gym.Description.Valid { - gym.Description = pokestop.Description + gym.SetDescription(pokestop.Description) } if pokestop.PartnerId.Valid && !gym.PartnerId.Valid { - gym.PartnerId = pokestop.PartnerId + gym.SetPartnerId(pokestop.PartnerId) } if pokestop.ArScanEligible.Valid && !gym.ArScanEligible.Valid { - gym.ArScanEligible = pokestop.ArScanEligible + gym.SetArScanEligible(pokestop.ArScanEligible) } if pokestop.PowerUpLevel.Valid && !gym.PowerUpLevel.Valid { - gym.PowerUpLevel = pokestop.PowerUpLevel + gym.SetPowerUpLevel(pokestop.PowerUpLevel) } if pokestop.PowerUpPoints.Valid && !gym.PowerUpPoints.Valid { - gym.PowerUpPoints = pokestop.PowerUpPoints + gym.SetPowerUpPoints(pokestop.PowerUpPoints) } if pokestop.PowerUpEndTimestamp.Valid && !gym.PowerUpEndTimestamp.Valid { - gym.PowerUpEndTimestamp = pokestop.PowerUpEndTimestamp + gym.SetPowerUpEndTimestamp(pokestop.PowerUpEndTimestamp) } } diff --git a/decoder/gym.go b/decoder/gym.go index 4f1784fe..a8b6541d 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -67,45 +67,67 @@ type Gym struct { Description null.String `db:"description" json:"description"` Defenders null.String `db:"defenders" json:"defenders"` Rsvps null.String `db:"rsvps" json:"rsvps"` - //`id` varchar(35) NOT NULL, - //`lat` double(18,14) NOT NULL, - //`lon` double(18,14) NOT NULL, - //`name` varchar(128) DEFAULT NULL, - //`url` varchar(200) DEFAULT NULL, - //`last_modified_timestamp` int unsigned DEFAULT NULL, - //`raid_end_timestamp` int unsigned DEFAULT NULL, - //`raid_spawn_timestamp` int unsigned DEFAULT NULL, - //`raid_battle_timestamp` int unsigned DEFAULT NULL, - //`updated` int unsigned NOT NULL, - //`raid_pokemon_id` smallint unsigned DEFAULT NULL, - //`guarding_pokemon_id` smallint unsigned DEFAULT NULL, - //`available_slots` smallint unsigned DEFAULT NULL, - //`availble_slots` smallint unsigned GENERATED ALWAYS AS (`available_slots`) VIRTUAL, - //`team_id` tinyint unsigned DEFAULT NULL, - //`raid_level` tinyint unsigned DEFAULT NULL, - //`enabled` tinyint unsigned DEFAULT NULL, - //`ex_raid_eligible` tinyint unsigned DEFAULT NULL, - //`in_battle` tinyint unsigned DEFAULT NULL, - //`raid_pokemon_move_1` smallint unsigned DEFAULT NULL, - //`raid_pokemon_move_2` smallint unsigned DEFAULT NULL, - //`raid_pokemon_form` smallint unsigned DEFAULT NULL, - //`raid_pokemon_cp` int unsigned DEFAULT NULL, - //`raid_is_exclusive` tinyint unsigned DEFAULT NULL, - //`cell_id` bigint unsigned DEFAULT NULL, - //`deleted` tinyint unsigned NOT NULL DEFAULT '0', - //`total_cp` int unsigned DEFAULT NULL, - //`first_seen_timestamp` int unsigned NOT NULL, - //`raid_pokemon_gender` tinyint unsigned DEFAULT NULL, - //`sponsor_id` smallint unsigned DEFAULT NULL, - //`partner_id` varchar(35) DEFAULT NULL, - //`raid_pokemon_costume` smallint unsigned DEFAULT NULL, - //`raid_pokemon_evolution` tinyint unsigned DEFAULT NULL, - //`ar_scan_eligible` tinyint unsigned DEFAULT NULL, - //`power_up_level` smallint unsigned DEFAULT NULL, - //`power_up_points` int unsigned DEFAULT NULL, - //`power_up_end_timestamp` int unsigned DEFAULT NULL, + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + oldValues GymOldValues `db:"-" json:"-"` // Old values for webhook comparison } +// GymOldValues holds old field values for webhook comparison (populated when loading from cache/DB) +type GymOldValues struct { + Name null.String + Url null.String + Description null.String + Lat float64 + Lon float64 + TeamId null.Int + AvailableSlots null.Int + RaidLevel null.Int + RaidPokemonId null.Int + RaidSpawnTimestamp null.Int + Rsvps null.String + InBattle null.Int +} + +//`id` varchar(35) NOT NULL, +//`lat` double(18,14) NOT NULL, +//`lon` double(18,14) NOT NULL, +//`name` varchar(128) DEFAULT NULL, +//`url` varchar(200) DEFAULT NULL, +//`last_modified_timestamp` int unsigned DEFAULT NULL, +//`raid_end_timestamp` int unsigned DEFAULT NULL, +//`raid_spawn_timestamp` int unsigned DEFAULT NULL, +//`raid_battle_timestamp` int unsigned DEFAULT NULL, +//`updated` int unsigned NOT NULL, +//`raid_pokemon_id` smallint unsigned DEFAULT NULL, +//`guarding_pokemon_id` smallint unsigned DEFAULT NULL, +//`available_slots` smallint unsigned DEFAULT NULL, +//`availble_slots` smallint unsigned GENERATED ALWAYS AS (`available_slots`) VIRTUAL, +//`team_id` tinyint unsigned DEFAULT NULL, +//`raid_level` tinyint unsigned DEFAULT NULL, +//`enabled` tinyint unsigned DEFAULT NULL, +//`ex_raid_eligible` tinyint unsigned DEFAULT NULL, +//`in_battle` tinyint unsigned DEFAULT NULL, +//`raid_pokemon_move_1` smallint unsigned DEFAULT NULL, +//`raid_pokemon_move_2` smallint unsigned DEFAULT NULL, +//`raid_pokemon_form` smallint unsigned DEFAULT NULL, +//`raid_pokemon_cp` int unsigned DEFAULT NULL, +//`raid_is_exclusive` tinyint unsigned DEFAULT NULL, +//`cell_id` bigint unsigned DEFAULT NULL, +//`deleted` tinyint unsigned NOT NULL DEFAULT '0', +//`total_cp` int unsigned DEFAULT NULL, +//`first_seen_timestamp` int unsigned NOT NULL, +//`raid_pokemon_gender` tinyint unsigned DEFAULT NULL, +//`sponsor_id` smallint unsigned DEFAULT NULL, +//`partner_id` varchar(35) DEFAULT NULL, +//`raid_pokemon_costume` smallint unsigned DEFAULT NULL, +//`raid_pokemon_evolution` tinyint unsigned DEFAULT NULL, +//`ar_scan_eligible` tinyint unsigned DEFAULT NULL, +//`power_up_level` smallint unsigned DEFAULT NULL, +//`power_up_points` int unsigned DEFAULT NULL, +//`power_up_end_timestamp` int unsigned DEFAULT NULL, + // //SELECT CONCAT("'", GROUP_CONCAT(column_name ORDER BY ordinal_position SEPARATOR "', '"), "'") AS columns //FROM information_schema.columns @@ -115,11 +137,323 @@ type Gym struct { //FROM information_schema.columns //WHERE table_schema = 'db_name' AND table_name = 'tbl_name' +// IsDirty returns true if any field has been modified +func (gym *Gym) IsDirty() bool { + return gym.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (gym *Gym) ClearDirty() { + gym.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (gym *Gym) IsNewRecord() bool { + return gym.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (gym *Gym) snapshotOldValues() { + gym.oldValues = GymOldValues{ + Name: gym.Name, + Url: gym.Url, + Description: gym.Description, + Lat: gym.Lat, + Lon: gym.Lon, + TeamId: gym.TeamId, + AvailableSlots: gym.AvailableSlots, + RaidLevel: gym.RaidLevel, + RaidPokemonId: gym.RaidPokemonId, + RaidSpawnTimestamp: gym.RaidSpawnTimestamp, + Rsvps: gym.Rsvps, + InBattle: gym.InBattle, + } +} + +// --- Set methods with dirty tracking --- + +func (gym *Gym) SetId(v string) { + if gym.Id != v { + gym.Id = v + gym.dirty = true + } +} + +func (gym *Gym) SetLat(v float64) { + if !floatAlmostEqual(gym.Lat, v, floatTolerance) { + gym.Lat = v + gym.dirty = true + } +} + +func (gym *Gym) SetLon(v float64) { + if !floatAlmostEqual(gym.Lon, v, floatTolerance) { + gym.Lon = v + gym.dirty = true + } +} + +func (gym *Gym) SetName(v null.String) { + if gym.Name != v { + gym.Name = v + gym.dirty = true + } +} + +func (gym *Gym) SetUrl(v null.String) { + if gym.Url != v { + gym.Url = v + gym.dirty = true + } +} + +func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { + if gym.LastModifiedTimestamp != v { + gym.LastModifiedTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidEndTimestamp(v null.Int) { + if gym.RaidEndTimestamp != v { + gym.RaidEndTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { + if gym.RaidSpawnTimestamp != v { + gym.RaidSpawnTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { + if gym.RaidBattleTimestamp != v { + gym.RaidBattleTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonId(v null.Int) { + if gym.RaidPokemonId != v { + gym.RaidPokemonId = v + gym.dirty = true + } +} + +func (gym *Gym) SetGuardingPokemonId(v null.Int) { + if gym.GuardingPokemonId != v { + gym.GuardingPokemonId = v + gym.dirty = true + } +} + +func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { + if gym.GuardingPokemonDisplay != v { + gym.GuardingPokemonDisplay = v + gym.dirty = true + } +} + +func (gym *Gym) SetAvailableSlots(v null.Int) { + if gym.AvailableSlots != v { + gym.AvailableSlots = v + gym.dirty = true + } +} + +func (gym *Gym) SetTeamId(v null.Int) { + if gym.TeamId != v { + gym.TeamId = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidLevel(v null.Int) { + if gym.RaidLevel != v { + gym.RaidLevel = v + gym.dirty = true + } +} + +func (gym *Gym) SetEnabled(v null.Int) { + if gym.Enabled != v { + gym.Enabled = v + gym.dirty = true + } +} + +func (gym *Gym) SetExRaidEligible(v null.Int) { + if gym.ExRaidEligible != v { + gym.ExRaidEligible = v + gym.dirty = true + } +} + +func (gym *Gym) SetInBattle(v null.Int) { + if gym.InBattle != v { + gym.InBattle = v + //Do not set to dirty, as don't trigger an update + //gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonMove1(v null.Int) { + if gym.RaidPokemonMove1 != v { + gym.RaidPokemonMove1 = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonMove2(v null.Int) { + if gym.RaidPokemonMove2 != v { + gym.RaidPokemonMove2 = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonForm(v null.Int) { + if gym.RaidPokemonForm != v { + gym.RaidPokemonForm = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { + if gym.RaidPokemonAlignment != v { + gym.RaidPokemonAlignment = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonCp(v null.Int) { + if gym.RaidPokemonCp != v { + gym.RaidPokemonCp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidIsExclusive(v null.Int) { + if gym.RaidIsExclusive != v { + gym.RaidIsExclusive = v + gym.dirty = true + } +} + +func (gym *Gym) SetCellId(v null.Int) { + if gym.CellId != v { + gym.CellId = v + gym.dirty = true + } +} + +func (gym *Gym) SetDeleted(v bool) { + if gym.Deleted != v { + gym.Deleted = v + gym.dirty = true + } +} + +func (gym *Gym) SetTotalCp(v null.Int) { + if gym.TotalCp != v { + gym.TotalCp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonGender(v null.Int) { + if gym.RaidPokemonGender != v { + gym.RaidPokemonGender = v + gym.dirty = true + } +} + +func (gym *Gym) SetSponsorId(v null.Int) { + if gym.SponsorId != v { + gym.SponsorId = v + gym.dirty = true + } +} + +func (gym *Gym) SetPartnerId(v null.String) { + if gym.PartnerId != v { + gym.PartnerId = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonCostume(v null.Int) { + if gym.RaidPokemonCostume != v { + gym.RaidPokemonCostume = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { + if gym.RaidPokemonEvolution != v { + gym.RaidPokemonEvolution = v + gym.dirty = true + } +} + +func (gym *Gym) SetArScanEligible(v null.Int) { + if gym.ArScanEligible != v { + gym.ArScanEligible = v + gym.dirty = true + } +} + +func (gym *Gym) SetPowerUpLevel(v null.Int) { + if gym.PowerUpLevel != v { + gym.PowerUpLevel = v + gym.dirty = true + } +} + +func (gym *Gym) SetPowerUpPoints(v null.Int) { + if gym.PowerUpPoints != v { + gym.PowerUpPoints = v + gym.dirty = true + } +} + +func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { + if gym.PowerUpEndTimestamp != v { + gym.PowerUpEndTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetDescription(v null.String) { + if gym.Description != v { + gym.Description = v + gym.dirty = true + } +} + +func (gym *Gym) SetDefenders(v null.String) { + if gym.Defenders != v { + gym.Defenders = v + //Do not set to dirty, as don't trigger an update + //gym.dirty = true + } +} + +func (gym *Gym) SetRsvps(v null.String) { + if gym.Rsvps != v { + gym.Rsvps = v + gym.dirty = true + } +} + func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, error) { inMemoryGym := gymCache.Get(fortId) if inMemoryGym != nil { gym := inMemoryGym.Value() - return &gym, nil + gym.snapshotOldValues() // Snapshot for webhook comparison + return gym, nil } gym := Gym{} err := db.GeneralDb.GetContext(ctx, &gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) @@ -133,7 +467,8 @@ func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, er return nil, err } - gymCache.Set(fortId, gym, ttlcache.DefaultTTL) + gym.snapshotOldValues() // Snapshot for webhook comparison + gymCache.Set(fortId, &gym, ttlcache.DefaultTTL) if config.Config.TestFortInMemory { fortRtreeUpdateGymOnGet(&gym) } @@ -183,13 +518,13 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64 Badge int `json:"badge,omitempty"` Background *int64 `json:"background,omitempty"` } - gym.Id = fortData.FortId - gym.Lat = fortData.Latitude //fmt.Sprintf("%f", fortData.Latitude) - gym.Lon = fortData.Longitude //fmt.Sprintf("%f", fortData.Longitude) - gym.Enabled = null.IntFrom(util.BoolToInt[int64](fortData.Enabled)) - gym.GuardingPokemonId = null.IntFrom(int64(fortData.GuardPokemonId)) + gym.SetId(fortData.FortId) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) + gym.SetEnabled(null.IntFrom(util.BoolToInt[int64](fortData.Enabled))) + gym.SetGuardingPokemonId(null.IntFrom(int64(fortData.GuardPokemonId))) if fortData.GuardPokemonDisplay == nil { - gym.GuardingPokemonDisplay = null.NewString("", false) + gym.SetGuardingPokemonDisplay(null.NewString("", false)) } else { display, _ := json.Marshal(pokemonDisplay{ Form: int(fortData.GuardPokemonDisplay.Form), @@ -202,90 +537,91 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64 Badge: int(fortData.GuardPokemonDisplay.PokemonBadge), Background: util.ExtractBackgroundFromDisplay(fortData.GuardPokemonDisplay), }) - gym.GuardingPokemonDisplay = null.StringFrom(string(display)) + gym.SetGuardingPokemonDisplay(null.StringFrom(string(display))) } - gym.TeamId = null.IntFrom(int64(fortData.Team)) + gym.SetTeamId(null.IntFrom(int64(fortData.Team))) if fortData.GymDisplay != nil { - gym.AvailableSlots = null.IntFrom(int64(fortData.GymDisplay.SlotsAvailable)) + gym.SetAvailableSlots(null.IntFrom(int64(fortData.GymDisplay.SlotsAvailable))) } else { - gym.AvailableSlots = null.IntFrom(6) // this may be an incorrect assumption + gym.SetAvailableSlots(null.IntFrom(6)) // this may be an incorrect assumption } - gym.LastModifiedTimestamp = null.IntFrom(fortData.LastModifiedMs / 1000) - gym.ExRaidEligible = null.IntFrom(util.BoolToInt[int64](fortData.IsExRaidEligible)) + gym.SetLastModifiedTimestamp(null.IntFrom(fortData.LastModifiedMs / 1000)) + gym.SetExRaidEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsExRaidEligible))) if fortData.ImageUrl != "" { - gym.Url = null.StringFrom(fortData.ImageUrl) + gym.SetUrl(null.StringFrom(fortData.ImageUrl)) } - gym.InBattle = null.IntFrom(util.BoolToInt[int64](fortData.IsInBattle)) - gym.ArScanEligible = null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible)) - gym.PowerUpPoints = null.IntFrom(int64(fortData.PowerUpProgressPoints)) + gym.SetInBattle(null.IntFrom(util.BoolToInt[int64](fortData.IsInBattle))) + gym.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) + gym.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) - gym.PowerUpLevel, gym.PowerUpEndTimestamp = calculatePowerUpPoints(fortData) + powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) + gym.SetPowerUpLevel(powerUpLevel) + gym.SetPowerUpEndTimestamp(powerUpEndTimestamp) if fortData.PartnerId == "" { - gym.PartnerId = null.NewString("", false) + gym.SetPartnerId(null.NewString("", false)) } else { - gym.PartnerId = null.StringFrom(fortData.PartnerId) + gym.SetPartnerId(null.StringFrom(fortData.PartnerId)) } if fortData.ImageUrl != "" { - gym.Url = null.StringFrom(fortData.ImageUrl) - + gym.SetUrl(null.StringFrom(fortData.ImageUrl)) } if fortData.Team == 0 { // check!! - gym.TotalCp = null.IntFrom(0) + gym.SetTotalCp(null.IntFrom(0)) } else { if fortData.GymDisplay != nil { totalCp := int64(fortData.GymDisplay.TotalGymCp) if gym.TotalCp.Int64-totalCp > 100 || totalCp-gym.TotalCp.Int64 > 100 { - gym.TotalCp = null.IntFrom(totalCp) + gym.SetTotalCp(null.IntFrom(totalCp)) } } else { - gym.TotalCp = null.IntFrom(0) + gym.SetTotalCp(null.IntFrom(0)) } } if fortData.RaidInfo != nil { - gym.RaidEndTimestamp = null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000) - gym.RaidSpawnTimestamp = null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000) + gym.SetRaidEndTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000)) + gym.SetRaidSpawnTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000)) raidBattleTimestamp := int64(fortData.RaidInfo.RaidBattleMs) / 1000 if gym.RaidBattleTimestamp.ValueOrZero() != raidBattleTimestamp { // We are reporting a new raid, clear rsvp data - gym.Rsvps = null.NewString("", false) + gym.SetRsvps(null.NewString("", false)) } - gym.RaidBattleTimestamp = null.IntFrom(raidBattleTimestamp) + gym.SetRaidBattleTimestamp(null.IntFrom(raidBattleTimestamp)) - gym.RaidLevel = null.IntFrom(int64(fortData.RaidInfo.RaidLevel)) + gym.SetRaidLevel(null.IntFrom(int64(fortData.RaidInfo.RaidLevel))) if fortData.RaidInfo.RaidPokemon != nil { - gym.RaidPokemonId = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonId)) - gym.RaidPokemonMove1 = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move1)) - gym.RaidPokemonMove2 = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move2)) - gym.RaidPokemonForm = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Form)) - gym.RaidPokemonAlignment = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Alignment)) - gym.RaidPokemonCp = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Cp)) - gym.RaidPokemonGender = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Gender)) - gym.RaidPokemonCostume = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Costume)) - gym.RaidPokemonEvolution = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.CurrentTempEvolution)) + gym.SetRaidPokemonId(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonId))) + gym.SetRaidPokemonMove1(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move1))) + gym.SetRaidPokemonMove2(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move2))) + gym.SetRaidPokemonForm(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Form))) + gym.SetRaidPokemonAlignment(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Alignment))) + gym.SetRaidPokemonCp(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Cp))) + gym.SetRaidPokemonGender(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Gender))) + gym.SetRaidPokemonCostume(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Costume))) + gym.SetRaidPokemonEvolution(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.CurrentTempEvolution))) } else { - gym.RaidPokemonId = null.IntFrom(0) - gym.RaidPokemonMove1 = null.IntFrom(0) - gym.RaidPokemonMove2 = null.IntFrom(0) - gym.RaidPokemonForm = null.IntFrom(0) - gym.RaidPokemonAlignment = null.IntFrom(0) - gym.RaidPokemonCp = null.IntFrom(0) - gym.RaidPokemonGender = null.IntFrom(0) - gym.RaidPokemonCostume = null.IntFrom(0) - gym.RaidPokemonEvolution = null.IntFrom(0) + gym.SetRaidPokemonId(null.IntFrom(0)) + gym.SetRaidPokemonMove1(null.IntFrom(0)) + gym.SetRaidPokemonMove2(null.IntFrom(0)) + gym.SetRaidPokemonForm(null.IntFrom(0)) + gym.SetRaidPokemonAlignment(null.IntFrom(0)) + gym.SetRaidPokemonCp(null.IntFrom(0)) + gym.SetRaidPokemonGender(null.IntFrom(0)) + gym.SetRaidPokemonCostume(null.IntFrom(0)) + gym.SetRaidPokemonEvolution(null.IntFrom(0)) } - gym.RaidIsExclusive = null.IntFrom(0) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive)) + gym.SetRaidIsExclusive(null.IntFrom(0)) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive)) } - gym.CellId = null.IntFrom(int64(cellId)) + gym.SetCellId(null.IntFrom(int64(cellId))) if gym.Deleted { - gym.Deleted = false + gym.SetDeleted(false) log.Warnf("Cleared Gym with id '%s' is found again in GMO, therefore un-deleted", gym.Id) // Restore in fort tracker if enabled if fortTracker != nil { @@ -297,32 +633,32 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64 } func (gym *Gym) updateGymFromFortProto(fortData *pogo.FortDetailsOutProto) *Gym { - gym.Id = fortData.Id - gym.Lat = fortData.Latitude //fmt.Sprintf("%f", fortData.Latitude) - gym.Lon = fortData.Longitude //fmt.Sprintf("%f", fortData.Longitude) + gym.SetId(fortData.Id) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) if len(fortData.ImageUrl) > 0 { - gym.Url = null.StringFrom(fortData.ImageUrl[0]) + gym.SetUrl(null.StringFrom(fortData.ImageUrl[0])) } - gym.Name = null.StringFrom(fortData.Name) + gym.SetName(null.StringFrom(fortData.Name)) return gym } func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) *Gym { - gym.Id = gymData.GymStatusAndDefenders.PokemonFortProto.FortId - gym.Lat = gymData.GymStatusAndDefenders.PokemonFortProto.Latitude - gym.Lon = gymData.GymStatusAndDefenders.PokemonFortProto.Longitude + gym.SetId(gymData.GymStatusAndDefenders.PokemonFortProto.FortId) + gym.SetLat(gymData.GymStatusAndDefenders.PokemonFortProto.Latitude) + gym.SetLon(gymData.GymStatusAndDefenders.PokemonFortProto.Longitude) // This will have gym defenders in it... if len(gymData.Url) > 0 { - gym.Url = null.StringFrom(gymData.Url) + gym.SetUrl(null.StringFrom(gymData.Url)) } - gym.Name = null.StringFrom(gymData.Name) + gym.SetName(null.StringFrom(gymData.Name)) if gymData.Description == "" { - gym.Description = null.NewString("", false) + gym.SetDescription(null.NewString("", false)) } else { - gym.Description = null.StringFrom(gymData.Description) + gym.SetDescription(null.StringFrom(gymData.Description)) } type pokemonGymDefender struct { @@ -377,22 +713,22 @@ func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) * defenders = append(defenders, defender) } bDefenders, _ := json.Marshal(defenders) - gym.Defenders = null.StringFrom(string(bDefenders)) + gym.SetDefenders(null.StringFrom(string(bDefenders))) // log.Debugf("Gym %s defenders %s ", gym.Id, string(bDefenders)) return gym } func (gym *Gym) updateGymFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto, skipName bool) *Gym { - gym.Id = fortData.Id - gym.Lat = fortData.Latitude - gym.Lon = fortData.Longitude + gym.SetId(fortData.Id) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) if len(fortData.Image) > 0 { - gym.Url = null.StringFrom(fortData.Image[0].Url) + gym.SetUrl(null.StringFrom(fortData.Image[0].Url)) } if !skipName { - gym.Name = null.StringFrom(fortData.Name) + gym.SetName(null.StringFrom(fortData.Name)) } if gym.Deleted { @@ -422,14 +758,14 @@ func (gym *Gym) updateGymFromRsvpProto(fortData *pogo.GetEventRsvpsOutProto) *Gy } if len(timeslots) == 0 { - gym.Rsvps = null.NewString("", false) + gym.SetRsvps(null.NewString("", false)) } else { slices.SortFunc(timeslots, func(a, b rsvpTimeslot) int { return cmp.Compare(a.Timeslot, b.Timeslot) }) bRsvps, _ := json.Marshal(timeslots) - gym.Rsvps = null.StringFrom(string(bRsvps)) + gym.SetRsvps(null.StringFrom(string(bRsvps))) } return gym @@ -524,19 +860,27 @@ type GymDetailsWebhook struct { //"ar_scan_eligible": arScanEligible ?? 0 } -func createGymFortWebhooks(oldGym *Gym, gym *Gym) { +func createGymFortWebhooks(gym *Gym) { fort := InitWebHookFortFromGym(gym) - oldFort := InitWebHookFortFromGym(oldGym) - if oldGym == nil { - CreateFortWebHooks(oldFort, fort, NEW) + if gym.newRecord { + CreateFortWebHooks(nil, fort, NEW) } else { + // Build old fort from saved old values + oldFort := &FortWebhook{ + Type: GYM.String(), + Id: gym.Id, + Name: gym.oldValues.Name.Ptr(), + ImageUrl: gym.oldValues.Url.Ptr(), + Description: gym.oldValues.Description.Ptr(), + Location: Location{Latitude: gym.oldValues.Lat, Longitude: gym.oldValues.Lon}, + } CreateFortWebHooks(oldFort, fort, EDIT) } } -func createGymWebhooks(oldGym *Gym, gym *Gym, areas []geo.AreaName) { - if oldGym == nil || - (oldGym.AvailableSlots != gym.AvailableSlots || oldGym.TeamId != gym.TeamId || oldGym.InBattle != gym.InBattle) { +func createGymWebhooks(gym *Gym, areas []geo.AreaName) { + if gym.newRecord || + (gym.oldValues.AvailableSlots != gym.AvailableSlots || gym.oldValues.TeamId != gym.TeamId || gym.oldValues.InBattle != gym.InBattle) { gymDetails := GymDetailsWebhook{ Id: gym.Id, Name: gym.Name.ValueOrZero(), @@ -567,9 +911,9 @@ func createGymWebhooks(oldGym *Gym, gym *Gym, areas []geo.AreaName) { } if gym.RaidSpawnTimestamp.ValueOrZero() > 0 && - (oldGym == nil || oldGym.RaidLevel != gym.RaidLevel || - oldGym.RaidPokemonId != gym.RaidPokemonId || - oldGym.RaidSpawnTimestamp != gym.RaidSpawnTimestamp || oldGym.Rsvps != gym.Rsvps) { + (gym.newRecord || gym.oldValues.RaidLevel != gym.RaidLevel || + gym.oldValues.RaidPokemonId != gym.RaidPokemonId || + gym.oldValues.RaidSpawnTimestamp != gym.RaidSpawnTimestamp || gym.oldValues.Rsvps != gym.Rsvps) { raidBattleTime := gym.RaidBattleTimestamp.ValueOrZero() raidEndTime := gym.RaidEndTimestamp.ValueOrZero() now := time.Now().Unix() @@ -626,30 +970,16 @@ func createGymWebhooks(oldGym *Gym, gym *Gym, areas []geo.AreaName) { } func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { - oldGym, _ := GetGymRecord(ctx, db, gym.Id) - now := time.Now().Unix() - if oldGym != nil && !hasChangesGym(oldGym, gym) { - if oldGym.Updated > now-900 { - // if a gym is unchanged, and we are within 15 minutes don't make any changes - // however, gym battle toggle a chance to trigger a web hook and make sure we - // save defender changes to internal cache - - if hasInternalChangesGym(oldGym, gym) { - areas := MatchStatsGeofence(gym.Lat, gym.Lon) - createGymWebhooks(oldGym, gym, areas) - - gymCache.Set(gym.Id, *gym, ttlcache.DefaultTTL) - } - + if !gym.IsNewRecord() && !gym.IsDirty() { + if gym.Updated > now-900 { + // if a gym is unchanged, but we did see it again after 15 minutes, then save again return } } - gym.Updated = now - //log.Traceln(cmp.Diff(oldGym, gym)) - if oldGym == nil { + if gym.IsNewRecord() { res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) @@ -710,11 +1040,13 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { _, _ = res, err } - gymCache.Set(gym.Id, *gym, ttlcache.DefaultTTL) + gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) areas := MatchStatsGeofence(gym.Lat, gym.Lon) - createGymWebhooks(oldGym, gym, areas) - createGymFortWebhooks(oldGym, gym) - updateRaidStats(oldGym, gym, areas) + createGymWebhooks(gym, areas) + createGymFortWebhooks(gym) + updateRaidStats(gym, areas) + gym.newRecord = false // After saving, it's no longer a new record + gym.ClearDirty() } func updateGymGetMapFortCache(gym *Gym, skipName bool) { @@ -738,7 +1070,7 @@ func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails } if gym == nil { - gym = &Gym{} + gym = &Gym{newRecord: true} } gym.updateGymFromFortProto(fort) @@ -759,7 +1091,7 @@ func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymIn } if gym == nil { - gym = &Gym{} + gym = &Gym{newRecord: true} } gym.updateGymFromGymInfoOutProto(gymInfo) @@ -825,7 +1157,7 @@ func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { } if gym.Rsvps.Valid { - gym.Rsvps = null.NewString("", false) + gym.SetRsvps(null.NewString("", false)) saveGymRecord(ctx, db, gym) } diff --git a/decoder/main.go b/decoder/main.go index 9173ad5d..8f820679 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -59,14 +59,14 @@ type webhooksSenderInterface interface { var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector var pokestopCache *ttlcache.Cache[string, *Pokestop] -var gymCache *ttlcache.Cache[string, Gym] +var gymCache *ttlcache.Cache[string, *Gym] var stationCache *ttlcache.Cache[string, Station] var tappableCache *ttlcache.Cache[uint64, Tappable] var weatherCache *ttlcache.Cache[int64, Weather] var weatherConsensusCache *ttlcache.Cache[int64, *WeatherConsensusState] var s2CellCache *ttlcache.Cache[uint64, S2Cell] var spawnpointCache *ttlcache.Cache[int64, Spawnpoint] -var pokemonCache []*ttlcache.Cache[uint64, Pokemon] +var pokemonCache []*ttlcache.Cache[uint64, *Pokemon] var incidentCache *ttlcache.Cache[string, Incident] var playerCache *ttlcache.Cache[string, Player] var routeCache *ttlcache.Cache[string, Route] @@ -101,15 +101,15 @@ func (cl *gohbemLogger) Print(message string) { log.Info("Gohbem - ", message) } -func getPokemonCache(key uint64) *ttlcache.Cache[uint64, Pokemon] { +func getPokemonCache(key uint64) *ttlcache.Cache[uint64, *Pokemon] { return pokemonCache[key%uint64(len(pokemonCache))] } -func setPokemonCache(key uint64, value Pokemon, ttl time.Duration) { +func setPokemonCache(key uint64, value *Pokemon, ttl time.Duration) { getPokemonCache(key).Set(key, value, ttl) } -func getPokemonFromCache(key uint64) *ttlcache.Item[uint64, Pokemon] { +func getPokemonFromCache(key uint64) *ttlcache.Item[uint64, *Pokemon] { return getPokemonCache(key).Get(key) } @@ -123,8 +123,8 @@ func initDataCache() { ) go pokestopCache.Start() - gymCache = ttlcache.New[string, Gym]( - ttlcache.WithTTL[string, Gym](60 * time.Minute), + gymCache = ttlcache.New[string, *Gym]( + ttlcache.WithTTL[string, *Gym](60 * time.Minute), ) go gymCache.Start() @@ -160,11 +160,11 @@ func initDataCache() { // pokemon is the most active table. Use an array of caches to increase concurrency for querying ttlcache, which places a global lock for each Get/Set operation // Initialize pokemon cache array: by picking it to be nproc, we should expect ~nproc*(1-1/e) ~ 63% concurrency - pokemonCache = make([]*ttlcache.Cache[uint64, Pokemon], runtime.NumCPU()) + pokemonCache = make([]*ttlcache.Cache[uint64, *Pokemon], runtime.NumCPU()) for i := 0; i < len(pokemonCache); i++ { - pokemonCache[i] = ttlcache.New[uint64, Pokemon]( - ttlcache.WithTTL[uint64, Pokemon](60*time.Minute), - ttlcache.WithDisableTouchOnHit[uint64, Pokemon](), // Pokemon will last 60 mins from when we first see them not last see them + pokemonCache[i] = ttlcache.New[uint64, *Pokemon]( + ttlcache.WithTTL[uint64, *Pokemon](60*time.Minute), + ttlcache.WithDisableTouchOnHit[uint64, *Pokemon](), // Pokemon will last 60 mins from when we first see them not last see them ) go pokemonCache[i].Start() } @@ -353,15 +353,14 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa continue } - isNewGym := gym == nil - if isNewGym { - gym = &Gym{} + if gym == nil { + gym = &Gym{newRecord: true} } gym.updateGymFromFort(fort.Data, fort.Cell) // If this is a new gym, check if it was converted from a pokestop and copy shared fields - if isNewGym { + if gym.IsNewRecord() { pokestop, _ := GetPokestopRecord(ctx, db, fortId) if pokestop != nil { gym.copySharedFieldsFrom(pokestop) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9244cc2b..482a5901 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -76,6 +76,21 @@ type Pokemon struct { IsEvent int8 `db:"is_event" json:"is_event"` internal grpc.PokemonInternal + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + // Note: newRecord tracked via FirstSeenTimestamp == 0 (see isNewRecord method) + + oldValues PokemonOldValues `db:"-" json:"-"` // Old values for webhook comparison and stats +} + +// PokemonOldValues holds old field values for webhook comparison, stats, and R-tree updates +type PokemonOldValues struct { + PokemonId int16 + Weather null.Int + Cp null.Int + SeenType null.String + Lat float64 + Lon float64 } // @@ -131,12 +146,248 @@ type Pokemon struct { //KEY `ix_iv` (`iv`) //) +// IsDirty returns true if any field has been modified +func (pokemon *Pokemon) IsDirty() bool { + return pokemon.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (pokemon *Pokemon) ClearDirty() { + pokemon.dirty = false +} + +// snapshotOldValues saves current values for webhook comparison, stats, and R-tree updates +// Call this after loading from cache/DB but before modifications +func (pokemon *Pokemon) snapshotOldValues() { + pokemon.oldValues = PokemonOldValues{ + PokemonId: pokemon.PokemonId, + Weather: pokemon.Weather, + Cp: pokemon.Cp, + SeenType: pokemon.SeenType, + Lat: pokemon.Lat, + Lon: pokemon.Lon, + } +} + +// --- Set methods with dirty tracking --- + +func (pokemon *Pokemon) SetPokestopId(v null.String) { + if pokemon.PokestopId != v { + pokemon.PokestopId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetSpawnId(v null.Int) { + if pokemon.SpawnId != v { + pokemon.SpawnId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetLat(v float64) { + if !floatAlmostEqual(pokemon.Lat, v, floatTolerance) { + pokemon.Lat = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetLon(v float64) { + if !floatAlmostEqual(pokemon.Lon, v, floatTolerance) { + pokemon.Lon = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetPokemonId(v int16) { + if pokemon.PokemonId != v { + pokemon.PokemonId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetForm(v null.Int) { + if pokemon.Form != v { + pokemon.Form = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCostume(v null.Int) { + if pokemon.Costume != v { + pokemon.Costume = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetGender(v null.Int) { + if pokemon.Gender != v { + pokemon.Gender = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetWeather(v null.Int) { + if pokemon.Weather != v { + pokemon.Weather = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetIsStrong(v null.Bool) { + if pokemon.IsStrong != v { + pokemon.IsStrong = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetExpireTimestamp(v null.Int) { + if pokemon.ExpireTimestamp != v { + pokemon.ExpireTimestamp = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetExpireTimestampVerified(v bool) { + if pokemon.ExpireTimestampVerified != v { + pokemon.ExpireTimestampVerified = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetSeenType(v null.String) { + if pokemon.SeenType != v { + pokemon.SeenType = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetUsername(v null.String) { + if pokemon.Username != v { + pokemon.Username = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCellId(v null.Int) { + if pokemon.CellId != v { + pokemon.CellId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetIsEvent(v int8) { + if pokemon.IsEvent != v { + pokemon.IsEvent = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetShiny(v null.Bool) { + if pokemon.Shiny != v { + pokemon.Shiny = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCp(v null.Int) { + if pokemon.Cp != v { + pokemon.Cp = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetLevel(v null.Int) { + if pokemon.Level != v { + pokemon.Level = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetMove1(v null.Int) { + if pokemon.Move1 != v { + pokemon.Move1 = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetMove2(v null.Int) { + if pokemon.Move2 != v { + pokemon.Move2 = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetHeight(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Height, v, floatTolerance) { + pokemon.Height = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetWeight(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Weight, v, floatTolerance) { + pokemon.Weight = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetSize(v null.Int) { + if pokemon.Size != v { + pokemon.Size = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetIsDitto(v bool) { + if pokemon.IsDitto != v { + pokemon.IsDitto = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { + if pokemon.DisplayPokemonId != v { + pokemon.DisplayPokemonId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetPvp(v null.String) { + if pokemon.Pvp != v { + pokemon.Pvp = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCapture1(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Capture1, v, floatTolerance) { + pokemon.Capture1 = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCapture2(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Capture2, v, floatTolerance) { + pokemon.Capture2 = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCapture3(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Capture3, v, floatTolerance) { + pokemon.Capture3 = v + pokemon.dirty = true + } +} + func getPokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, error) { if db.UsePokemonCache { inMemoryPokemon := getPokemonFromCache(encounterId) if inMemoryPokemon != nil { pokemon := inMemoryPokemon.Value() - return &pokemon, nil + pokemon.snapshotOldValues() // Snapshot for webhook comparison + return pokemon, nil } } if config.Config.PokemonMemoryOnly { @@ -160,8 +411,9 @@ func getPokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) return nil, err } + pokemon.snapshotOldValues() // Snapshot for webhook comparison if db.UsePokemonCache { - setPokemonCache(encounterId, pokemon, ttlcache.DefaultTTL) + setPokemonCache(encounterId, &pokemon, ttlcache.DefaultTTL) } pokemonRtreeUpdatePokemonOnGet(&pokemon) return &pokemon, nil @@ -174,7 +426,7 @@ func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId } pokemon = &Pokemon{Id: encounterId} if db.UsePokemonCache { - setPokemonCache(encounterId, *pokemon, ttlcache.DefaultTTL) + setPokemonCache(encounterId, pokemon, ttlcache.DefaultTTL) } return pokemon, nil } @@ -222,19 +474,12 @@ func hasChangesPokemon(old *Pokemon, new *Pokemon) bool { } func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Pokemon, isEncounter, writeDB, webhook bool, now int64) { - oldPokemon, _ := getPokemonRecord(ctx, db, pokemon.Id) - - if oldPokemon != nil && !hasChangesPokemon(oldPokemon, pokemon) { + if !pokemon.isNewRecord() && !pokemon.IsDirty() { return } - // Blank, non-persisted record are now inserted into the cache to save on DB calls - if oldPokemon != nil && oldPokemon.isNewRecord() { - oldPokemon = nil - } - // uncomment to debug excessive writes - //if oldPokemon != nil && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { + //if !pokemon.isNewRecord() && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { // log.Errorf("Why are we updating this? %s", cmp.Diff(oldPokemon, pokemon, cmp.Options{ // ignoreNearFloats, ignoreNearNullFloats, // cmpopts.IgnoreFields(Pokemon{}, "Username", "Iv", "Pvp"), @@ -246,18 +491,17 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } pokemon.Updated = null.IntFrom(now) - if oldPokemon == nil || oldPokemon.PokemonId != pokemon.PokemonId || oldPokemon.Cp != pokemon.Cp { + if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId || pokemon.oldValues.Cp != pokemon.Cp { pokemon.Changed = now } changePvpField := false var pvpResults map[string][]gohbem.PokemonEntry if ohbem != nil { - // Calculating PVP data - if pokemon.AtkIv.Valid && (oldPokemon == nil || oldPokemon.PokemonId != pokemon.PokemonId || - oldPokemon.Level != pokemon.Level || oldPokemon.Form != pokemon.Form || - oldPokemon.Costume != pokemon.Costume || oldPokemon.Gender != pokemon.Gender || - oldPokemon.Weather != pokemon.Weather) { + // Calculating PVP data - check for changes in pokemon properties that affect PVP rankings + // For new records, always calculate; for existing, check if relevant fields changed + shouldCalculatePvp := pokemon.AtkIv.Valid && (pokemon.isNewRecord() || pokemon.IsDirty()) + if shouldCalculatePvp { pvp, err := ohbem.QueryPvPRank(int(pokemon.PokemonId), int(pokemon.Form.ValueOrZero()), int(pokemon.Costume.ValueOrZero()), @@ -274,19 +518,13 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po pvpResults = pvp } } - if !pokemon.AtkIv.Valid && (oldPokemon == nil || oldPokemon.AtkIv.Valid) { + if !pokemon.AtkIv.Valid && pokemon.isNewRecord() { pokemon.Pvp = null.NewString("", false) changePvpField = true } } - var oldSeenType string - if oldPokemon == nil { - oldSeenType = "n/a" - } else { - oldSeenType = oldPokemon.SeenType.ValueOrZero() - } - log.Debugf("Updating pokemon [%d] from %s->%s", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero()) + log.Debugf("Updating pokemon [%d] to %s", pokemon.Id, pokemon.SeenType.ValueOrZero()) //log.Println(cmp.Diff(oldPokemon, pokemon)) if writeDB && !config.Config.PokemonMemoryOnly { @@ -306,7 +544,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po log.Errorf("[POKEMON] Failed to marshal internal data for %d, data may be lost: %s", pokemon.Id, err) } } - if oldPokemon == nil { + if pokemon.isNewRecord() { pvpField, pvpValue := "", "" if changePvpField { pvpField, pvpValue = "pvp, ", ":pvp, " @@ -388,31 +626,32 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } // Update pokemon rtree - if oldPokemon == nil { + if pokemon.isNewRecord() { + addPokemonToTree(pokemon) + } else if pokemon.Lat != pokemon.oldValues.Lat || pokemon.Lon != pokemon.oldValues.Lon { + // Position changed - update R-tree by removing from old position and adding to new + removePokemonFromTree(pokemon.Id, pokemon.oldValues.Lat, pokemon.oldValues.Lon) addPokemonToTree(pokemon) - } else { - if pokemon.Lat != oldPokemon.Lat || pokemon.Lon != oldPokemon.Lon { - removePokemonFromTree(oldPokemon) - addPokemonToTree(pokemon) - } } updatePokemonLookup(pokemon, changePvpField, pvpResults) areas := MatchStatsGeofence(pokemon.Lat, pokemon.Lon) if webhook { - createPokemonWebhooks(ctx, db, oldPokemon, pokemon, areas) + createPokemonWebhooks(ctx, db, pokemon, areas) } - updatePokemonStats(oldPokemon, pokemon, areas, now) + updatePokemonStats(pokemon, areas, now) + + pokemon.ClearDirty() pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache if db.UsePokemonCache { - setPokemonCache(pokemon.Id, *pokemon, pokemon.remainingDuration(now)) + setPokemonCache(pokemon.Id, pokemon, pokemon.remainingDuration(now)) } } -func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, new *Pokemon, areas []geo.AreaName) { +func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemon, areas []geo.AreaName) { //nullString := func (v null.Int) interface{} { // if !v.Valid { // return "null" @@ -420,29 +659,29 @@ func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, n // return v.ValueOrZero() //} - if old == nil || - old.PokemonId != new.PokemonId || - old.Weather != new.Weather || - old.Cp != new.Cp { + if pokemon.isNewRecord() || + pokemon.oldValues.PokemonId != pokemon.PokemonId || + pokemon.oldValues.Weather != pokemon.Weather || + pokemon.oldValues.Cp != pokemon.Cp { pokemonHook := map[string]interface{}{ "spawnpoint_id": func() string { - if !new.SpawnId.Valid { + if !pokemon.SpawnId.Valid { return "None" } - return strconv.FormatInt(new.SpawnId.ValueOrZero(), 16) + return strconv.FormatInt(pokemon.SpawnId.ValueOrZero(), 16) }(), "pokestop_id": func() string { - if !new.PokestopId.Valid { + if !pokemon.PokestopId.Valid { return "None" } else { - return new.PokestopId.ValueOrZero() + return pokemon.PokestopId.ValueOrZero() } }(), "pokestop_name": func() *string { - if !new.PokestopId.Valid { + if !pokemon.PokestopId.Valid { return nil } else { - pokestop, _ := GetPokestopRecord(ctx, db, new.PokestopId.String) + pokestop, _ := GetPokestopRecord(ctx, db, pokemon.PokestopId.String) name := "Unknown" if pokestop != nil { name = pokestop.Name.ValueOrZero() @@ -450,46 +689,46 @@ func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, n return &name } }(), - "encounter_id": strconv.FormatUint(new.Id, 10), - "pokemon_id": new.PokemonId, - "latitude": new.Lat, - "longitude": new.Lon, - "disappear_time": new.ExpireTimestamp.ValueOrZero(), - "disappear_time_verified": new.ExpireTimestampVerified, - "first_seen": new.FirstSeenTimestamp, - "last_modified_time": new.Updated, - "gender": new.Gender, - "cp": new.Cp, - "form": new.Form, - "costume": new.Costume, - "individual_attack": new.AtkIv, - "individual_defense": new.DefIv, - "individual_stamina": new.StaIv, - "pokemon_level": new.Level, - "move_1": new.Move1, - "move_2": new.Move2, - "weight": new.Weight, - "size": new.Size, - "height": new.Height, - "weather": new.Weather, - "capture_1": new.Capture1.ValueOrZero(), - "capture_2": new.Capture2.ValueOrZero(), - "capture_3": new.Capture3.ValueOrZero(), - "shiny": new.Shiny, - "username": new.Username, - "display_pokemon_id": new.DisplayPokemonId, - "is_event": new.IsEvent, - "seen_type": new.SeenType, + "encounter_id": strconv.FormatUint(pokemon.Id, 10), + "pokemon_id": pokemon.PokemonId, + "latitude": pokemon.Lat, + "longitude": pokemon.Lon, + "disappear_time": pokemon.ExpireTimestamp.ValueOrZero(), + "disappear_time_verified": pokemon.ExpireTimestampVerified, + "first_seen": pokemon.FirstSeenTimestamp, + "last_modified_time": pokemon.Updated, + "gender": pokemon.Gender, + "cp": pokemon.Cp, + "form": pokemon.Form, + "costume": pokemon.Costume, + "individual_attack": pokemon.AtkIv, + "individual_defense": pokemon.DefIv, + "individual_stamina": pokemon.StaIv, + "pokemon_level": pokemon.Level, + "move_1": pokemon.Move1, + "move_2": pokemon.Move2, + "weight": pokemon.Weight, + "size": pokemon.Size, + "height": pokemon.Height, + "weather": pokemon.Weather, + "capture_1": pokemon.Capture1.ValueOrZero(), + "capture_2": pokemon.Capture2.ValueOrZero(), + "capture_3": pokemon.Capture3.ValueOrZero(), + "shiny": pokemon.Shiny, + "username": pokemon.Username, + "display_pokemon_id": pokemon.DisplayPokemonId, + "is_event": pokemon.IsEvent, + "seen_type": pokemon.SeenType, "pvp": func() interface{} { - if !new.Pvp.Valid { + if !pokemon.Pvp.Valid { return nil } else { - return json.RawMessage(new.Pvp.ValueOrZero()) + return json.RawMessage(pokemon.Pvp.ValueOrZero()) } }(), } - if new.AtkIv.Valid && new.DefIv.Valid && new.StaIv.Valid { + if pokemon.AtkIv.Valid && pokemon.DefIv.Valid && pokemon.StaIv.Valid { webhooksSender.AddMessage(webhooks.PokemonIV, pokemonHook, areas) } else { webhooksSender.AddMessage(webhooks.PokemonNoIV, pokemonHook, areas) @@ -557,14 +796,14 @@ func (pokemon *Pokemon) addWildPokemon(ctx context.Context, db db.DbDetails, wil if wildPokemon.EncounterId != pokemon.Id { panic("Unmatched EncounterId") } - pokemon.Lat = wildPokemon.Latitude - pokemon.Lon = wildPokemon.Longitude + pokemon.SetLat(wildPokemon.Latitude) + pokemon.SetLon(wildPokemon.Longitude) spawnId, err := strconv.ParseInt(wildPokemon.SpawnPointId, 16, 64) if err != nil { panic(err) } - pokemon.SpawnId = null.IntFrom(spawnId) + pokemon.SetSpawnId(null.IntFrom(spawnId)) pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, trustworthyTimestamp) pokemon.setPokemonDisplay(int16(wildPokemon.Pokemon.PokemonId), wildPokemon.Pokemon.PokemonDisplay) @@ -587,15 +826,15 @@ func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto } func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) switch pokemon.SeenType.ValueOrZero() { case "", SeenType_Cell, SeenType_NearbyStop: - pokemon.SeenType = null.StringFrom(SeenType_Wild) + pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) } pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs, true) pokemon.recomputeCpIfNeeded(ctx, db, weather) - pokemon.Username = null.StringFrom(username) - pokemon.CellId = null.IntFrom(cellId) + pokemon.SetUsername(null.StringFrom(username)) + pokemon.SetCellId(null.IntFrom(cellId)) } func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapPokemon *pogo.MapPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { @@ -605,7 +844,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP return } - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) pokemon.Id = mapPokemon.EncounterId @@ -616,10 +855,10 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP // Unrecognised pokestop return } - pokemon.PokestopId = null.StringFrom(pokestop.Id) - pokemon.Lat = pokestop.Lat - pokemon.Lon = pokestop.Lon - pokemon.SeenType = null.StringFrom(SeenType_LureWild) + pokemon.SetPokestopId(null.StringFrom(pokestop.Id)) + pokemon.SetLat(pokestop.Lat) + pokemon.SetLon(pokestop.Lon) + pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) if mapPokemon.PokemonDisplay != nil { pokemon.setPokemonDisplay(int16(mapPokemon.PokedexTypeId), mapPokemon.PokemonDisplay) @@ -630,34 +869,38 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP log.Warnf("[POKEMON] MapPokemonProto missing PokemonDisplay for %d", pokemon.Id) } if !pokemon.Username.Valid { - pokemon.Username = null.StringFrom(username) + pokemon.SetUsername(null.StringFrom(username)) } if mapPokemon.ExpirationTimeMs > 0 && !pokemon.ExpireTimestampVerified { - pokemon.ExpireTimestamp = null.IntFrom(mapPokemon.ExpirationTimeMs / 1000) - pokemon.ExpireTimestampVerified = true + pokemon.SetExpireTimestamp(null.IntFrom(mapPokemon.ExpirationTimeMs / 1000)) + pokemon.SetExpireTimestampVerified(true) // if we have cached an encounter for this pokemon, update the TTL. encounterCache.UpdateTTL(pokemon.Id, pokemon.remainingDuration(timestampMs/1000)) } else { - pokemon.ExpireTimestampVerified = false + pokemon.SetExpireTimestampVerified(false) } - pokemon.CellId = null.IntFrom(cellId) + pokemon.SetCellId(null.IntFrom(cellId)) } func (pokemon *Pokemon) calculateIv(a int64, d int64, s int64) { - pokemon.AtkIv = null.IntFrom(a) - pokemon.DefIv = null.IntFrom(d) - pokemon.StaIv = null.IntFrom(s) - pokemon.Iv = null.FloatFrom(float64(a+d+s) / .45) + if pokemon.AtkIv.ValueOrZero() != a || pokemon.DefIv.ValueOrZero() != d || pokemon.StaIv.ValueOrZero() != s || + !pokemon.AtkIv.Valid || !pokemon.DefIv.Valid || !pokemon.StaIv.Valid { + pokemon.AtkIv = null.IntFrom(a) + pokemon.DefIv = null.IntFrom(d) + pokemon.StaIv = null.IntFrom(s) + pokemon.Iv = null.FloatFrom(float64(a+d+s) / .45) + pokemon.dirty = true + } } func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, nearbyPokemon *pogo.NearbyPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) pokestopId := nearbyPokemon.FortId pokemon.setPokemonDisplay(int16(nearbyPokemon.PokedexNumber), nearbyPokemon.PokemonDisplay) pokemon.recomputeCpIfNeeded(ctx, db, weather) - pokemon.Username = null.StringFrom(username) + pokemon.SetUsername(null.StringFrom(username)) var lat, lon float64 overrideLatLon := pokemon.isNewRecord() @@ -675,8 +918,8 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n // Unrecognised pokestop, rollback changes overrideLatLon = pokemon.isNewRecord() } else { - pokemon.SeenType = null.StringFrom(SeenType_NearbyStop) - pokemon.PokestopId = null.StringFrom(pokestopId) + pokemon.SetSeenType(null.StringFrom(SeenType_NearbyStop)) + pokemon.SetPokestopId(null.StringFrom(pokestopId)) lat, lon = pokestop.Lat, pokestop.Lon useCellLatLon = false } @@ -692,17 +935,18 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n lat = s2cell.CapBound().RectBound().Center().Lat.Degrees() lon = s2cell.CapBound().RectBound().Center().Lng.Degrees() - pokemon.SeenType = null.StringFrom(SeenType_Cell) + pokemon.SetSeenType(null.StringFrom(SeenType_Cell)) } if overrideLatLon { - pokemon.Lat, pokemon.Lon = lat, lon + pokemon.SetLat(lat) + pokemon.SetLon(lon) } else { midpoint := s2.LatLngFromPoint(s2.Point{s2.PointFromLatLng(s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon)). Add(s2.PointFromLatLng(s2.LatLngFromDegrees(lat, lon)).Vector)}) - pokemon.Lat = midpoint.Lat.Degrees() - pokemon.Lon = midpoint.Lng.Degrees() + pokemon.SetLat(midpoint.Lat.Degrees()) + pokemon.SetLon(midpoint.Lng.Degrees()) } - pokemon.CellId = null.IntFrom(cellId) + pokemon.SetCellId(null.IntFrom(cellId)) pokemon.setUnknownTimestamp(timestampMs / 1000) } @@ -745,8 +989,8 @@ func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db if despawnOffset < 0 { despawnOffset += 3600 } - pokemon.ExpireTimestamp = null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset)) - pokemon.ExpireTimestampVerified = true + pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) + pokemon.SetExpireTimestampVerified(true) } else { pokemon.setUnknownTimestamp(timestampMs / 1000) } @@ -754,10 +998,10 @@ func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db func (pokemon *Pokemon) setUnknownTimestamp(now int64) { if !pokemon.ExpireTimestamp.Valid { - pokemon.ExpireTimestamp = null.IntFrom(now + 20*60) // should be configurable, add on 20min + pokemon.SetExpireTimestamp(null.IntFrom(now + 20*60)) // should be configurable, add on 20min } else { if pokemon.ExpireTimestamp.Int64 < now { - pokemon.ExpireTimestamp = null.IntFrom(now + 10*60) // should be configurable, add on 10min + pokemon.SetExpireTimestamp(null.IntFrom(now + 10*60)) // should be configurable, add on 10min } } } @@ -772,18 +1016,18 @@ func checkScans(old *grpc.PokemonScan, new *grpc.PokemonScan) error { func (pokemon *Pokemon) setDittoAttributes(mode string, isDitto bool, old, new *grpc.PokemonScan) { if isDitto { log.Debugf("[POKEMON] %d: %s Ditto found %s -> %s", pokemon.Id, mode, old, new) - pokemon.IsDitto = true - pokemon.DisplayPokemonId = null.IntFrom(int64(pokemon.PokemonId)) - pokemon.PokemonId = int16(pogo.HoloPokemonId_DITTO) + pokemon.SetIsDitto(true) + pokemon.SetDisplayPokemonId(null.IntFrom(int64(pokemon.PokemonId))) + pokemon.SetPokemonId(int16(pogo.HoloPokemonId_DITTO)) } else { log.Debugf("[POKEMON] %d: %s not Ditto found %s -> %s", pokemon.Id, mode, old, new) } } func (pokemon *Pokemon) resetDittoAttributes(mode string, old, aux, new *grpc.PokemonScan) (*grpc.PokemonScan, error) { log.Debugf("[POKEMON] %d: %s Ditto was reset %s (%s) -> %s", pokemon.Id, mode, old, aux, new) - pokemon.IsDitto = false - pokemon.DisplayPokemonId = null.NewInt(0, false) - pokemon.PokemonId = int16(pokemon.DisplayPokemonId.Int64) + pokemon.SetIsDitto(false) + pokemon.SetDisplayPokemonId(null.NewInt(0, false)) + pokemon.SetPokemonId(int16(pokemon.DisplayPokemonId.Int64)) return new, checkScans(old, new) } @@ -1085,6 +1329,9 @@ func (pokemon *Pokemon) detectDitto(scan *grpc.PokemonScan) (*grpc.PokemonScan, } func (pokemon *Pokemon) clearIv(cp bool) { + if pokemon.AtkIv.Valid || pokemon.DefIv.Valid || pokemon.StaIv.Valid || pokemon.Iv.Valid { + pokemon.dirty = true + } pokemon.AtkIv = null.NewInt(0, false) pokemon.DefIv = null.NewInt(0, false) pokemon.StaIv = null.NewInt(0, false) @@ -1092,25 +1339,25 @@ func (pokemon *Pokemon) clearIv(cp bool) { if cp { switch pokemon.SeenType.ValueOrZero() { case SeenType_LureEncounter: - pokemon.SeenType = null.StringFrom(SeenType_LureWild) + pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) case SeenType_Encounter: - pokemon.SeenType = null.StringFrom(SeenType_Wild) + pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) } - pokemon.Cp = null.NewInt(0, false) - pokemon.Pvp = null.NewString("", false) + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) } } // caller should setPokemonDisplay prior to calling this func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails, proto *pogo.PokemonProto, username string) { - pokemon.Username = null.StringFrom(username) - pokemon.Shiny = null.BoolFrom(proto.PokemonDisplay.Shiny) - pokemon.Cp = null.IntFrom(int64(proto.Cp)) - pokemon.Move1 = null.IntFrom(int64(proto.Move1)) - pokemon.Move2 = null.IntFrom(int64(proto.Move2)) - pokemon.Height = null.FloatFrom(float64(proto.HeightM)) - pokemon.Size = null.IntFrom(int64(proto.Size)) - pokemon.Weight = null.FloatFrom(float64(proto.WeightKg)) + pokemon.SetUsername(null.StringFrom(username)) + pokemon.SetShiny(null.BoolFrom(proto.PokemonDisplay.Shiny)) + pokemon.SetCp(null.IntFrom(int64(proto.Cp))) + pokemon.SetMove1(null.IntFrom(int64(proto.Move1))) + pokemon.SetMove2(null.IntFrom(int64(proto.Move2))) + pokemon.SetHeight(null.FloatFrom(float64(proto.HeightM))) + pokemon.SetSize(null.IntFrom(int64(proto.Size))) + pokemon.SetWeight(null.FloatFrom(float64(proto.WeightKg))) scan := grpc.PokemonScan{ Weather: int32(pokemon.Weather.Int64), @@ -1146,10 +1393,10 @@ func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails log.Errorf("[POKEMON] Unexpected %d: %s", pokemon.Id, err) } if caughtIv == nil { // this can only happen for a 0P Ditto - pokemon.Level = null.IntFrom(int64(scan.Level - 5)) + pokemon.SetLevel(null.IntFrom(int64(scan.Level - 5))) pokemon.clearIv(false) } else { - pokemon.Level = null.IntFrom(int64(caughtIv.Level)) + pokemon.SetLevel(null.IntFrom(int64(caughtIv.Level))) pokemon.calculateIv(int64(caughtIv.Attack), int64(caughtIv.Defense), int64(caughtIv.Stamina)) } if err == nil { @@ -1175,18 +1422,18 @@ func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails } func (pokemon *Pokemon) updatePokemonFromEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.EncounterOutProto, username string, timestampMs int64) { - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) pokemon.addWildPokemon(ctx, db, encounterData.Pokemon, timestampMs, false) // tappable encounter can also be available in seen as normal encounter once tapped if pokemon.isSeenFromTappable() { - pokemon.SeenType = null.StringFrom(SeenType_Encounter) + pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) } pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon.Pokemon, username) if pokemon.CellId.Valid == false { centerCoord := s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon) cellID := s2.CellIDFromLatLng(centerCoord).Parent(15) - pokemon.CellId = null.IntFrom(int64(cellID)) + pokemon.SetCellId(null.IntFrom(int64(cellID))) } } @@ -1195,37 +1442,37 @@ func (pokemon *Pokemon) isSeenFromTappable() bool { } func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.DiskEncounterOutProto, username string) { - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) - pokemon.SeenType = null.StringFrom(SeenType_LureEncounter) + pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) } func (pokemon *Pokemon) updatePokemonFromTappableEncounterProto(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounterData *pogo.TappableEncounterProto, username string, timestampMs int64) { - pokemon.IsEvent = 0 - pokemon.Lat = request.LocationHintLat - pokemon.Lon = request.LocationHintLng + pokemon.SetIsEvent(0) + pokemon.SetLat(request.LocationHintLat) + pokemon.SetLon(request.LocationHintLng) if spawnpointId := request.GetLocation().GetSpawnpointId(); spawnpointId != "" { - pokemon.SeenType = null.StringFrom(SeenType_TappableEncounter) + pokemon.SetSeenType(null.StringFrom(SeenType_TappableEncounter)) spawnId, err := strconv.ParseInt(spawnpointId, 16, 64) if err != nil { panic(err) } - pokemon.SpawnId = null.IntFrom(spawnId) + pokemon.SetSpawnId(null.IntFrom(spawnId)) pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, false) } else if fortId := request.GetLocation().GetFortId(); fortId != "" { - pokemon.SeenType = null.StringFrom(SeenType_TappableLureEncounter) + pokemon.SetSeenType(null.StringFrom(SeenType_TappableLureEncounter)) - pokemon.PokestopId = null.StringFrom(fortId) + pokemon.SetPokestopId(null.StringFrom(fortId)) // we don't know any despawn times from lured/fort tappables - pokemon.ExpireTimestamp = null.IntFrom(int64(timestampMs)/1000 + int64(120)) - pokemon.ExpireTimestampVerified = false + pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) + pokemon.SetExpireTimestampVerified(false) } if !pokemon.Username.Valid { - pokemon.Username = null.StringFrom(username) + pokemon.SetUsername(null.StringFrom(username)) } pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) @@ -1248,29 +1495,29 @@ func (pokemon *Pokemon) setPokemonDisplay(pokemonId int16, display *pogo.Pokemon pokemon.Form.ValueOrZero(), pokemon.Costume.ValueOrZero(), pokemon.Gender.ValueOrZero(), pokemon.IsStrong.ValueOrZero(), pokemonId, display.Form, display.Costume, display.Gender, display.IsStrongPokemon) - pokemon.Weight = null.NewFloat(0, false) - pokemon.Height = null.NewFloat(0, false) - pokemon.Size = null.NewInt(0, false) - pokemon.Move1 = null.NewInt(0, false) - pokemon.Move2 = null.NewInt(0, false) - pokemon.Cp = null.NewInt(0, false) - pokemon.Shiny = null.NewBool(false, false) - pokemon.IsDitto = false - pokemon.DisplayPokemonId = null.NewInt(0, false) - pokemon.Pvp = null.NewString("", false) + pokemon.SetWeight(null.NewFloat(0, false)) + pokemon.SetHeight(null.NewFloat(0, false)) + pokemon.SetSize(null.NewInt(0, false)) + pokemon.SetMove1(null.NewInt(0, false)) + pokemon.SetMove2(null.NewInt(0, false)) + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetShiny(null.NewBool(false, false)) + pokemon.SetIsDitto(false) + pokemon.SetDisplayPokemonId(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) } } if pokemon.isNewRecord() || !pokemon.IsDitto { - pokemon.PokemonId = pokemonId + pokemon.SetPokemonId(pokemonId) } - pokemon.Gender = null.IntFrom(int64(display.Gender)) - pokemon.Form = null.IntFrom(int64(display.Form)) - pokemon.Costume = null.IntFrom(int64(display.Costume)) + pokemon.SetGender(null.IntFrom(int64(display.Gender))) + pokemon.SetForm(null.IntFrom(int64(display.Form))) + pokemon.SetCostume(null.IntFrom(int64(display.Costume))) if !pokemon.isNewRecord() { pokemon.repopulateIv(int64(display.WeatherBoostedCondition), display.IsStrongPokemon) } - pokemon.Weather = null.IntFrom(int64(display.WeatherBoostedCondition)) - pokemon.IsStrong = null.BoolFrom(display.IsStrongPokemon) + pokemon.SetWeather(null.IntFrom(int64(display.WeatherBoostedCondition))) + pokemon.SetIsStrong(null.BoolFrom(display.IsStrongPokemon)) } func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { @@ -1296,7 +1543,7 @@ func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { matchingScan, isBoostedMatches := pokemon.locateScan(isStrong, isBoosted) var oldAtk, oldDef, oldSta int64 if matchingScan == nil { - pokemon.Level = null.NewInt(0, false) + pokemon.SetLevel(null.NewInt(0, false)) pokemon.clearIv(true) } else { oldLevel := pokemon.Level.ValueOrZero() @@ -1309,29 +1556,30 @@ func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { oldDef = -1 oldSta = -1 } - pokemon.Level = null.IntFrom(int64(matchingScan.Level)) + newLevel := int64(matchingScan.Level) if isBoostedMatches || isStrong { // strong Pokemon IV is unaffected by weather pokemon.calculateIv(int64(matchingScan.Attack), int64(matchingScan.Defense), int64(matchingScan.Stamina)) switch pokemon.SeenType.ValueOrZero() { case SeenType_LureWild: - pokemon.SeenType = null.StringFrom(SeenType_LureEncounter) + pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) case SeenType_Wild: - pokemon.SeenType = null.StringFrom(SeenType_Encounter) + pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) } } else { pokemon.clearIv(true) } if !isBoostedMatches { if isBoosted { - pokemon.Level.Int64 += 5 + newLevel += 5 } else { - pokemon.Level.Int64 -= 5 + newLevel -= 5 } } - if pokemon.Level.Int64 != oldLevel || pokemon.AtkIv.Valid && + pokemon.SetLevel(null.IntFrom(newLevel)) + if newLevel != oldLevel || pokemon.AtkIv.Valid && (pokemon.AtkIv.Int64 != oldAtk || pokemon.DefIv.Int64 != oldDef || pokemon.StaIv.Int64 != oldSta) { - pokemon.Cp = null.NewInt(0, false) - pokemon.Pvp = null.NewString("", false) + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) } } } @@ -1387,7 +1635,7 @@ func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails float64(pokemon.Level.Int64)) } if err == nil { - pokemon.Cp = null.IntFrom(int64(cp)) + pokemon.SetCp(null.IntFrom(int64(cp))) } else { log.Warnf("Pokemon %d %d CP unset due to error %s", pokemon.Id, displayPokemon, err) } diff --git a/decoder/pokemonRtree.go b/decoder/pokemonRtree.go index 89e1edde..02037386 100644 --- a/decoder/pokemonRtree.go +++ b/decoder/pokemonRtree.go @@ -52,9 +52,9 @@ func initPokemonRtree() { // Set up OnEviction callbacks for each cache in the array for i := range pokemonCache { - pokemonCache[i].OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, Pokemon]) { - r := v.Value() - removePokemonFromTree(&r) + pokemonCache[i].OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, *Pokemon]) { + pokemon := v.Value() + removePokemonFromTree(pokemon.Id, pokemon.Lat, pokemon.Lon) // Rely on the pokemon pvp lookup caches to remove themselves rather than trying to synchronise }) } @@ -160,16 +160,15 @@ func addPokemonToTree(pokemon *Pokemon) { pokemonTreeMutex.Unlock() } -func removePokemonFromTree(pokemon *Pokemon) { - pokemonId := pokemon.Id +func removePokemonFromTree(pokemonId uint64, lat, lon float64) { pokemonTreeMutex.Lock() beforeLen := pokemonTree.Len() - pokemonTree.Delete([2]float64{pokemon.Lon, pokemon.Lat}, [2]float64{pokemon.Lon, pokemon.Lat}, pokemonId) + pokemonTree.Delete([2]float64{lon, lat}, [2]float64{lon, lat}, pokemonId) afterLen := pokemonTree.Len() pokemonTreeMutex.Unlock() pokemonLookupCache.Delete(pokemonId) if beforeLen != afterLen+1 { - log.Infof("PokemonRtree - UNEXPECTED removing %d, lat %f lon %f size %d->%d Map Len %d", pokemonId, pokemon.Lat, pokemon.Lon, beforeLen, afterLen, pokemonLookupCache.Size()) + log.Infof("PokemonRtree - UNEXPECTED removing %d, lat %f lon %f size %d->%d Map Len %d", pokemonId, lat, lon, beforeLen, afterLen, pokemonLookupCache.Size()) } } diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 7be500ae..face7098 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -71,57 +71,60 @@ type Pokestop struct { dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record - // Old values for webhook comparison (populated when loading from cache/DB) - oldQuestType null.Int `db:"-" json:"-"` - oldAlternativeQuestType null.Int `db:"-" json:"-"` - oldLureExpireTimestamp null.Int `db:"-" json:"-"` - oldLureId int16 `db:"-" json:"-"` - oldPowerUpEndTimestamp null.Int `db:"-" json:"-"` - oldName null.String `db:"-" json:"-"` - oldUrl null.String `db:"-" json:"-"` - oldDescription null.String `db:"-" json:"-"` - oldLat float64 `db:"-" json:"-"` - oldLon float64 `db:"-" json:"-"` - - //`id` varchar(35) NOT NULL, - //`lat` double(18,14) NOT NULL, - //`lon` double(18,14) NOT NULL, - //`name` varchar(128) DEFAULT NULL, - //`url` varchar(200) DEFAULT NULL, - //`lure_expire_timestamp` int unsigned DEFAULT NULL, - //`last_modified_timestamp` int unsigned DEFAULT NULL, - //`updated` int unsigned NOT NULL, - //`enabled` tinyint unsigned DEFAULT NULL, - //`quest_type` int unsigned DEFAULT NULL, - //`quest_timestamp` int unsigned DEFAULT NULL, - //`quest_target` smallint unsigned DEFAULT NULL, - //`quest_conditions` text, - //`quest_rewards` text, - //`quest_template` varchar(100) DEFAULT NULL, - //`quest_title` varchar(100) DEFAULT NULL, - //`quest_reward_type` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].type'),_utf8mb4'$[0]')) VIRTUAL, - //`quest_item_id` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.item_id'),_utf8mb4'$[0]')) VIRTUAL, - //`quest_reward_amount` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.amount'),_utf8mb4'$[0]')) VIRTUAL, - //`cell_id` bigint unsigned DEFAULT NULL, - //`deleted` tinyint unsigned NOT NULL DEFAULT '0', - //`lure_id` smallint DEFAULT '0', - //`first_seen_timestamp` int unsigned NOT NULL, - //`sponsor_id` smallint unsigned DEFAULT NULL, - //`partner_id` varchar(35) DEFAULT NULL, - //`quest_pokemon_id` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.pokemon_id'),_utf8mb4'$[0]')) VIRTUAL, - //`ar_scan_eligible` tinyint unsigned DEFAULT NULL, - //`power_up_level` smallint unsigned DEFAULT NULL, - //`power_up_points` int unsigned DEFAULT NULL, - //`power_up_end_timestamp` int unsigned DEFAULT NULL, - //`alternative_quest_type` int unsigned DEFAULT NULL, - //`alternative_quest_timestamp` int unsigned DEFAULT NULL, - //`alternative_quest_target` smallint unsigned DEFAULT NULL, - //`alternative_quest_conditions` text, - //`alternative_quest_rewards` text, - //`alternative_quest_template` varchar(100) DEFAULT NULL, - //`alternative_quest_title` varchar(100) DEFAULT NULL, - -} + oldValues PokestopOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// PokestopOldValues holds old field values for webhook comparison (populated when loading from cache/DB) +type PokestopOldValues struct { + QuestType null.Int + AlternativeQuestType null.Int + LureExpireTimestamp null.Int + LureId int16 + PowerUpEndTimestamp null.Int + Name null.String + Url null.String + Description null.String + Lat float64 + Lon float64 +} + +//`id` varchar(35) NOT NULL, +//`lat` double(18,14) NOT NULL, +//`lon` double(18,14) NOT NULL, +//`name` varchar(128) DEFAULT NULL, +//`url` varchar(200) DEFAULT NULL, +//`lure_expire_timestamp` int unsigned DEFAULT NULL, +//`last_modified_timestamp` int unsigned DEFAULT NULL, +//`updated` int unsigned NOT NULL, +//`enabled` tinyint unsigned DEFAULT NULL, +//`quest_type` int unsigned DEFAULT NULL, +//`quest_timestamp` int unsigned DEFAULT NULL, +//`quest_target` smallint unsigned DEFAULT NULL, +//`quest_conditions` text, +//`quest_rewards` text, +//`quest_template` varchar(100) DEFAULT NULL, +//`quest_title` varchar(100) DEFAULT NULL, +//`quest_reward_type` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].type'),_utf8mb4'$[0]')) VIRTUAL, +//`quest_item_id` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.item_id'),_utf8mb4'$[0]')) VIRTUAL, +//`quest_reward_amount` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.amount'),_utf8mb4'$[0]')) VIRTUAL, +//`cell_id` bigint unsigned DEFAULT NULL, +//`deleted` tinyint unsigned NOT NULL DEFAULT '0', +//`lure_id` smallint DEFAULT '0', +//`first_seen_timestamp` int unsigned NOT NULL, +//`sponsor_id` smallint unsigned DEFAULT NULL, +//`partner_id` varchar(35) DEFAULT NULL, +//`quest_pokemon_id` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.pokemon_id'),_utf8mb4'$[0]')) VIRTUAL, +//`ar_scan_eligible` tinyint unsigned DEFAULT NULL, +//`power_up_level` smallint unsigned DEFAULT NULL, +//`power_up_points` int unsigned DEFAULT NULL, +//`power_up_end_timestamp` int unsigned DEFAULT NULL, +//`alternative_quest_type` int unsigned DEFAULT NULL, +//`alternative_quest_timestamp` int unsigned DEFAULT NULL, +//`alternative_quest_target` smallint unsigned DEFAULT NULL, +//`alternative_quest_conditions` text, +//`alternative_quest_rewards` text, +//`alternative_quest_template` varchar(100) DEFAULT NULL, +//`alternative_quest_title` varchar(100) DEFAULT NULL, // IsDirty returns true if any field has been modified func (p *Pokestop) IsDirty() bool { @@ -141,16 +144,18 @@ func (p *Pokestop) IsNewRecord() bool { // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (p *Pokestop) snapshotOldValues() { - p.oldQuestType = p.QuestType - p.oldAlternativeQuestType = p.AlternativeQuestType - p.oldLureExpireTimestamp = p.LureExpireTimestamp - p.oldLureId = p.LureId - p.oldPowerUpEndTimestamp = p.PowerUpEndTimestamp - p.oldName = p.Name - p.oldUrl = p.Url - p.oldDescription = p.Description - p.oldLat = p.Lat - p.oldLon = p.Lon + p.oldValues = PokestopOldValues{ + QuestType: p.QuestType, + AlternativeQuestType: p.AlternativeQuestType, + LureExpireTimestamp: p.LureExpireTimestamp, + LureId: p.LureId, + PowerUpEndTimestamp: p.PowerUpEndTimestamp, + Name: p.Name, + Url: p.Url, + Description: p.Description, + Lat: p.Lat, + Lon: p.Lon, + } } // --- Set methods with dirty tracking --- @@ -958,10 +963,10 @@ func createPokestopFortWebhooks(stop *Pokestop) { oldFort := &FortWebhook{ Type: POKESTOP.String(), Id: stop.Id, - Name: stop.oldName.Ptr(), - ImageUrl: stop.oldUrl.Ptr(), - Description: stop.oldDescription.Ptr(), - Location: Location{Latitude: stop.oldLat, Longitude: stop.oldLon}, + Name: stop.oldValues.Name.Ptr(), + ImageUrl: stop.oldValues.Url.Ptr(), + Description: stop.oldValues.Description.Ptr(), + Location: Location{Latitude: stop.oldValues.Lat, Longitude: stop.oldValues.Lon}, } CreateFortWebHooks(oldFort, fort, EDIT) } @@ -971,7 +976,7 @@ func createPokestopWebhooks(stop *Pokestop) { areas := MatchStatsGeofence(stop.Lat, stop.Lon) - if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldAlternativeQuestType) { + if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldValues.AlternativeQuestType) { questHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -997,7 +1002,7 @@ func createPokestopWebhooks(stop *Pokestop) { webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } - if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldQuestType) { + if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldValues.QuestType) { questHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -1022,7 +1027,7 @@ func createPokestopWebhooks(stop *Pokestop) { } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } - if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldLureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldPowerUpEndTimestamp)) { + if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldValues.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldValues.PowerUpEndTimestamp)) { pokestopHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, diff --git a/decoder/stats.go b/decoder/stats.go index 86ea7956..ee7c1d36 100644 --- a/decoder/stats.go +++ b/decoder/stats.go @@ -241,7 +241,7 @@ func updateEncounterStats(pokemon *Pokemon) { } } -func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now int64) { +func updatePokemonStats(pokemon *Pokemon, areas []geo.AreaName, now int64) { if len(areas) == 0 { areas = []geo.AreaName{ { @@ -273,15 +273,12 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in populateEncounterCacheVal := func() { if encounterCacheVal == nil { - encounterCacheVal = encounterCache.GetOrCreate(new.Id) + encounterCacheVal = encounterCache.GetOrCreate(pokemon.Id) } } - currentSeenType := new.SeenType.ValueOrZero() - oldSeenType := "" - if old != nil { - oldSeenType = old.SeenType.ValueOrZero() - } + currentSeenType := pokemon.SeenType.ValueOrZero() + oldSeenType := pokemon.oldValues.SeenType.ValueOrZero() if currentSeenType != oldSeenType { if oldSeenType == "" || oldSeenType == SeenType_NearbyStop || oldSeenType == SeenType_Cell { @@ -291,7 +288,7 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in // transition to wild for the first time.. populateEncounterCacheVal() encounterCacheVal.FirstEncounter = 0 - encounterCacheVal.FirstWild = new.Updated.ValueOrZero() + encounterCacheVal.FirstWild = pokemon.Updated.ValueOrZero() // This will be put into the cache later. } @@ -305,7 +302,7 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in populateEncounterCacheVal() if encounterCacheVal.FirstEncounter == 0 { // This is first encounter - encounterCacheVal.FirstEncounter = new.Updated.ValueOrZero() + encounterCacheVal.FirstEncounter = pokemon.Updated.ValueOrZero() if encounterCacheVal.FirstWild > 0 { timeToEncounter = encounterCacheVal.FirstEncounter - encounterCacheVal.FirstWild @@ -313,8 +310,8 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in monsIvIncr = 1 - if new.ExpireTimestampVerified { - tth := new.ExpireTimestamp.ValueOrZero() - new.Updated.ValueOrZero() // relies on Updated being set + if pokemon.ExpireTimestampVerified { + tth := pokemon.ExpireTimestamp.ValueOrZero() - pokemon.Updated.ValueOrZero() // relies on Updated being set bucket = tth / (5 * 60) if bucket > 11 { bucket = 11 @@ -325,8 +322,8 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in unverifiedEncIncr = 1 } } else { - if new.ExpireTimestampVerified { - tth := new.ExpireTimestamp.ValueOrZero() - new.Updated.ValueOrZero() // relies on Updated being set + if pokemon.ExpireTimestampVerified { + tth := pokemon.ExpireTimestamp.ValueOrZero() - pokemon.Updated.ValueOrZero() // relies on Updated being set verifiedReEncounterIncr = 1 verifiedReEncSecTotalIncr = tth @@ -337,12 +334,12 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in // If we have a cache entry, it means we updated it. So now let's store it. if encounterCacheVal != nil { - encounterCache.Put(new.Id, encounterCacheVal, new.remainingDuration(now)) + encounterCache.Put(pokemon.Id, encounterCacheVal, pokemon.remainingDuration(now)) } if (currentSeenType == SeenType_Wild && oldSeenType == SeenType_Encounter) || (currentSeenType == SeenType_Encounter && oldSeenType == SeenType_Encounter && - new.PokemonId != old.PokemonId) { + pokemon.PokemonId != pokemon.oldValues.PokemonId) { // stats reset statsResetCountIncr = 1 } @@ -352,10 +349,10 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in var isHundo bool var isNundo bool - if new.Cp.Valid && new.AtkIv.Valid && new.DefIv.Valid && new.StaIv.Valid { - atk := new.AtkIv.ValueOrZero() - def := new.DefIv.ValueOrZero() - sta := new.StaIv.ValueOrZero() + if pokemon.Cp.Valid && pokemon.AtkIv.Valid && pokemon.DefIv.Valid && pokemon.StaIv.Valid { + atk := pokemon.AtkIv.ValueOrZero() + def := pokemon.DefIv.ValueOrZero() + sta := pokemon.StaIv.ValueOrZero() if atk == 15 && def == 15 && sta == 15 { isHundo = true } else if atk == 0 && def == 0 && sta == 0 { @@ -369,7 +366,7 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in // Count stats - if old == nil || old.Cp != new.Cp { // pokemon is new or CP has changed (encountered or re-encountered) + if pokemon.isNewRecord() || pokemon.oldValues.Cp != pokemon.Cp { // pokemon is new or CP has changed (encountered or re-encountered) if !locked { pokemonStatsLock.Lock() locked = true @@ -387,18 +384,18 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in pokemonCount[area] = countStats } - formId := int(new.Form.ValueOrZero()) - pf := pokemonForm{pokemonId: new.PokemonId, formId: formId} + formId := int(pokemon.Form.ValueOrZero()) + pf := pokemonForm{pokemonId: pokemon.PokemonId, formId: formId} - if old == nil || old.PokemonId != new.PokemonId { // pokemon is new or type has changed + if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId { // pokemon is new or type has changed countStats.count[pf]++ statsCollector.IncPokemonCountNew(fullAreaName) - if new.ExpireTimestampVerified { - statsCollector.UpdateVerifiedTtl(area, new.SeenType, new.ExpireTimestamp) + if pokemon.ExpireTimestampVerified { + statsCollector.UpdateVerifiedTtl(area, pokemon.SeenType, pokemon.ExpireTimestamp) } } - if new.Cp.Valid { + if pokemon.Cp.Valid { countStats.ivCount[pf]++ statsCollector.IncPokemonCountIv(fullAreaName) if isHundo { @@ -448,7 +445,7 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in } } -func updateRaidStats(old *Gym, new *Gym, areas []geo.AreaName) { +func updateRaidStats(gym *Gym, areas []geo.AreaName) { if len(areas) == 0 { areas = []geo.AreaName{{Parent: "unmatched", Name: "unmatched"}} } @@ -459,8 +456,8 @@ func updateRaidStats(old *Gym, new *Gym, areas []geo.AreaName) { for i := 0; i < len(areas); i++ { area := areas[i] - if new.RaidPokemonId.ValueOrZero() > 0 && - (old == nil || old.RaidPokemonId != new.RaidPokemonId || old.RaidEndTimestamp != new.RaidEndTimestamp) { + if gym.RaidPokemonId.ValueOrZero() > 0 && + (gym.newRecord || gym.oldValues.RaidPokemonId != gym.RaidPokemonId || gym.oldValues.RaidSpawnTimestamp != gym.RaidSpawnTimestamp) { if !locked { raidStatsLock.Lock() @@ -471,13 +468,13 @@ func updateRaidStats(old *Gym, new *Gym, areas []geo.AreaName) { raidCount[area] = make(map[int64]*areaRaidCountDetail) } countStats := raidCount[area] - raidLevel := new.RaidLevel.ValueOrZero() + raidLevel := gym.RaidLevel.ValueOrZero() if countStats[raidLevel] == nil { countStats[raidLevel] = &areaRaidCountDetail{count: make(map[pokemonForm]int)} } pf := pokemonForm{ - pokemonId: int16(new.RaidPokemonId.ValueOrZero()), - formId: int(new.RaidPokemonForm.ValueOrZero()), + pokemonId: int16(gym.RaidPokemonId.ValueOrZero()), + formId: int(gym.RaidPokemonForm.ValueOrZero()), } countStats[raidLevel].count[pf]++ } diff --git a/decoder/weather_iv.go b/decoder/weather_iv.go index 582e1760..8560c3ec 100644 --- a/decoder/weather_iv.go +++ b/decoder/weather_iv.go @@ -203,7 +203,7 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath pokemonLocked := 0 pokemonUpdated := 0 pokemonCpUpdated := 0 - var pokemon Pokemon + var pokemon *Pokemon pokemonTree2.Search([2]float64{cellLo.Lng.Degrees(), cellLo.Lat.Degrees()}, [2]float64{cellHi.Lng.Degrees(), cellHi.Lat.Degrees()}, func(min, max [2]float64, pokemonId uint64) bool { if !weatherCell.ContainsPoint(s2.PointFromLatLng(s2.LatLngFromDegrees(min[1], min[0]))) { return true @@ -237,7 +237,7 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath pokemon.recomputeCpIfNeeded(ctx, db, map[int64]pogo.GameplayWeatherProto_WeatherCondition{ weatherUpdate.S2CellId: pogo.GameplayWeatherProto_WeatherCondition(newWeather), }) - savePokemonRecordAsAtTime(ctx, db, &pokemon, false, toDB && pokemon.Cp.Valid, pokemon.Cp.Valid, timestamp) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, toDB && pokemon.Cp.Valid, pokemon.Cp.Valid, timestamp) pokemonUpdated++ if pokemon.Cp.Valid { pokemonCpUpdated++ From 80e1826311625d5c1fc5fd364dca8c58a58be65c Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 12:35:28 +0000 Subject: [PATCH 06/78] Update isNewRecord logic --- decoder/pokemon.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 482a5901..8c6c4d7f 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -77,8 +77,8 @@ type Pokemon struct { internal grpc.PokemonInternal - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - // Note: newRecord tracked via FirstSeenTimestamp == 0 (see isNewRecord method) + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` oldValues PokemonOldValues `db:"-" json:"-"` // Old values for webhook comparison and stats } @@ -424,7 +424,7 @@ func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId if pokemon != nil || err != nil { return pokemon, err } - pokemon = &Pokemon{Id: encounterId} + pokemon = &Pokemon{Id: encounterId, newRecord: true} if db.UsePokemonCache { setPokemonCache(encounterId, pokemon, ttlcache.DefaultTTL) } @@ -474,7 +474,7 @@ func hasChangesPokemon(old *Pokemon, new *Pokemon) bool { } func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Pokemon, isEncounter, writeDB, webhook bool, now int64) { - if !pokemon.isNewRecord() && !pokemon.IsDirty() { + if !pokemon.newRecord && !pokemon.IsDirty() { return } @@ -524,7 +524,14 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } - log.Debugf("Updating pokemon [%d] to %s", pokemon.Id, pokemon.SeenType.ValueOrZero()) + var oldSeenType string + if !pokemon.oldValues.SeenType.Valid { + oldSeenType = "n/a" + } else { + oldSeenType = pokemon.oldValues.SeenType.ValueOrZero() + } + + log.Debugf("Updating pokemon [%d] from %s->%s - newRecord: %t", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero(), pokemon.isNewRecord()) //log.Println(cmp.Diff(oldPokemon, pokemon)) if writeDB && !config.Config.PokemonMemoryOnly { @@ -568,7 +575,8 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po return } - _, _ = res, err + rows, rowsErr := res.RowsAffected() + log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) } else { pvpUpdate := "" if changePvpField { @@ -642,6 +650,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } updatePokemonStats(pokemon, areas, now) + pokemon.newRecord = false // After saving, it's no longer a new record pokemon.ClearDirty() pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache @@ -778,7 +787,7 @@ func (pokemon *Pokemon) locateAllScans() (unboosted, boosted, strong *grpc.Pokem } func (pokemon *Pokemon) isNewRecord() bool { - return pokemon.FirstSeenTimestamp == 0 + return pokemon.newRecord } func (pokemon *Pokemon) remainingDuration(now int64) time.Duration { From 2b5b7fb14937b3464ee4b778f6e61c8c30706e3c Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 16:18:28 +0000 Subject: [PATCH 07/78] Additional types converted to new model --- decoder/gym.go | 2 +- decoder/incident.go | 233 +++++++++--- decoder/main.go | 76 ++-- decoder/player.go | 834 ++++++++++++++++++++++++++++++++---------- decoder/pokemon.go | 32 +- decoder/pokestop.go | 2 +- decoder/routes.go | 273 ++++++++++---- decoder/s2cell.go | 17 +- decoder/spawnpoint.go | 135 ++++--- decoder/station.go | 364 ++++++++++++++---- decoder/stats.go | 10 +- decoder/tappable.go | 164 ++++++--- decoder/weather.go | 219 ++++++++--- 13 files changed, 1782 insertions(+), 579 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index 9b101839..dba5fb34 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -1052,7 +1052,7 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { _, _ = res, err } - gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) areas := MatchStatsGeofence(gym.Lat, gym.Lon) createGymWebhooks(gym, areas) createGymFortWebhooks(gym) diff --git a/decoder/incident.go b/decoder/incident.go index 7e5fbde5..dbc0b067 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -5,7 +5,6 @@ import ( "database/sql" "time" - "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" null "gopkg.in/guregu/null.v4" @@ -15,7 +14,7 @@ import ( ) // Incident struct. -// REMINDER! Keep hasChangesIncident updated after making changes +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Incident struct { Id string `db:"id"` PokestopId string `db:"pokestop_id"` @@ -32,6 +31,20 @@ type Incident struct { Slot2Form null.Int `db:"slot_2_form"` Slot3PokemonId null.Int `db:"slot_3_pokemon_id"` Slot3Form null.Int `db:"slot_3_form"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + oldValues IncidentOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// IncidentOldValues holds old field values for webhook comparison and stats +type IncidentOldValues struct { + StartTime int64 + ExpirationTime int64 + Character int16 + Confirmed bool + Slot1PokemonId null.Int } type webhookLineup struct { @@ -69,11 +82,139 @@ type IncidentWebhook struct { //-> `character` smallint unsigned NOT NULL, //-> `updated` int unsigned NOT NULL, +// IsDirty returns true if any field has been modified +func (incident *Incident) IsDirty() bool { + return incident.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (incident *Incident) ClearDirty() { + incident.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (incident *Incident) IsNewRecord() bool { + return incident.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (incident *Incident) snapshotOldValues() { + incident.oldValues = IncidentOldValues{ + StartTime: incident.StartTime, + ExpirationTime: incident.ExpirationTime, + Character: incident.Character, + Confirmed: incident.Confirmed, + Slot1PokemonId: incident.Slot1PokemonId, + } +} + +// --- Set methods with dirty tracking --- + +func (incident *Incident) SetId(v string) { + if incident.Id != v { + incident.Id = v + incident.dirty = true + } +} + +func (incident *Incident) SetPokestopId(v string) { + if incident.PokestopId != v { + incident.PokestopId = v + incident.dirty = true + } +} + +func (incident *Incident) SetStartTime(v int64) { + if incident.StartTime != v { + incident.StartTime = v + incident.dirty = true + } +} + +func (incident *Incident) SetExpirationTime(v int64) { + if incident.ExpirationTime != v { + incident.ExpirationTime = v + incident.dirty = true + } +} + +func (incident *Incident) SetDisplayType(v int16) { + if incident.DisplayType != v { + incident.DisplayType = v + incident.dirty = true + } +} + +func (incident *Incident) SetStyle(v int16) { + if incident.Style != v { + incident.Style = v + incident.dirty = true + } +} + +func (incident *Incident) SetCharacter(v int16) { + if incident.Character != v { + incident.Character = v + incident.dirty = true + } +} + +func (incident *Incident) SetConfirmed(v bool) { + if incident.Confirmed != v { + incident.Confirmed = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot1PokemonId(v null.Int) { + if incident.Slot1PokemonId != v { + incident.Slot1PokemonId = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot1Form(v null.Int) { + if incident.Slot1Form != v { + incident.Slot1Form = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot2PokemonId(v null.Int) { + if incident.Slot2PokemonId != v { + incident.Slot2PokemonId = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot2Form(v null.Int) { + if incident.Slot2Form != v { + incident.Slot2Form = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot3PokemonId(v null.Int) { + if incident.Slot3PokemonId != v { + incident.Slot3PokemonId = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot3Form(v null.Int) { + if incident.Slot3Form != v { + incident.Slot3Form = v + incident.dirty = true + } +} + func getIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, error) { inMemoryIncident := incidentCache.Get(incidentId) if inMemoryIncident != nil { incident := inMemoryIncident.Value() - return &incident, nil + incident.snapshotOldValues() + return incident, nil } incident := Incident{} @@ -90,44 +231,19 @@ func getIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string) return nil, err } - incidentCache.Set(incidentId, incident, ttlcache.DefaultTTL) + incident.snapshotOldValues() return &incident, nil } -// hasChangesIncident compares two Incident structs -func hasChangesIncident(old *Incident, new *Incident) bool { - return old.Id != new.Id || - old.PokestopId != new.PokestopId || - old.StartTime != new.StartTime || - old.ExpirationTime != new.ExpirationTime || - old.DisplayType != new.DisplayType || - old.Style != new.Style || - old.Character != new.Character || - old.Confirmed != new.Confirmed || - old.Updated != new.Updated || - old.Slot1PokemonId != new.Slot1PokemonId || - old.Slot1Form != new.Slot1Form || - old.Slot2PokemonId != new.Slot2PokemonId || - old.Slot2Form != new.Slot2Form || - old.Slot3PokemonId != new.Slot3PokemonId || - old.Slot3Form != new.Slot3Form - -} - func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident) { - oldIncident, _ := getIncidentRecord(ctx, db, incident.Id) - - if oldIncident != nil && !hasChangesIncident(oldIncident, incident) { + // Skip save if not dirty and not new + if !incident.IsDirty() && !incident.IsNewRecord() { return } - //log.Traceln(cmp.Diff(oldIncident, incident)) - incident.Updated = time.Now().Unix() - //log.Println(cmp.Diff(oldIncident, incident)) - - if oldIncident == nil { + if incident.IsNewRecord() { res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) @@ -161,8 +277,7 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident _, _ = res, err } - incidentCache.Set(incident.Id, *incident, ttlcache.DefaultTTL) - createIncidentWebhooks(ctx, db, oldIncident, incident) + createIncidentWebhooks(ctx, db, incident) stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId) if stop == nil { @@ -170,11 +285,18 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident } areas := MatchStatsGeofence(stop.Lat, stop.Lon) - updateIncidentStats(oldIncident, incident, areas) + updateIncidentStats(incident, areas) + + incident.ClearDirty() + incident.newRecord = false + //incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) } -func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *Incident, incident *Incident) { - if oldIncident == nil || (oldIncident.ExpirationTime != incident.ExpirationTime || oldIncident.Character != incident.Character || oldIncident.Confirmed != incident.Confirmed || oldIncident.Slot1PokemonId != incident.Slot1PokemonId) { +func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { + old := &incident.oldValues + isNew := incident.IsNewRecord() + + if isNew || (old.ExpirationTime != incident.ExpirationTime || old.Character != incident.Character || old.Confirmed != incident.Confirmed || old.Slot1PokemonId != incident.Slot1PokemonId) { stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId) if stop == nil { stop = &Pokestop{} @@ -233,10 +355,10 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *I } func (incident *Incident) updateFromPokestopIncidentDisplay(pokestopDisplay *pogo.PokestopIncidentDisplayProto) { - incident.Id = pokestopDisplay.IncidentId - incident.StartTime = int64(pokestopDisplay.IncidentStartMs / 1000) - incident.ExpirationTime = int64(pokestopDisplay.IncidentExpirationMs / 1000) - incident.DisplayType = int16(pokestopDisplay.IncidentDisplayType) + incident.SetId(pokestopDisplay.IncidentId) + incident.SetStartTime(int64(pokestopDisplay.IncidentStartMs / 1000)) + incident.SetExpirationTime(int64(pokestopDisplay.IncidentExpirationMs / 1000)) + incident.SetDisplayType(int16(pokestopDisplay.IncidentDisplayType)) if (incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE)) && incident.Confirmed { log.Debugf("Incident has already been confirmed as a decoy: %s", incident.Id) return @@ -244,35 +366,36 @@ func (incident *Incident) updateFromPokestopIncidentDisplay(pokestopDisplay *pog characterDisplay := pokestopDisplay.GetCharacterDisplay() if characterDisplay != nil { // team := pokestopDisplay.Open - incident.Style = int16(characterDisplay.Style) - incident.Character = int16(characterDisplay.Character) + incident.SetStyle(int16(characterDisplay.Style)) + incident.SetCharacter(int16(characterDisplay.Character)) } else { - incident.Style, incident.Character = 0, 0 + incident.SetStyle(0) + incident.SetCharacter(0) } } func (incident *Incident) updateFromOpenInvasionCombatSessionOut(protoRes *pogo.OpenInvasionCombatSessionOutProto) { - incident.Slot1PokemonId = null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokedexId.Number()), true) - incident.Slot1Form = null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokemonDisplay.Form.Number()), true) + incident.SetSlot1PokemonId(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokedexId.Number()), true)) + incident.SetSlot1Form(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokemonDisplay.Form.Number()), true)) for i, pokemon := range protoRes.Combat.Opponent.ReservePokemon { if i == 0 { - incident.Slot2PokemonId = null.NewInt(int64(pokemon.PokedexId.Number()), true) - incident.Slot2Form = null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true) + incident.SetSlot2PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) + incident.SetSlot2Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) } else if i == 1 { - incident.Slot3PokemonId = null.NewInt(int64(pokemon.PokedexId.Number()), true) - incident.Slot3Form = null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true) + incident.SetSlot3PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) + incident.SetSlot3Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) } } - incident.Confirmed = true + incident.SetConfirmed(true) } func (incident *Incident) updateFromStartIncidentOut(proto *pogo.StartIncidentOutProto) { - incident.Character = int16(proto.GetIncident().GetStep()[0].GetPokestopDialogue().GetDialogueLine()[0].GetCharacter()) + incident.SetCharacter(int16(proto.GetIncident().GetStep()[0].GetPokestopDialogue().GetDialogueLine()[0].GetCharacter())) if incident.Character == int16(pogo.EnumWrapper_CHARACTER_GIOVANNI) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE) { - incident.Confirmed = true + incident.SetConfirmed(true) } - incident.StartTime = int64(proto.Incident.GetCompletionDisplay().GetIncidentStartMs() / 1000) - incident.ExpirationTime = int64(proto.Incident.GetCompletionDisplay().GetIncidentExpirationMs() / 1000) + incident.SetStartTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentStartMs() / 1000)) + incident.SetExpirationTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentExpirationMs() / 1000)) } diff --git a/decoder/main.go b/decoder/main.go index 9f3bdad5..e398e5e9 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -60,27 +60,27 @@ var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector var pokestopCache *ttlcache.Cache[string, *Pokestop] var gymCache *ttlcache.Cache[string, *Gym] -var stationCache *ttlcache.Cache[string, Station] -var tappableCache *ttlcache.Cache[uint64, Tappable] -var weatherCache *ttlcache.Cache[int64, Weather] +var stationCache *ttlcache.Cache[string, *Station] +var tappableCache *ttlcache.Cache[uint64, *Tappable] +var weatherCache *ttlcache.Cache[int64, *Weather] var weatherConsensusCache *ttlcache.Cache[int64, *WeatherConsensusState] -var s2CellCache *ttlcache.Cache[uint64, S2Cell] -var spawnpointCache *ttlcache.Cache[int64, Spawnpoint] +var s2CellCache *ttlcache.Cache[uint64, *S2Cell] +var spawnpointCache *ttlcache.Cache[int64, *Spawnpoint] var pokemonCache []*ttlcache.Cache[uint64, *Pokemon] -var incidentCache *ttlcache.Cache[string, Incident] -var playerCache *ttlcache.Cache[string, Player] -var routeCache *ttlcache.Cache[string, Route] +var incidentCache *ttlcache.Cache[string, *Incident] +var playerCache *ttlcache.Cache[string, *Player] +var routeCache *ttlcache.Cache[string, *Route] var diskEncounterCache *ttlcache.Cache[uint64, *pogo.DiskEncounterOutProto] var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto] -var gymStripedMutex = stripedmutex.New(128) -var pokestopStripedMutex = stripedmutex.New(128) -var stationStripedMutex = stripedmutex.New(128) +var gymStripedMutex = stripedmutex.New(1103) +var pokestopStripedMutex = stripedmutex.New(1103) +var stationStripedMutex = stripedmutex.New(1103) var tappableStripedMutex = intstripedmutex.New(563) -var incidentStripedMutex = stripedmutex.New(128) +var incidentStripedMutex = stripedmutex.New(157) var pokemonStripedMutex = intstripedmutex.New(1103) var weatherStripedMutex = intstripedmutex.New(157) -var routeStripedMutex = stripedmutex.New(128) +var routeStripedMutex = stripedmutex.New(157) var ProactiveIVSwitchSem chan bool @@ -128,18 +128,18 @@ func initDataCache() { ) go gymCache.Start() - stationCache = ttlcache.New[string, Station]( - ttlcache.WithTTL[string, Station](60 * time.Minute), + stationCache = ttlcache.New[string, *Station]( + ttlcache.WithTTL[string, *Station](60 * time.Minute), ) go stationCache.Start() - tappableCache = ttlcache.New[uint64, Tappable]( - ttlcache.WithTTL[uint64, Tappable](60 * time.Minute), + tappableCache = ttlcache.New[uint64, *Tappable]( + ttlcache.WithTTL[uint64, *Tappable](60 * time.Minute), ) go tappableCache.Start() - weatherCache = ttlcache.New[int64, Weather]( - ttlcache.WithTTL[int64, Weather](60 * time.Minute), + weatherCache = ttlcache.New[int64, *Weather]( + ttlcache.WithTTL[int64, *Weather](60 * time.Minute), ) go weatherCache.Start() @@ -148,13 +148,13 @@ func initDataCache() { ) go weatherConsensusCache.Start() - s2CellCache = ttlcache.New[uint64, S2Cell]( - ttlcache.WithTTL[uint64, S2Cell](60 * time.Minute), + s2CellCache = ttlcache.New[uint64, *S2Cell]( + ttlcache.WithTTL[uint64, *S2Cell](60 * time.Minute), ) go s2CellCache.Start() - spawnpointCache = ttlcache.New[int64, Spawnpoint]( - ttlcache.WithTTL[int64, Spawnpoint](60 * time.Minute), + spawnpointCache = ttlcache.New[int64, *Spawnpoint]( + ttlcache.WithTTL[int64, *Spawnpoint](60 * time.Minute), ) go spawnpointCache.Start() @@ -171,13 +171,13 @@ func initDataCache() { initPokemonRtree() initFortRtree() - incidentCache = ttlcache.New[string, Incident]( - ttlcache.WithTTL[string, Incident](60 * time.Minute), + incidentCache = ttlcache.New[string, *Incident]( + ttlcache.WithTTL[string, *Incident](60 * time.Minute), ) go incidentCache.Start() - playerCache = ttlcache.New[string, Player]( - ttlcache.WithTTL[string, Player](60 * time.Minute), + playerCache = ttlcache.New[string, *Player]( + ttlcache.WithTTL[string, *Player](60 * time.Minute), ) go playerCache.Start() @@ -193,8 +193,8 @@ func initDataCache() { ) go getMapFortsCache.Start() - routeCache = ttlcache.New[string, Route]( - ttlcache.WithTTL[string, Route](60 * time.Minute), + routeCache = ttlcache.New[string, *Route]( + ttlcache.WithTTL[string, *Route](60 * time.Minute), ) go routeCache.Start() } @@ -331,6 +331,7 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa if incident == nil { incident = &Incident{ PokestopId: fortId, + newRecord: true, } } incident.updateFromPokestopIncidentDisplay(incidentProto) @@ -385,7 +386,7 @@ func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters Sca continue } if station == nil { - station = &Station{} + station = &Station{newRecord: true} } station.updateFromStationProto(stationProto.Data, stationProto.Cell) saveStationRecord(ctx, db, station) @@ -440,8 +441,11 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca if err != nil { log.Printf("getOrCreatePokemonRecord: %s", err) } else { - pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) + updateTime := nearby.Timestamp / 1000 + if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { + pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) + } } pokemonMutex.Unlock() @@ -489,12 +493,12 @@ func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.Cl publishProto = weatherProto } if weather == nil { - weather = &Weather{} + weather = &Weather{newRecord: true} } weather.UpdatedMs = timestampMs - oldWeather := weather.updateWeatherFromClientWeatherProto(publishProto) + weather.updateWeatherFromClientWeatherProto(publishProto) saveWeatherRecord(ctx, db, weather) - if oldWeather != weather.GameplayCondition { + if weather.oldValues.GameplayCondition != weather.GameplayCondition { updates = append(updates, WeatherUpdate{ S2CellId: publishProto.S2CellId, NewWeather: int32(publishProto.GetGameplayWeather().GetGameplayCondition()), @@ -527,6 +531,7 @@ func UpdateIncidentLineup(ctx context.Context, db db.DbDetails, protoReq *pogo.O incident = &Incident{ Id: protoReq.IncidentLookup.IncidentId, PokestopId: protoReq.IncidentLookup.FortId, + newRecord: true, } } incident.updateFromOpenInvasionCombatSessionOut(protoRes) @@ -550,6 +555,7 @@ func ConfirmIncident(ctx context.Context, db db.DbDetails, proto *pogo.StartInci incident = &Incident{ Id: proto.Incident.IncidentId, PokestopId: proto.Incident.FortId, + newRecord: true, } } incident.updateFromStartIncidentOut(proto) diff --git a/decoder/player.go b/decoder/player.go index 6c359f1f..721f7d88 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -15,7 +15,7 @@ import ( ) // Player struct. Name is the primary key. -// REMINDER! Keep hasChangesPlayer updated after making changes +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Player struct { // Name is the primary key Name string `db:"name"` @@ -102,6 +102,535 @@ type Player struct { CaughtDragon null.Int `db:"caught_dragon"` CaughtDark null.Int `db:"caught_dark"` CaughtFairy null.Int `db:"caught_fairy"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record +} + +// IsDirty returns true if any field has been modified +func (p *Player) IsDirty() bool { + return p.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (p *Player) ClearDirty() { + p.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (p *Player) IsNewRecord() bool { + return p.newRecord +} + +// setFieldDirty marks the dirty flag. Used by reflection-based updates. +func (p *Player) setFieldDirty() { + p.dirty = true +} + +// --- Set methods with dirty tracking --- + +func (p *Player) SetFriendshipId(v null.String) { + if p.FriendshipId != v { + p.FriendshipId = v + p.dirty = true + } +} + +func (p *Player) SetFriendCode(v null.String) { + if p.FriendCode != v { + p.FriendCode = v + p.dirty = true + } +} + +func (p *Player) SetTeam(v null.Int) { + if p.Team != v { + p.Team = v + p.dirty = true + } +} + +func (p *Player) SetLevel(v null.Int) { + if p.Level != v { + p.Level = v + p.dirty = true + } +} + +func (p *Player) SetXp(v null.Int) { + if p.Xp != v { + p.Xp = v + p.dirty = true + } +} + +func (p *Player) SetBattlesWon(v null.Int) { + if p.BattlesWon != v { + p.BattlesWon = v + p.dirty = true + } +} + +func (p *Player) SetKmWalked(v null.Float) { + if !nullFloatAlmostEqual(p.KmWalked, v, 0.001) { + p.KmWalked = v + p.dirty = true + } +} + +func (p *Player) SetCaughtPokemon(v null.Int) { + if p.CaughtPokemon != v { + p.CaughtPokemon = v + p.dirty = true + } +} + +func (p *Player) SetGblRank(v null.Int) { + if p.GblRank != v { + p.GblRank = v + p.dirty = true + } +} + +func (p *Player) SetGblRating(v null.Int) { + if p.GblRating != v { + p.GblRating = v + p.dirty = true + } +} + +func (p *Player) SetEventBadges(v null.String) { + if p.EventBadges != v { + p.EventBadges = v + p.dirty = true + } +} + +func (p *Player) SetStopsSpun(v null.Int) { + if p.StopsSpun != v { + p.StopsSpun = v + p.dirty = true + } +} +func (p *Player) SetEvolved(v null.Int) { + if p.Evolved != v { + p.Evolved = v + p.dirty = true + } +} +func (p *Player) SetHatched(v null.Int) { + if p.Hatched != v { + p.Hatched = v + p.dirty = true + } +} +func (p *Player) SetQuests(v null.Int) { + if p.Quests != v { + p.Quests = v + p.dirty = true + } +} +func (p *Player) SetTrades(v null.Int) { + if p.Trades != v { + p.Trades = v + p.dirty = true + } +} +func (p *Player) SetPhotobombs(v null.Int) { + if p.Photobombs != v { + p.Photobombs = v + p.dirty = true + } +} +func (p *Player) SetPurified(v null.Int) { + if p.Purified != v { + p.Purified = v + p.dirty = true + } +} +func (p *Player) SetGruntsDefeated(v null.Int) { + if p.GruntsDefeated != v { + p.GruntsDefeated = v + p.dirty = true + } +} +func (p *Player) SetGymBattlesWon(v null.Int) { + if p.GymBattlesWon != v { + p.GymBattlesWon = v + p.dirty = true + } +} +func (p *Player) SetNormalRaidsWon(v null.Int) { + if p.NormalRaidsWon != v { + p.NormalRaidsWon = v + p.dirty = true + } +} +func (p *Player) SetLegendaryRaidsWon(v null.Int) { + if p.LegendaryRaidsWon != v { + p.LegendaryRaidsWon = v + p.dirty = true + } +} +func (p *Player) SetTrainingsWon(v null.Int) { + if p.TrainingsWon != v { + p.TrainingsWon = v + p.dirty = true + } +} +func (p *Player) SetBerriesFed(v null.Int) { + if p.BerriesFed != v { + p.BerriesFed = v + p.dirty = true + } +} +func (p *Player) SetHoursDefended(v null.Int) { + if p.HoursDefended != v { + p.HoursDefended = v + p.dirty = true + } +} +func (p *Player) SetBestFriends(v null.Int) { + if p.BestFriends != v { + p.BestFriends = v + p.dirty = true + } +} +func (p *Player) SetBestBuddies(v null.Int) { + if p.BestBuddies != v { + p.BestBuddies = v + p.dirty = true + } +} +func (p *Player) SetGiovanniDefeated(v null.Int) { + if p.GiovanniDefeated != v { + p.GiovanniDefeated = v + p.dirty = true + } +} +func (p *Player) SetMegaEvos(v null.Int) { + if p.MegaEvos != v { + p.MegaEvos = v + p.dirty = true + } +} +func (p *Player) SetCollectionsDone(v null.Int) { + if p.CollectionsDone != v { + p.CollectionsDone = v + p.dirty = true + } +} +func (p *Player) SetUniqueStopsSpun(v null.Int) { + if p.UniqueStopsSpun != v { + p.UniqueStopsSpun = v + p.dirty = true + } +} +func (p *Player) SetUniqueMegaEvos(v null.Int) { + if p.UniqueMegaEvos != v { + p.UniqueMegaEvos = v + p.dirty = true + } +} +func (p *Player) SetUniqueRaidBosses(v null.Int) { + if p.UniqueRaidBosses != v { + p.UniqueRaidBosses = v + p.dirty = true + } +} +func (p *Player) SetUniqueUnown(v null.Int) { + if p.UniqueUnown != v { + p.UniqueUnown = v + p.dirty = true + } +} +func (p *Player) SetSevenDayStreaks(v null.Int) { + if p.SevenDayStreaks != v { + p.SevenDayStreaks = v + p.dirty = true + } +} +func (p *Player) SetTradeKm(v null.Int) { + if p.TradeKm != v { + p.TradeKm = v + p.dirty = true + } +} +func (p *Player) SetRaidsWithFriends(v null.Int) { + if p.RaidsWithFriends != v { + p.RaidsWithFriends = v + p.dirty = true + } +} +func (p *Player) SetCaughtAtLure(v null.Int) { + if p.CaughtAtLure != v { + p.CaughtAtLure = v + p.dirty = true + } +} +func (p *Player) SetWayfarerAgreements(v null.Int) { + if p.WayfarerAgreements != v { + p.WayfarerAgreements = v + p.dirty = true + } +} +func (p *Player) SetTrainersReferred(v null.Int) { + if p.TrainersReferred != v { + p.TrainersReferred = v + p.dirty = true + } +} +func (p *Player) SetRaidAchievements(v null.Int) { + if p.RaidAchievements != v { + p.RaidAchievements = v + p.dirty = true + } +} +func (p *Player) SetXlKarps(v null.Int) { + if p.XlKarps != v { + p.XlKarps = v + p.dirty = true + } +} +func (p *Player) SetXsRats(v null.Int) { + if p.XsRats != v { + p.XsRats = v + p.dirty = true + } +} +func (p *Player) SetPikachuCaught(v null.Int) { + if p.PikachuCaught != v { + p.PikachuCaught = v + p.dirty = true + } +} +func (p *Player) SetLeagueGreatWon(v null.Int) { + if p.LeagueGreatWon != v { + p.LeagueGreatWon = v + p.dirty = true + } +} +func (p *Player) SetLeagueUltraWon(v null.Int) { + if p.LeagueUltraWon != v { + p.LeagueUltraWon = v + p.dirty = true + } +} +func (p *Player) SetLeagueMasterWon(v null.Int) { + if p.LeagueMasterWon != v { + p.LeagueMasterWon = v + p.dirty = true + } +} +func (p *Player) SetTinyPokemonCaught(v null.Int) { + if p.TinyPokemonCaught != v { + p.TinyPokemonCaught = v + p.dirty = true + } +} +func (p *Player) SetJumboPokemonCaught(v null.Int) { + if p.JumboPokemonCaught != v { + p.JumboPokemonCaught = v + p.dirty = true + } +} +func (p *Player) SetVivillon(v null.Int) { + if p.Vivillon != v { + p.Vivillon = v + p.dirty = true + } +} +func (p *Player) SetMaxSizeFirstPlace(v null.Int) { + if p.MaxSizeFirstPlace != v { + p.MaxSizeFirstPlace = v + p.dirty = true + } +} +func (p *Player) SetTotalRoutePlay(v null.Int) { + if p.TotalRoutePlay != v { + p.TotalRoutePlay = v + p.dirty = true + } +} +func (p *Player) SetPartiesCompleted(v null.Int) { + if p.PartiesCompleted != v { + p.PartiesCompleted = v + p.dirty = true + } +} +func (p *Player) SetEventCheckIns(v null.Int) { + if p.EventCheckIns != v { + p.EventCheckIns = v + p.dirty = true + } +} +func (p *Player) SetDexGen1(v null.Int) { + if p.DexGen1 != v { + p.DexGen1 = v + p.dirty = true + } +} +func (p *Player) SetDexGen2(v null.Int) { + if p.DexGen2 != v { + p.DexGen2 = v + p.dirty = true + } +} +func (p *Player) SetDexGen3(v null.Int) { + if p.DexGen3 != v { + p.DexGen3 = v + p.dirty = true + } +} +func (p *Player) SetDexGen4(v null.Int) { + if p.DexGen4 != v { + p.DexGen4 = v + p.dirty = true + } +} +func (p *Player) SetDexGen5(v null.Int) { + if p.DexGen5 != v { + p.DexGen5 = v + p.dirty = true + } +} +func (p *Player) SetDexGen6(v null.Int) { + if p.DexGen6 != v { + p.DexGen6 = v + p.dirty = true + } +} +func (p *Player) SetDexGen7(v null.Int) { + if p.DexGen7 != v { + p.DexGen7 = v + p.dirty = true + } +} +func (p *Player) SetDexGen8(v null.Int) { + if p.DexGen8 != v { + p.DexGen8 = v + p.dirty = true + } +} +func (p *Player) SetDexGen8A(v null.Int) { + if p.DexGen8A != v { + p.DexGen8A = v + p.dirty = true + } +} +func (p *Player) SetDexGen9(v null.Int) { + if p.DexGen9 != v { + p.DexGen9 = v + p.dirty = true + } +} +func (p *Player) SetCaughtNormal(v null.Int) { + if p.CaughtNormal != v { + p.CaughtNormal = v + p.dirty = true + } +} +func (p *Player) SetCaughtFighting(v null.Int) { + if p.CaughtFighting != v { + p.CaughtFighting = v + p.dirty = true + } +} +func (p *Player) SetCaughtFlying(v null.Int) { + if p.CaughtFlying != v { + p.CaughtFlying = v + p.dirty = true + } +} +func (p *Player) SetCaughtPoison(v null.Int) { + if p.CaughtPoison != v { + p.CaughtPoison = v + p.dirty = true + } +} +func (p *Player) SetCaughtGround(v null.Int) { + if p.CaughtGround != v { + p.CaughtGround = v + p.dirty = true + } +} +func (p *Player) SetCaughtRock(v null.Int) { + if p.CaughtRock != v { + p.CaughtRock = v + p.dirty = true + } +} +func (p *Player) SetCaughtBug(v null.Int) { + if p.CaughtBug != v { + p.CaughtBug = v + p.dirty = true + } +} +func (p *Player) SetCaughtGhost(v null.Int) { + if p.CaughtGhost != v { + p.CaughtGhost = v + p.dirty = true + } +} +func (p *Player) SetCaughtSteel(v null.Int) { + if p.CaughtSteel != v { + p.CaughtSteel = v + p.dirty = true + } +} +func (p *Player) SetCaughtFire(v null.Int) { + if p.CaughtFire != v { + p.CaughtFire = v + p.dirty = true + } +} +func (p *Player) SetCaughtWater(v null.Int) { + if p.CaughtWater != v { + p.CaughtWater = v + p.dirty = true + } +} +func (p *Player) SetCaughtGrass(v null.Int) { + if p.CaughtGrass != v { + p.CaughtGrass = v + p.dirty = true + } +} +func (p *Player) SetCaughtElectric(v null.Int) { + if p.CaughtElectric != v { + p.CaughtElectric = v + p.dirty = true + } +} +func (p *Player) SetCaughtPsychic(v null.Int) { + if p.CaughtPsychic != v { + p.CaughtPsychic = v + p.dirty = true + } +} +func (p *Player) SetCaughtIce(v null.Int) { + if p.CaughtIce != v { + p.CaughtIce = v + p.dirty = true + } +} +func (p *Player) SetCaughtDragon(v null.Int) { + if p.CaughtDragon != v { + p.CaughtDragon = v + p.dirty = true + } +} +func (p *Player) SetCaughtDark(v null.Int) { + if p.CaughtDark != v { + p.CaughtDark = v + p.dirty = true + } +} +func (p *Player) SetCaughtFairy(v null.Int) { + if p.CaughtFairy != v { + p.CaughtFairy = v + p.dirty = true + } } var badgeTypeToPlayerKey = map[pogo.HoloBadgeType]string{ @@ -194,7 +723,7 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo inMemoryPlayer := playerCache.Get(name) if inMemoryPlayer != nil { player := inMemoryPlayer.Value() - return &player, nil + return player, nil } player := Player{} @@ -202,7 +731,7 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo ` SELECT * FROM player - WHERE player.name = ? + WHERE player.name = ? `, name, ) @@ -213,7 +742,7 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo ` SELECT * FROM player - WHERE player.friendship_id = ? + WHERE player.friendship_id = ? `, friendshipId, ) @@ -223,7 +752,7 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo ` SELECT * FROM player - WHERE player.friend_code = ? + WHERE player.friend_code = ? `, friendCode, ) @@ -243,111 +772,19 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo return nil, err } - playerCache.Set(name, player, ttlcache.DefaultTTL) + playerCache.Set(name, &player, ttlcache.DefaultTTL) return &player, nil } -// hasChangesPlayer compares two Player structs -// Float tolerance: KmWalked = 0.001 -func hasChangesPlayer(old *Player, new *Player) bool { - return old.Name != new.Name || - old.FriendshipId != new.FriendshipId || - old.LastSeen != new.LastSeen || - old.FriendCode != new.FriendCode || - old.Team != new.Team || - old.Level != new.Level || - old.Xp != new.Xp || - old.BattlesWon != new.BattlesWon || - old.CaughtPokemon != new.CaughtPokemon || - old.GblRank != new.GblRank || - old.GblRating != new.GblRating || - old.EventBadges != new.EventBadges || - old.StopsSpun != new.StopsSpun || - old.Evolved != new.Evolved || - old.Hatched != new.Hatched || - old.Quests != new.Quests || - old.Trades != new.Trades || - old.Photobombs != new.Photobombs || - old.Purified != new.Purified || - old.GruntsDefeated != new.GruntsDefeated || - old.GymBattlesWon != new.GymBattlesWon || - old.NormalRaidsWon != new.NormalRaidsWon || - old.LegendaryRaidsWon != new.LegendaryRaidsWon || - old.TrainingsWon != new.TrainingsWon || - old.BerriesFed != new.BerriesFed || - old.HoursDefended != new.HoursDefended || - old.BestFriends != new.BestFriends || - old.BestBuddies != new.BestBuddies || - old.GiovanniDefeated != new.GiovanniDefeated || - old.MegaEvos != new.MegaEvos || - old.CollectionsDone != new.CollectionsDone || - old.UniqueStopsSpun != new.UniqueStopsSpun || - old.UniqueMegaEvos != new.UniqueMegaEvos || - old.UniqueRaidBosses != new.UniqueRaidBosses || - old.UniqueUnown != new.UniqueUnown || - old.SevenDayStreaks != new.SevenDayStreaks || - old.TradeKm != new.TradeKm || - old.RaidsWithFriends != new.RaidsWithFriends || - old.CaughtAtLure != new.CaughtAtLure || - old.WayfarerAgreements != new.WayfarerAgreements || - old.TrainersReferred != new.TrainersReferred || - old.RaidAchievements != new.RaidAchievements || - old.XlKarps != new.XlKarps || - old.XsRats != new.XsRats || - old.PikachuCaught != new.PikachuCaught || - old.LeagueGreatWon != new.LeagueGreatWon || - old.LeagueUltraWon != new.LeagueUltraWon || - old.LeagueMasterWon != new.LeagueMasterWon || - old.TinyPokemonCaught != new.TinyPokemonCaught || - old.JumboPokemonCaught != new.JumboPokemonCaught || - old.Vivillon != new.Vivillon || - old.MaxSizeFirstPlace != new.MaxSizeFirstPlace || - old.TotalRoutePlay != new.TotalRoutePlay || - old.PartiesCompleted != new.PartiesCompleted || - old.EventCheckIns != new.EventCheckIns || - old.DexGen1 != new.DexGen1 || - old.DexGen2 != new.DexGen2 || - old.DexGen3 != new.DexGen3 || - old.DexGen4 != new.DexGen4 || - old.DexGen5 != new.DexGen5 || - old.DexGen6 != new.DexGen6 || - old.DexGen7 != new.DexGen7 || - old.DexGen8 != new.DexGen8 || - old.DexGen8A != new.DexGen8A || - old.DexGen9 != new.DexGen9 || - old.CaughtNormal != new.CaughtNormal || - old.CaughtFighting != new.CaughtFighting || - old.CaughtFlying != new.CaughtFlying || - old.CaughtPoison != new.CaughtPoison || - old.CaughtGround != new.CaughtGround || - old.CaughtRock != new.CaughtRock || - old.CaughtBug != new.CaughtBug || - old.CaughtGhost != new.CaughtGhost || - old.CaughtSteel != new.CaughtSteel || - old.CaughtFire != new.CaughtFire || - old.CaughtWater != new.CaughtWater || - old.CaughtGrass != new.CaughtGrass || - old.CaughtElectric != new.CaughtElectric || - old.CaughtPsychic != new.CaughtPsychic || - old.CaughtIce != new.CaughtIce || - old.CaughtDragon != new.CaughtDragon || - old.CaughtDark != new.CaughtDark || - old.CaughtFairy != new.CaughtFairy || - !nullFloatAlmostEqual(old.KmWalked, new.KmWalked, 0.001) -} - func savePlayerRecord(db db.DbDetails, player *Player) { - oldPlayer, _ := getPlayerRecord(db, player.Name, player.FriendshipId.String, player.FriendCode.String) - - if oldPlayer != nil && !hasChangesPlayer(oldPlayer, player) { + // Skip save if not dirty and not new + if !player.IsDirty() && !player.IsNewRecord() { return } - //log.Traceln(cmp.Diff(oldPlayer, player, transformNullFloats, ignoreApproxFloats)) - player.LastSeen = time.Now().Unix() - if oldPlayer == nil { + if player.IsNewRecord() { _, err := db.GeneralDb.NamedExec( ` INSERT INTO player (name, friendship_id, friend_code, last_seen, team, level, xp, battles_won, km_walked, caught_pokemon, gbl_rank, gbl_rating, @@ -384,88 +821,88 @@ func savePlayerRecord(db db.DbDetails, player *Player) { } else { _, err := db.GeneralDb.NamedExec( `UPDATE player SET - friendship_id = :friendship_id, - last_seen = :last_seen, - team = :team, - level = :level, - xp = :xp, - battles_won = :battles_won, - km_walked = :km_walked, - caught_pokemon = :caught_pokemon, - gbl_rank = :gbl_rank, - gbl_rating = :gbl_rating, - event_badges = :event_badges, - stops_spun = :stops_spun, - evolved = :evolved, - hatched = :hatched, - quests = :quests, - trades = :trades, - photobombs = :photobombs, - purified = :purified, - grunts_defeated = :grunts_defeated, - gym_battles_won = :gym_battles_won, - normal_raids_won = :normal_raids_won, - legendary_raids_won = :legendary_raids_won, - trainings_won = :trainings_won, - berries_fed = :berries_fed, - hours_defended = :hours_defended, - best_friends = :best_friends, - best_buddies = :best_buddies, - giovanni_defeated = :giovanni_defeated, - mega_evos = :mega_evos, - collections_done = :collections_done, - unique_stops_spun = :unique_stops_spun, - unique_mega_evos = :unique_mega_evos, - unique_raid_bosses = :unique_raid_bosses, - unique_unown = :unique_unown, - seven_day_streaks = :seven_day_streaks, - trade_km = :trade_km, - raids_with_friends = :raids_with_friends, - caught_at_lure = :caught_at_lure, - wayfarer_agreements = :wayfarer_agreements, - trainers_referred = :trainers_referred, - raid_achievements = :raid_achievements, - xl_karps = :xl_karps, - xs_rats = :xs_rats, - pikachu_caught = :pikachu_caught, - league_great_won = :league_great_won, - league_ultra_won = :league_ultra_won, - league_master_won = :league_master_won, - tiny_pokemon_caught = :tiny_pokemon_caught, - jumbo_pokemon_caught = :jumbo_pokemon_caught, - vivillon = :vivillon, + friendship_id = :friendship_id, + last_seen = :last_seen, + team = :team, + level = :level, + xp = :xp, + battles_won = :battles_won, + km_walked = :km_walked, + caught_pokemon = :caught_pokemon, + gbl_rank = :gbl_rank, + gbl_rating = :gbl_rating, + event_badges = :event_badges, + stops_spun = :stops_spun, + evolved = :evolved, + hatched = :hatched, + quests = :quests, + trades = :trades, + photobombs = :photobombs, + purified = :purified, + grunts_defeated = :grunts_defeated, + gym_battles_won = :gym_battles_won, + normal_raids_won = :normal_raids_won, + legendary_raids_won = :legendary_raids_won, + trainings_won = :trainings_won, + berries_fed = :berries_fed, + hours_defended = :hours_defended, + best_friends = :best_friends, + best_buddies = :best_buddies, + giovanni_defeated = :giovanni_defeated, + mega_evos = :mega_evos, + collections_done = :collections_done, + unique_stops_spun = :unique_stops_spun, + unique_mega_evos = :unique_mega_evos, + unique_raid_bosses = :unique_raid_bosses, + unique_unown = :unique_unown, + seven_day_streaks = :seven_day_streaks, + trade_km = :trade_km, + raids_with_friends = :raids_with_friends, + caught_at_lure = :caught_at_lure, + wayfarer_agreements = :wayfarer_agreements, + trainers_referred = :trainers_referred, + raid_achievements = :raid_achievements, + xl_karps = :xl_karps, + xs_rats = :xs_rats, + pikachu_caught = :pikachu_caught, + league_great_won = :league_great_won, + league_ultra_won = :league_ultra_won, + league_master_won = :league_master_won, + tiny_pokemon_caught = :tiny_pokemon_caught, + jumbo_pokemon_caught = :jumbo_pokemon_caught, + vivillon = :vivillon, showcase_max_size_first_place = :showcase_max_size_first_place, total_route_play = :total_route_play, parties_completed = :parties_completed, - event_check_ins = :event_check_ins, - dex_gen1 = :dex_gen1, - dex_gen2 = :dex_gen2, - dex_gen3 = :dex_gen3, - dex_gen4 = :dex_gen4, - dex_gen5 = :dex_gen5, - dex_gen6 = :dex_gen6, - dex_gen7 = :dex_gen7, - dex_gen8 = :dex_gen8, - dex_gen8a = :dex_gen8a, + event_check_ins = :event_check_ins, + dex_gen1 = :dex_gen1, + dex_gen2 = :dex_gen2, + dex_gen3 = :dex_gen3, + dex_gen4 = :dex_gen4, + dex_gen5 = :dex_gen5, + dex_gen6 = :dex_gen6, + dex_gen7 = :dex_gen7, + dex_gen8 = :dex_gen8, + dex_gen8a = :dex_gen8a, dex_gen9 = :dex_gen9, - caught_normal = :caught_normal, - caught_fighting = :caught_fighting, - caught_flying = :caught_flying, - caught_poison = :caught_poison, - caught_ground = :caught_ground, - caught_rock = :caught_rock, - caught_bug = :caught_bug, - caught_ghost = :caught_ghost, - caught_steel = :caught_steel, - caught_fire = :caught_fire, - caught_water = :caught_water, - caught_grass = :caught_grass, - caught_electric = :caught_electric, - caught_psychic = :caught_psychic, - caught_ice = :caught_ice, - caught_dragon = :caught_dragon, - caught_dark = :caught_dark, - caught_fairy = :caught_fairy + caught_normal = :caught_normal, + caught_fighting = :caught_fighting, + caught_flying = :caught_flying, + caught_poison = :caught_poison, + caught_ground = :caught_ground, + caught_rock = :caught_rock, + caught_bug = :caught_bug, + caught_ghost = :caught_ghost, + caught_steel = :caught_steel, + caught_fire = :caught_fire, + caught_water = :caught_water, + caught_grass = :caught_grass, + caught_electric = :caught_electric, + caught_psychic = :caught_psychic, + caught_ice = :caught_ice, + caught_dragon = :caught_dragon, + caught_dark = :caught_dark, + caught_fairy = :caught_fairy WHERE name = :name`, player, ) @@ -476,19 +913,21 @@ func savePlayerRecord(db db.DbDetails, player *Player) { } } - playerCache.Set(player.Name, *player, ttlcache.DefaultTTL) + player.ClearDirty() + player.newRecord = false + //playerCache.Set(player.Name, player, ttlcache.DefaultTTL) } func (player *Player) updateFromPublicProfile(publicProfile *pogo.PlayerPublicProfileProto) { - player.Name = publicProfile.GetName() - player.Team = null.IntFrom(int64(publicProfile.GetTeam())) - player.Level = null.IntFrom(int64(publicProfile.GetLevel())) - player.Xp = null.IntFrom(publicProfile.GetExperience()) - player.BattlesWon = null.IntFrom(int64(publicProfile.GetBattlesWon())) - player.KmWalked = null.FloatFrom(float64(publicProfile.GetKmWalked())) - player.CaughtPokemon = null.IntFrom(int64(publicProfile.GetCaughtPokemon())) - player.GblRank = null.IntFrom(int64(publicProfile.GetCombatRank())) - player.GblRating = null.IntFrom(int64(publicProfile.GetCombatRating())) + player.Name = publicProfile.GetName() // Name is primary key, don't track as dirty + player.SetTeam(null.IntFrom(int64(publicProfile.GetTeam()))) + player.SetLevel(null.IntFrom(int64(publicProfile.GetLevel()))) + player.SetXp(null.IntFrom(publicProfile.GetExperience())) + player.SetBattlesWon(null.IntFrom(int64(publicProfile.GetBattlesWon()))) + player.SetKmWalked(null.FloatFrom(float64(publicProfile.GetKmWalked()))) + player.SetCaughtPokemon(null.IntFrom(int64(publicProfile.GetCaughtPokemon()))) + player.SetGblRank(null.IntFrom(int64(publicProfile.GetCombatRank()))) + player.SetGblRating(null.IntFrom(int64(publicProfile.GetCombatRating()))) eventBadges := "" @@ -514,13 +953,15 @@ func (player *Player) updateFromPublicProfile(publicProfile *pogo.PlayerPublicPr field := reflect.ValueOf(player).Elem().FieldByName(playerKey) if field.IsValid() && field.CanSet() { - field.Set(reflect.ValueOf(newValue)) + oldValue := field.Interface().(null.Int) + if oldValue != newValue { + field.Set(reflect.ValueOf(newValue)) + player.setFieldDirty() + } } } - if eventBadges != "" { - player.EventBadges = null.StringFrom(eventBadges) - } + player.SetEventBadges(null.StringFrom(eventBadges)) } func UpdatePlayerRecordWithPlayerSummary(db db.DbDetails, playerSummary *pogo.InternalPlayerSummaryProto, publicProfile *pogo.PlayerPublicProfileProto, friendCode string, friendshipId string) error { @@ -531,15 +972,16 @@ func UpdatePlayerRecordWithPlayerSummary(db db.DbDetails, playerSummary *pogo.In if player == nil { player = &Player{ - Name: playerSummary.GetCodename(), + Name: playerSummary.GetCodename(), + newRecord: true, } } if player.FriendshipId.IsZero() && friendshipId != "" { - player.FriendshipId = null.StringFrom(friendshipId) + player.SetFriendshipId(null.StringFrom(friendshipId)) } if player.FriendCode.IsZero() && friendCode != "" { - player.FriendCode = null.StringFrom(friendCode) + player.SetFriendCode(null.StringFrom(friendCode)) } player.updateFromPublicProfile(publicProfile) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9d60f1f0..d18c6d16 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -265,7 +265,7 @@ func (pokemon *Pokemon) SetSeenType(v null.String) { func (pokemon *Pokemon) SetUsername(v null.String) { if pokemon.Username != v { pokemon.Username = v - pokemon.dirty = true + //pokemon.dirty = true } } @@ -862,6 +862,36 @@ func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) } +// wildSignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and +// should be written. +func (pokemon *Pokemon) nearbySignificantUpdate(wildPokemon *pogo.NearbyPokemonProto, time int64) bool { + pokemonDisplay := wildPokemon.PokemonDisplay + // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired + + pokemonChanged := pokemon.PokemonId != int16(pokemonDisplay.DisplayId) || + pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || + pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || + pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || + pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) + + if pokemonChanged { + return true + } + + hasExpired := (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) + + if hasExpired { + return true + } + + if pokemon.SeenType.ValueOrZero() == SeenType_Cell { + return true + } + + // if it's at a nearby stop, or encounter and no other details have changed update is not worthwhile + return false +} + func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { pokemon.SetIsEvent(0) switch pokemon.SeenType.ValueOrZero() { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 4aa1c736..c42200fc 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1192,7 +1192,7 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } _ = res } - pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) pokestop.newRecord = false // After saving, it's no longer a new record pokestop.ClearDirty() createPokestopWebhooks(pokestop) diff --git a/decoder/routes.go b/decoder/routes.go index 892413ac..eb7565be 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -4,16 +4,19 @@ import ( "database/sql" "encoding/json" "fmt" + "time" + "golbat/db" "golbat/pogo" "golbat/util" - "time" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) +// Route struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Route struct { Id string `db:"id"` Name string `db:"name"` @@ -37,13 +40,173 @@ type Route struct { Updated int64 `db:"updated"` Version int64 `db:"version"` Waypoints string `db:"waypoints"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record +} + +// IsDirty returns true if any field has been modified +func (r *Route) IsDirty() bool { + return r.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (r *Route) ClearDirty() { + r.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (r *Route) IsNewRecord() bool { + return r.newRecord +} + +// --- Set methods with dirty tracking --- + +func (r *Route) SetName(v string) { + if r.Name != v { + r.Name = v + r.dirty = true + } +} + +func (r *Route) SetShortcode(v string) { + if r.Shortcode != v { + r.Shortcode = v + r.dirty = true + } +} + +func (r *Route) SetDescription(v string) { + if r.Description != v { + r.Description = v + r.dirty = true + } +} + +func (r *Route) SetDistanceMeters(v int64) { + if r.DistanceMeters != v { + r.DistanceMeters = v + r.dirty = true + } +} + +func (r *Route) SetDurationSeconds(v int64) { + if r.DurationSeconds != v { + r.DurationSeconds = v + r.dirty = true + } +} + +func (r *Route) SetEndFortId(v string) { + if r.EndFortId != v { + r.EndFortId = v + r.dirty = true + } +} + +func (r *Route) SetEndImage(v string) { + if r.EndImage != v { + r.EndImage = v + r.dirty = true + } +} + +func (r *Route) SetEndLat(v float64) { + if !floatAlmostEqual(r.EndLat, v, floatTolerance) { + r.EndLat = v + r.dirty = true + } +} + +func (r *Route) SetEndLon(v float64) { + if !floatAlmostEqual(r.EndLon, v, floatTolerance) { + r.EndLon = v + r.dirty = true + } +} + +func (r *Route) SetImage(v string) { + if r.Image != v { + r.Image = v + r.dirty = true + } +} + +func (r *Route) SetImageBorderColor(v string) { + if r.ImageBorderColor != v { + r.ImageBorderColor = v + r.dirty = true + } +} + +func (r *Route) SetReversible(v bool) { + if r.Reversible != v { + r.Reversible = v + r.dirty = true + } +} + +func (r *Route) SetStartFortId(v string) { + if r.StartFortId != v { + r.StartFortId = v + r.dirty = true + } +} + +func (r *Route) SetStartImage(v string) { + if r.StartImage != v { + r.StartImage = v + r.dirty = true + } +} + +func (r *Route) SetStartLat(v float64) { + if !floatAlmostEqual(r.StartLat, v, floatTolerance) { + r.StartLat = v + r.dirty = true + } +} + +func (r *Route) SetStartLon(v float64) { + if !floatAlmostEqual(r.StartLon, v, floatTolerance) { + r.StartLon = v + r.dirty = true + } +} + +func (r *Route) SetTags(v null.String) { + if r.Tags != v { + r.Tags = v + r.dirty = true + } +} + +func (r *Route) SetType(v int8) { + if r.Type != v { + r.Type = v + r.dirty = true + } +} + +func (r *Route) SetVersion(v int64) { + if r.Version != v { + r.Version = v + r.dirty = true + } +} + +func (r *Route) SetWaypoints(v string) { + if r.Waypoints != v { + r.Waypoints = v + r.dirty = true + } } func getRouteRecord(db db.DbDetails, id string) (*Route, error) { inMemoryRoute := routeCache.Get(id) if inMemoryRoute != nil { route := inMemoryRoute.Value() - return &route, nil + return route, nil } route := Route{} @@ -62,61 +225,40 @@ func getRouteRecord(db db.DbDetails, id string) (*Route, error) { return nil, err } - routeCache.Set(id, route, ttlcache.DefaultTTL) + routeCache.Set(id, &route, ttlcache.DefaultTTL) return &route, nil } -// hasChangesRoute compares two Route structs -func hasChangesRoute(old *Route, new *Route) bool { - return old.Name != new.Name || - old.Shortcode != new.Shortcode || - old.Description != new.Description || - old.DistanceMeters != new.DistanceMeters || - old.DurationSeconds != new.DurationSeconds || - old.EndFortId != new.EndFortId || - !floatAlmostEqual(old.EndLat, new.EndLat, floatTolerance) || - !floatAlmostEqual(old.EndLon, new.EndLon, floatTolerance) || - old.Image != new.Image || - old.ImageBorderColor != new.ImageBorderColor || - old.Reversible != new.Reversible || - old.StartFortId != new.StartFortId || - !floatAlmostEqual(old.StartLat, new.StartLat, floatTolerance) || - !floatAlmostEqual(old.StartLon, new.StartLon, floatTolerance) || - old.Tags != new.Tags || - old.Type != new.Type || - old.Version != new.Version || - old.Waypoints != new.Waypoints -} - func saveRouteRecord(db db.DbDetails, route *Route) error { - oldRoute, _ := getRouteRecord(db, route.Id) - - if oldRoute != nil && !hasChangesRoute(oldRoute, route) { - if oldRoute.Updated > time.Now().Unix()-900 { + // Skip save if not dirty and not new, unless 15-minute debounce expired + if !route.IsDirty() && !route.IsNewRecord() { + if route.Updated > time.Now().Unix()-900 { // if a route is unchanged, but we did see it again after 15 minutes, then save again return nil } } - if oldRoute == nil { + route.Updated = time.Now().Unix() + + if route.IsNewRecord() { _, err := db.GeneralDb.NamedExec( ` INSERT INTO route ( id, name, shortcode, description, distance_meters, duration_seconds, end_fort_id, end_image, - end_lat, end_lon, image, image_border_color, - reversible, start_fort_id, start_image, - start_lat, start_lon, tags, type, + end_lat, end_lon, image, image_border_color, + reversible, start_fort_id, start_image, + start_lat, start_lon, tags, type, updated, version, waypoints ) VALUES ( :id, :name, :shortcode, :description, :distance_meters, :duration_seconds, :end_fort_id, - :end_image, :end_lat, :end_lon, :image, - :image_border_color, :reversible, - :start_fort_id, :start_image, :start_lat, - :start_lon, :tags, :type, :updated, + :end_image, :end_lat, :end_lon, :image, + :image_border_color, :reversible, + :start_fort_id, :start_image, :start_lat, + :start_lon, :tags, :type, :updated, :version, :waypoints ) `, @@ -162,47 +304,49 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { } } - routeCache.Set(route.Id, *route, ttlcache.DefaultTTL) + route.ClearDirty() + route.newRecord = false + //routeCache.Set(route.Id, route, ttlcache.DefaultTTL) return nil } func (route *Route) updateFromSharedRouteProto(sharedRouteProto *pogo.SharedRouteProto) { - route.Name = sharedRouteProto.GetName() + route.SetName(sharedRouteProto.GetName()) if sharedRouteProto.GetShortCode() != "" { - route.Shortcode = sharedRouteProto.GetShortCode() + route.SetShortcode(sharedRouteProto.GetShortCode()) } - route.Description = sharedRouteProto.GetDescription() + description := sharedRouteProto.GetDescription() // NOTE: Some descriptions have more than 255 runes, which won't fit in our // varchar(255). - if truncateStr, truncated := util.TruncateUTF8(route.Description, 255); truncated { + if truncateStr, truncated := util.TruncateUTF8(description, 255); truncated { log.Warnf("truncating description for route id '%s'. Orig description: %s", route.Id, - route.Description, + description, ) - route.Description = truncateStr - } - route.DistanceMeters = sharedRouteProto.GetRouteDistanceMeters() - route.DurationSeconds = sharedRouteProto.GetRouteDurationSeconds() - route.EndFortId = sharedRouteProto.GetEndPoi().GetAnchor().GetFortId() - route.EndImage = sharedRouteProto.GetEndPoi().GetImageUrl() - route.EndLat = sharedRouteProto.GetEndPoi().GetAnchor().GetLatDegrees() - route.EndLon = sharedRouteProto.GetEndPoi().GetAnchor().GetLngDegrees() - route.Image = sharedRouteProto.GetImage().GetImageUrl() - route.ImageBorderColor = sharedRouteProto.GetImage().GetBorderColorHex() - route.Reversible = sharedRouteProto.GetReversible() - route.StartFortId = sharedRouteProto.GetStartPoi().GetAnchor().GetFortId() - route.StartImage = sharedRouteProto.GetStartPoi().GetImageUrl() - route.StartLat = sharedRouteProto.GetStartPoi().GetAnchor().GetLatDegrees() - route.StartLon = sharedRouteProto.GetStartPoi().GetAnchor().GetLngDegrees() - route.Type = int8(sharedRouteProto.GetType()) - route.Updated = time.Now().Unix() - route.Version = sharedRouteProto.GetVersion() + description = truncateStr + } + route.SetDescription(description) + route.SetDistanceMeters(sharedRouteProto.GetRouteDistanceMeters()) + route.SetDurationSeconds(sharedRouteProto.GetRouteDurationSeconds()) + route.SetEndFortId(sharedRouteProto.GetEndPoi().GetAnchor().GetFortId()) + route.SetEndImage(sharedRouteProto.GetEndPoi().GetImageUrl()) + route.SetEndLat(sharedRouteProto.GetEndPoi().GetAnchor().GetLatDegrees()) + route.SetEndLon(sharedRouteProto.GetEndPoi().GetAnchor().GetLngDegrees()) + route.SetImage(sharedRouteProto.GetImage().GetImageUrl()) + route.SetImageBorderColor(sharedRouteProto.GetImage().GetBorderColorHex()) + route.SetReversible(sharedRouteProto.GetReversible()) + route.SetStartFortId(sharedRouteProto.GetStartPoi().GetAnchor().GetFortId()) + route.SetStartImage(sharedRouteProto.GetStartPoi().GetImageUrl()) + route.SetStartLat(sharedRouteProto.GetStartPoi().GetAnchor().GetLatDegrees()) + route.SetStartLon(sharedRouteProto.GetStartPoi().GetAnchor().GetLngDegrees()) + route.SetType(int8(sharedRouteProto.GetType())) + route.SetVersion(sharedRouteProto.GetVersion()) waypoints, _ := json.Marshal(sharedRouteProto.GetWaypoints()) - route.Waypoints = string(waypoints) + route.SetWaypoints(string(waypoints)) if len(sharedRouteProto.GetTags()) > 0 { tags, _ := json.Marshal(sharedRouteProto.GetTags()) - route.Tags = null.StringFrom(string(tags)) + route.SetTags(null.StringFrom(string(tags))) } } @@ -218,7 +362,8 @@ func UpdateRouteRecordWithSharedRouteProto(db db.DbDetails, sharedRouteProto *po if route == nil { route = &Route{ - Id: sharedRouteProto.GetId(), + Id: sharedRouteProto.GetId(), + newRecord: true, } } diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 506eb952..07c9c311 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -2,11 +2,11 @@ package decoder import ( "context" - "golbat/db" "time" + "golbat/db" + "github.com/golang/geo/s2" - "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -30,11 +30,11 @@ type S2Cell struct { func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { now := time.Now().Unix() - outputCellIds := []S2Cell{} + var outputCellIds []*S2Cell // prepare list of cells to update for _, cellId := range cellIds { - var s2Cell = S2Cell{} + var s2Cell *S2Cell if c := s2CellCache.Get(cellId); c != nil { cachedCell := c.Value() @@ -44,6 +44,7 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { s2Cell = cachedCell } else { mapS2Cell := s2.CellFromCellID(s2.CellID(cellId)) + s2Cell = &S2Cell{} s2Cell.Id = cellId s2Cell.Latitude = mapS2Cell.CapBound().RectBound().Center().Lat.Degrees() s2Cell.Longitude = mapS2Cell.CapBound().RectBound().Center().Lng.Degrees() @@ -71,8 +72,8 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { return } - // set cache - for _, cellId := range outputCellIds { - s2CellCache.Set(cellId.Id, cellId, ttlcache.DefaultTTL) - } + // since cache is now a pointer, ttl will already have been updated + //for _, cellId := range outputCellIds { + // s2CellCache.Set(cellId.Id, cellId, ttlcache.DefaultTTL) + //} } diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 5867daa1..e4a70fcb 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -14,7 +14,7 @@ import ( ) // Spawnpoint struct. -// REMINDER! Keep hasChangesSpawnpoint updated after making changes +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Spawnpoint struct { Id int64 `db:"id"` Lat float64 `db:"lat"` @@ -22,6 +22,9 @@ type Spawnpoint struct { Updated int64 `db:"updated"` LastSeen int64 `db:"last_seen"` DespawnSec null.Int `db:"despawn_sec"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record } //CREATE TABLE `spawnpoint` ( @@ -37,11 +40,75 @@ type Spawnpoint struct { //KEY `ix_last_seen` (`last_seen`) //) +// IsDirty returns true if any field has been modified +func (s *Spawnpoint) IsDirty() bool { + return s.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (s *Spawnpoint) ClearDirty() { + s.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (s *Spawnpoint) IsNewRecord() bool { + return s.newRecord +} + +// --- Set methods with dirty tracking --- + +func (s *Spawnpoint) SetLat(v float64) { + if !floatAlmostEqual(s.Lat, v, floatTolerance) { + s.Lat = v + s.dirty = true + } +} + +func (s *Spawnpoint) SetLon(v float64) { + if !floatAlmostEqual(s.Lon, v, floatTolerance) { + s.Lon = v + s.dirty = true + } +} + +// SetDespawnSec sets despawn_sec with 2-second tolerance logic +func (s *Spawnpoint) SetDespawnSec(v null.Int) { + // Handle validity changes + if (s.DespawnSec.Valid && !v.Valid) || (!s.DespawnSec.Valid && v.Valid) { + s.DespawnSec = v + s.dirty = true + return + } + + // Both invalid - no change + if !s.DespawnSec.Valid && !v.Valid { + return + } + + // Both valid - check with tolerance + oldVal := s.DespawnSec.Int64 + newVal := v.Int64 + + // Handle wraparound at hour boundary (0/3600) + if oldVal <= 1 && newVal >= 3598 { + return + } + if newVal <= 1 && oldVal >= 3598 { + return + } + + // Allow 2-second tolerance for despawn time + if Abs(oldVal-newVal) > 2 { + s.DespawnSec = v + s.dirty = true + } +} + func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, error) { inMemorySpawnpoint := spawnpointCache.Get(spawnpointId) if inMemorySpawnpoint != nil { spawnpoint := inMemorySpawnpoint.Value() - return &spawnpoint, nil + return spawnpoint, nil } spawnpoint := Spawnpoint{} @@ -56,7 +123,6 @@ func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int6 return &Spawnpoint{Id: spawnpointId}, err } - spawnpointCache.Set(spawnpointId, spawnpoint, ttlcache.DefaultTTL) return &spawnpoint, nil } @@ -67,31 +133,6 @@ func Abs(x int64) int64 { return x } -func hasChangesSpawnpoint(old *Spawnpoint, new *Spawnpoint) bool { - if !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) || - (old.DespawnSec.Valid && !new.DespawnSec.Valid) || - (!old.DespawnSec.Valid && new.DespawnSec.Valid) { - return true - } - if !old.DespawnSec.Valid && !new.DespawnSec.Valid { - return false - } - - // Ignore small movements in despawn time - oldDespawnSec := old.DespawnSec.Int64 - newDespawnSec := new.DespawnSec.Int64 - - if oldDespawnSec <= 1 && newDespawnSec >= 3598 { - return false - } - if newDespawnSec <= 1 && oldDespawnSec >= 3598 { - return false - } - - return Abs(old.DespawnSec.Int64-new.DespawnSec.Int64) > 2 -} - func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, timestampMs int64) { spawnId, err := strconv.ParseInt(wildPokemon.SpawnPointId, 16, 64) if err != nil { @@ -103,22 +144,25 @@ func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon date := time.Unix(expireTimeStamp, 0) secondOfHour := date.Second() + date.Minute()*60 - spawnpoint := Spawnpoint{ - Id: spawnId, - Lat: wildPokemon.Latitude, - Lon: wildPokemon.Longitude, - DespawnSec: null.IntFrom(int64(secondOfHour)), + + spawnpoint, _ := getSpawnpointRecord(ctx, db, spawnId) + if spawnpoint == nil { + spawnpoint = &Spawnpoint{Id: spawnId, newRecord: true} } - spawnpointUpdate(ctx, db, &spawnpoint) + spawnpoint.SetLat(wildPokemon.Latitude) + spawnpoint.SetLon(wildPokemon.Longitude) + spawnpoint.SetDespawnSec(null.IntFrom(int64(secondOfHour))) + spawnpointUpdate(ctx, db, spawnpoint) } else { spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) if spawnPoint == nil { - spawnpoint := Spawnpoint{ - Id: spawnId, - Lat: wildPokemon.Latitude, - Lon: wildPokemon.Longitude, + spawnpoint := &Spawnpoint{ + Id: spawnId, + Lat: wildPokemon.Latitude, + Lon: wildPokemon.Longitude, + newRecord: true, } - spawnpointUpdate(ctx, db, &spawnpoint) + spawnpointUpdate(ctx, db, spawnpoint) } else { spawnpointSeen(ctx, db, spawnId) } @@ -126,14 +170,11 @@ func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon } func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoint) { - oldSpawnpoint, _ := getSpawnpointRecord(ctx, db, spawnpoint.Id) - - if oldSpawnpoint != nil && !hasChangesSpawnpoint(oldSpawnpoint, spawnpoint) { + // Skip save if not dirty and not new + if !spawnpoint.IsDirty() && !spawnpoint.IsNewRecord() { return } - //log.Println(cmp.Diff(oldSpawnpoint, spawnpoint)) - spawnpoint.Updated = time.Now().Unix() // ensure future updates are set correctly spawnpoint.LastSeen = time.Now().Unix() // ensure future updates are set correctly @@ -152,7 +193,9 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi return } - spawnpointCache.Set(spawnpoint.Id, *spawnpoint, ttlcache.DefaultTTL) + spawnpoint.ClearDirty() + spawnpoint.newRecord = false + spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) } func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { @@ -176,6 +219,6 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { log.Printf("Error updating spawnpoint last seen %s", err) return } - spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) + // Cache already contains a pointer, no need to update } } diff --git a/decoder/station.go b/decoder/station.go index f02b46f1..ccbaef43 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -6,17 +6,19 @@ import ( "encoding/json" "errors" "fmt" + "time" + "golbat/db" "golbat/pogo" "golbat/util" "golbat/webhooks" - "time" - "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) +// Station struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Station struct { Id string `db:"id"` Lat float64 `db:"lat"` @@ -47,6 +49,235 @@ type Station struct { TotalStationedPokemon null.Int `db:"total_stationed_pokemon"` TotalStationedGmax null.Int `db:"total_stationed_gmax"` StationedPokemon null.String `db:"stationed_pokemon"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + oldValues StationOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// StationOldValues holds old field values for webhook comparison +type StationOldValues struct { + EndTime int64 + BattleEnd null.Int + BattlePokemonId null.Int + BattlePokemonForm null.Int + BattlePokemonCostume null.Int + BattlePokemonGender null.Int + BattlePokemonBreadMode null.Int +} + +// IsDirty returns true if any field has been modified +func (station *Station) IsDirty() bool { + return station.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (station *Station) ClearDirty() { + station.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (station *Station) IsNewRecord() bool { + return station.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (station *Station) snapshotOldValues() { + station.oldValues = StationOldValues{ + EndTime: station.EndTime, + BattleEnd: station.BattleEnd, + BattlePokemonId: station.BattlePokemonId, + BattlePokemonForm: station.BattlePokemonForm, + BattlePokemonCostume: station.BattlePokemonCostume, + BattlePokemonGender: station.BattlePokemonGender, + BattlePokemonBreadMode: station.BattlePokemonBreadMode, + } +} + +// --- Set methods with dirty tracking --- + +func (station *Station) SetId(v string) { + if station.Id != v { + station.Id = v + station.dirty = true + } +} + +func (station *Station) SetLat(v float64) { + if !floatAlmostEqual(station.Lat, v, floatTolerance) { + station.Lat = v + station.dirty = true + } +} + +func (station *Station) SetLon(v float64) { + if !floatAlmostEqual(station.Lon, v, floatTolerance) { + station.Lon = v + station.dirty = true + } +} + +func (station *Station) SetName(v string) { + if station.Name != v { + station.Name = v + station.dirty = true + } +} + +func (station *Station) SetCellId(v int64) { + if station.CellId != v { + station.CellId = v + station.dirty = true + } +} + +func (station *Station) SetStartTime(v int64) { + if station.StartTime != v { + station.StartTime = v + station.dirty = true + } +} + +func (station *Station) SetEndTime(v int64) { + if station.EndTime != v { + station.EndTime = v + station.dirty = true + } +} + +func (station *Station) SetCooldownComplete(v int64) { + if station.CooldownComplete != v { + station.CooldownComplete = v + station.dirty = true + } +} + +func (station *Station) SetIsBattleAvailable(v bool) { + if station.IsBattleAvailable != v { + station.IsBattleAvailable = v + station.dirty = true + } +} + +func (station *Station) SetIsInactive(v bool) { + if station.IsInactive != v { + station.IsInactive = v + station.dirty = true + } +} + +func (station *Station) SetBattleLevel(v null.Int) { + if station.BattleLevel != v { + station.BattleLevel = v + station.dirty = true + } +} + +func (station *Station) SetBattleStart(v null.Int) { + if station.BattleStart != v { + station.BattleStart = v + station.dirty = true + } +} + +func (station *Station) SetBattleEnd(v null.Int) { + if station.BattleEnd != v { + station.BattleEnd = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonId(v null.Int) { + if station.BattlePokemonId != v { + station.BattlePokemonId = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonForm(v null.Int) { + if station.BattlePokemonForm != v { + station.BattlePokemonForm = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonCostume(v null.Int) { + if station.BattlePokemonCostume != v { + station.BattlePokemonCostume = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonGender(v null.Int) { + if station.BattlePokemonGender != v { + station.BattlePokemonGender = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonAlignment(v null.Int) { + if station.BattlePokemonAlignment != v { + station.BattlePokemonAlignment = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonBreadMode(v null.Int) { + if station.BattlePokemonBreadMode != v { + station.BattlePokemonBreadMode = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonMove1(v null.Int) { + if station.BattlePokemonMove1 != v { + station.BattlePokemonMove1 = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonMove2(v null.Int) { + if station.BattlePokemonMove2 != v { + station.BattlePokemonMove2 = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonStamina(v null.Int) { + if station.BattlePokemonStamina != v { + station.BattlePokemonStamina = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { + if !nullFloatAlmostEqual(station.BattlePokemonCpMultiplier, v, floatTolerance) { + station.BattlePokemonCpMultiplier = v + station.dirty = true + } +} + +func (station *Station) SetTotalStationedPokemon(v null.Int) { + if station.TotalStationedPokemon != v { + station.TotalStationedPokemon = v + station.dirty = true + } +} + +func (station *Station) SetTotalStationedGmax(v null.Int) { + if station.TotalStationedGmax != v { + station.TotalStationedGmax = v + station.dirty = true + } +} + +func (station *Station) SetStationedPokemon(v null.String) { + if station.StationedPokemon != v { + station.StationedPokemon = v + station.dirty = true + } } type StationWebhook struct { @@ -77,7 +308,8 @@ func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (* inMemoryStation := stationCache.Get(stationId) if inMemoryStation != nil { station := inMemoryStation.Value() - return &station, nil + station.snapshotOldValues() + return station, nil } station := Station{} err := db.GeneralDb.GetContext(ctx, &station, @@ -94,23 +326,23 @@ func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (* if err != nil { return nil, err } + station.snapshotOldValues() return &station, nil } func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { - oldStation, _ := getStationRecord(ctx, db, station.Id) now := time.Now().Unix() - if oldStation != nil && !hasChangesStation(oldStation, station) { - if oldStation.Updated > now-900 { - // if a gym is unchanged, but we did see it again after 15 minutes, then save again + + // Skip save if not dirty and was updated recently (15-min debounce) + if !station.IsDirty() && !station.IsNewRecord() { + if station.Updated > now-900 { return } } station.Updated = now - //log.Traceln(cmp.Diff(oldStation, station)) - if oldStation == nil { + if station.IsNewRecord() { res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) @@ -163,41 +395,15 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { _, _ = res, err } - stationCache.Set(station.Id, *station, ttlcache.DefaultTTL) - createStationWebhooks(oldStation, station) - -} - -// hasChangesStation compares two Station structs -// Float tolerance: Lat, Lon -func hasChangesStation(old *Station, new *Station) bool { - return old.Id != new.Id || - old.Name != new.Name || - old.StartTime != new.StartTime || - old.EndTime != new.EndTime || - old.StationedPokemon != new.StationedPokemon || - old.CooldownComplete != new.CooldownComplete || - old.IsBattleAvailable != new.IsBattleAvailable || - old.BattleLevel != new.BattleLevel || - old.BattleStart != new.BattleStart || - old.BattleEnd != new.BattleEnd || - old.BattlePokemonId != new.BattlePokemonId || - old.BattlePokemonForm != new.BattlePokemonForm || - old.BattlePokemonCostume != new.BattlePokemonCostume || - old.BattlePokemonGender != new.BattlePokemonGender || - old.BattlePokemonAlignment != new.BattlePokemonAlignment || - old.BattlePokemonBreadMode != new.BattlePokemonBreadMode || - old.BattlePokemonMove1 != new.BattlePokemonMove1 || - old.BattlePokemonMove2 != new.BattlePokemonMove2 || - old.BattlePokemonStamina != new.BattlePokemonStamina || - old.BattlePokemonCpMultiplier != new.BattlePokemonCpMultiplier || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) + station.ClearDirty() + station.newRecord = false + //stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + createStationWebhooks(station) } func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, cellId uint64) *Station { - station.Id = stationProto.Id - station.Name = stationProto.Name + station.SetId(stationProto.Id) + name := stationProto.Name // NOTE: Some names have more than 255 runes, which won't fit in our // varchar(255). if truncateStr, truncated := util.TruncateUTF8(stationProto.Name, 255); truncated { @@ -205,35 +411,36 @@ func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, stationProto.Id, stationProto.Name, ) - station.Name = truncateStr - } - station.Lat = stationProto.Lat - station.Lon = stationProto.Lng - station.StartTime = stationProto.StartTimeMs / 1000 - station.EndTime = stationProto.EndTimeMs / 1000 - station.CooldownComplete = stationProto.CooldownCompleteMs - station.IsBattleAvailable = stationProto.IsBreadBattleAvailable + name = truncateStr + } + station.SetName(name) + station.SetLat(stationProto.Lat) + station.SetLon(stationProto.Lng) + station.SetStartTime(stationProto.StartTimeMs / 1000) + station.SetEndTime(stationProto.EndTimeMs / 1000) + station.SetCooldownComplete(stationProto.CooldownCompleteMs) + station.SetIsBattleAvailable(stationProto.IsBreadBattleAvailable) if battleDetails := stationProto.BattleDetails; battleDetails != nil { - station.BattleLevel = null.IntFrom(int64(battleDetails.BattleLevel)) - station.BattleStart = null.IntFrom(battleDetails.BattleWindowStartMs / 1000) - station.BattleEnd = null.IntFrom(battleDetails.BattleWindowEndMs / 1000) + station.SetBattleLevel(null.IntFrom(int64(battleDetails.BattleLevel))) + station.SetBattleStart(null.IntFrom(battleDetails.BattleWindowStartMs / 1000)) + station.SetBattleEnd(null.IntFrom(battleDetails.BattleWindowEndMs / 1000)) if pokemon := battleDetails.BattlePokemon; pokemon != nil { - station.BattlePokemonId = null.IntFrom(int64(pokemon.PokemonId)) - station.BattlePokemonMove1 = null.IntFrom(int64(pokemon.Move1)) - station.BattlePokemonMove2 = null.IntFrom(int64(pokemon.Move2)) - station.BattlePokemonForm = null.IntFrom(int64(pokemon.PokemonDisplay.Form)) - station.BattlePokemonCostume = null.IntFrom(int64(pokemon.PokemonDisplay.Costume)) - station.BattlePokemonGender = null.IntFrom(int64(pokemon.PokemonDisplay.Gender)) - station.BattlePokemonAlignment = null.IntFrom(int64(pokemon.PokemonDisplay.Alignment)) - station.BattlePokemonBreadMode = null.IntFrom(int64(pokemon.PokemonDisplay.BreadModeEnum)) - station.BattlePokemonStamina = null.IntFrom(int64(pokemon.Stamina)) - station.BattlePokemonCpMultiplier = null.FloatFrom(float64(pokemon.CpMultiplier)) + station.SetBattlePokemonId(null.IntFrom(int64(pokemon.PokemonId))) + station.SetBattlePokemonMove1(null.IntFrom(int64(pokemon.Move1))) + station.SetBattlePokemonMove2(null.IntFrom(int64(pokemon.Move2))) + station.SetBattlePokemonForm(null.IntFrom(int64(pokemon.PokemonDisplay.Form))) + station.SetBattlePokemonCostume(null.IntFrom(int64(pokemon.PokemonDisplay.Costume))) + station.SetBattlePokemonGender(null.IntFrom(int64(pokemon.PokemonDisplay.Gender))) + station.SetBattlePokemonAlignment(null.IntFrom(int64(pokemon.PokemonDisplay.Alignment))) + station.SetBattlePokemonBreadMode(null.IntFrom(int64(pokemon.PokemonDisplay.BreadModeEnum))) + station.SetBattlePokemonStamina(null.IntFrom(int64(pokemon.Stamina))) + station.SetBattlePokemonCpMultiplier(null.FloatFrom(float64(pokemon.CpMultiplier))) if rewardPokemon := battleDetails.RewardPokemon; rewardPokemon != nil && pokemon.PokemonId != rewardPokemon.PokemonId { log.Infof("[DYNAMAX] Pokemon reward differs from battle: Battle %v - Reward %v", pokemon, rewardPokemon) } } } - station.CellId = int64(cellId) + station.SetCellId(int64(cellId)) return station } @@ -275,17 +482,17 @@ func (station *Station) updateFromGetStationedPokemonDetailsOutProto(stationProt } } jsonString, _ := json.Marshal(stationedPokemon) - station.StationedPokemon = null.StringFrom(string(jsonString)) - station.TotalStationedPokemon = null.IntFrom(int64(stationProto.TotalNumStationedPokemon)) - station.TotalStationedGmax = null.IntFrom(stationedGmax) + station.SetStationedPokemon(null.StringFrom(string(jsonString))) + station.SetTotalStationedPokemon(null.IntFrom(int64(stationProto.TotalNumStationedPokemon))) + station.SetTotalStationedGmax(null.IntFrom(stationedGmax)) return station } func (station *Station) resetStationedPokemonFromStationDetailsNotFound() *Station { jsonString, _ := json.Marshal([]string{}) - station.StationedPokemon = null.StringFrom(string(jsonString)) - station.TotalStationedPokemon = null.IntFrom(0) - station.TotalStationedGmax = null.IntFrom(0) + station.SetStationedPokemon(null.StringFrom(string(jsonString))) + station.SetTotalStationedPokemon(null.IntFrom(0)) + station.SetTotalStationedGmax(null.IntFrom(0)) return station } @@ -333,14 +540,17 @@ func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, reque return fmt.Sprintf("StationedPokemonDetails %s", stationId) } -func createStationWebhooks(oldStation *Station, station *Station) { - if oldStation == nil || station.BattlePokemonId.Valid && (oldStation.EndTime != station.EndTime || - oldStation.BattleEnd != station.BattleEnd || - oldStation.BattlePokemonId != station.BattlePokemonId || - oldStation.BattlePokemonForm != station.BattlePokemonForm || - oldStation.BattlePokemonCostume != station.BattlePokemonCostume || - oldStation.BattlePokemonGender != station.BattlePokemonGender || - oldStation.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { +func createStationWebhooks(station *Station) { + old := &station.oldValues + isNew := station.IsNewRecord() + + if isNew || station.BattlePokemonId.Valid && (old.EndTime != station.EndTime || + old.BattleEnd != station.BattleEnd || + old.BattlePokemonId != station.BattlePokemonId || + old.BattlePokemonForm != station.BattlePokemonForm || + old.BattlePokemonCostume != station.BattlePokemonCostume || + old.BattlePokemonGender != station.BattlePokemonGender || + old.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { stationHook := StationWebhook{ Id: station.Id, Latitude: station.Lat, diff --git a/decoder/stats.go b/decoder/stats.go index ee7c1d36..d5312bf9 100644 --- a/decoder/stats.go +++ b/decoder/stats.go @@ -485,7 +485,7 @@ func updateRaidStats(gym *Gym, areas []geo.AreaName) { } } -func updateIncidentStats(old *Incident, new *Incident, areas []geo.AreaName) { +func updateIncidentStats(incident *Incident, areas []geo.AreaName) { if len(areas) == 0 { areas = []geo.AreaName{ { @@ -501,13 +501,15 @@ func updateIncidentStats(old *Incident, new *Incident, areas []geo.AreaName) { }) locked := false + old := &incident.oldValues + isNew := incident.IsNewRecord() // Loop though all areas for i := 0; i < len(areas); i++ { area := areas[i] // Check if StartTime has changed, then we can assume a new Incident has appeared. - if old == nil || old.StartTime != new.StartTime { + if isNew || old.StartTime != incident.StartTime { if !locked { incidentStatsLock.Lock() @@ -521,8 +523,8 @@ func updateIncidentStats(old *Incident, new *Incident, areas []geo.AreaName) { } // Exclude Kecleon, Showcases and other UNSET characters for invasionStats. - if new.Character != 0 { - invasionStats.count[new.Character]++ + if incident.Character != 0 { + invasionStats.count[incident.Character]++ } } } diff --git a/decoder/tappable.go b/decoder/tappable.go index bbca89a4..95c8da4b 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -5,16 +5,18 @@ import ( "database/sql" "errors" "fmt" - "golbat/db" - "golbat/pogo" "strconv" "time" - "github.com/jellydator/ttlcache/v3" + "golbat/db" + "golbat/pogo" + log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) +// Tappable struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Tappable struct { Id uint64 `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` @@ -28,39 +30,129 @@ type Tappable struct { ExpireTimestamp null.Int `db:"expire_timestamp" json:"expire_timestamp"` ExpireTimestampVerified bool `db:"expire_timestamp_verified" json:"expire_timestamp_verified"` Updated int64 `db:"updated" json:"updated"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record +} + +// IsDirty returns true if any field has been modified +func (ta *Tappable) IsDirty() bool { + return ta.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (ta *Tappable) ClearDirty() { + ta.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (ta *Tappable) IsNewRecord() bool { + return ta.newRecord +} + +// --- Set methods with dirty tracking --- + +func (ta *Tappable) SetLat(v float64) { + if !floatAlmostEqual(ta.Lat, v, floatTolerance) { + ta.Lat = v + ta.dirty = true + } +} + +func (ta *Tappable) SetLon(v float64) { + if !floatAlmostEqual(ta.Lon, v, floatTolerance) { + ta.Lon = v + ta.dirty = true + } +} + +func (ta *Tappable) SetFortId(v null.String) { + if ta.FortId != v { + ta.FortId = v + ta.dirty = true + } +} + +func (ta *Tappable) SetSpawnId(v null.Int) { + if ta.SpawnId != v { + ta.SpawnId = v + ta.dirty = true + } +} + +func (ta *Tappable) SetType(v string) { + if ta.Type != v { + ta.Type = v + ta.dirty = true + } +} + +func (ta *Tappable) SetEncounter(v null.Int) { + if ta.Encounter != v { + ta.Encounter = v + ta.dirty = true + } +} + +func (ta *Tappable) SetItemId(v null.Int) { + if ta.ItemId != v { + ta.ItemId = v + ta.dirty = true + } +} + +func (ta *Tappable) SetCount(v null.Int) { + if ta.Count != v { + ta.Count = v + ta.dirty = true + } +} + +func (ta *Tappable) SetExpireTimestamp(v null.Int) { + if ta.ExpireTimestamp != v { + ta.ExpireTimestamp = v + ta.dirty = true + } +} + +func (ta *Tappable) SetExpireTimestampVerified(v bool) { + if ta.ExpireTimestampVerified != v { + ta.ExpireTimestampVerified = v + ta.dirty = true + } } func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.DbDetails, tappable *pogo.ProcessTappableOutProto, request *pogo.ProcessTappableProto, timestampMs int64) { // update from request - ta.Id = request.EncounterId + ta.Id = request.EncounterId // Id is primary key, don't track as dirty location := request.GetLocation() if spawnPointId := location.GetSpawnpointId(); spawnPointId != "" { spawnId, err := strconv.ParseInt(spawnPointId, 16, 64) if err != nil { panic(err) } - ta.SpawnId = null.IntFrom(spawnId) + ta.SetSpawnId(null.IntFrom(spawnId)) } if fortId := location.GetFortId(); fortId != "" { - ta.FortId = null.StringFrom(fortId) + ta.SetFortId(null.StringFrom(fortId)) } - ta.Type = request.TappableTypeId - ta.Lat = request.LocationHintLat - ta.Lon = request.LocationHintLng + ta.SetType(request.TappableTypeId) + ta.SetLat(request.LocationHintLat) + ta.SetLon(request.LocationHintLng) ta.setExpireTimestamp(ctx, db, timestampMs) // update from tappable if encounter := tappable.GetEncounter(); encounter != nil { // tappable is a Pokèmon, encounter is sent in a separate proto // we store this to link tappable with Pokèmon from encounter proto - ta.Encounter = null.IntFrom(int64(encounter.Pokemon.PokemonId)) + ta.SetEncounter(null.IntFrom(int64(encounter.Pokemon.PokemonId))) } else if reward := tappable.GetReward(); reward != nil { for _, lootProto := range reward { for _, itemProto := range lootProto.GetLootItem() { switch t := itemProto.Type.(type) { case *pogo.LootItemProto_Item: - ta.ItemId = null.IntFrom(int64(t.Item)) - ta.Count = null.IntFrom(int64(itemProto.Count)) + ta.SetItemId(null.IntFrom(int64(t.Item))) + ta.SetCount(null.IntFrom(int64(itemProto.Count))) case *pogo.LootItemProto_Stardust: log.Warnf("[TAPPABLE] Reward is Stardust: %t", t.Stardust) case *pogo.LootItemProto_Pokecoin: @@ -96,7 +188,7 @@ func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.Db } func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, timestampMs int64) { - ta.ExpireTimestampVerified = false + ta.SetExpireTimestampVerified(false) if spawnId := ta.SpawnId.ValueOrZero(); spawnId != 0 { spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) if spawnPoint != nil && spawnPoint.DespawnSec.Valid { @@ -109,23 +201,23 @@ func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, tim if despawnOffset < 0 { despawnOffset += 3600 } - ta.ExpireTimestamp = null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset)) - ta.ExpireTimestampVerified = true + ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) + ta.SetExpireTimestampVerified(true) } else { ta.setUnknownTimestamp(timestampMs / 1000) } } else if fortId := ta.FortId.ValueOrZero(); fortId != "" { // we don't know any despawn times from lured/fort tappables - ta.ExpireTimestamp = null.IntFrom(int64(timestampMs)/1000 + int64(120)) + ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) } } func (ta *Tappable) setUnknownTimestamp(now int64) { if !ta.ExpireTimestamp.Valid { - ta.ExpireTimestamp = null.IntFrom(now + 20*60) + ta.SetExpireTimestamp(null.IntFrom(now + 20*60)) } else { if ta.ExpireTimestamp.Int64 < now { - ta.ExpireTimestamp = null.IntFrom(now + 10*60) + ta.SetExpireTimestamp(null.IntFrom(now + 10*60)) } } } @@ -134,12 +226,12 @@ func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappab inMemoryTappable := tappableCache.Get(id) if inMemoryTappable != nil { tappable := inMemoryTappable.Value() - return &tappable, nil + return tappable, nil } tappable := Tappable{} err := db.GeneralDb.GetContext(ctx, &tappable, `SELECT id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated - FROM tappable + FROM tappable WHERE id = ?`, strconv.FormatUint(id, 10)) statsCollector.IncDbQuery("select tappable", err) if errors.Is(err, sql.ErrNoRows) { @@ -153,13 +245,15 @@ func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappab } func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tappable) { - oldTappable, _ := GetTappableRecord(ctx, details, tappable.Id) - now := time.Now().Unix() - if oldTappable != nil && !hasChangesTappable(oldTappable, tappable) { + // Skip save if not dirty and not new + if !tappable.IsDirty() && !tappable.IsNewRecord() { return } + + now := time.Now().Unix() tappable.Updated = now - if oldTappable == nil { + + if tappable.IsNewRecord() { res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` INSERT INTO tappable ( id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated @@ -184,7 +278,7 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap pokemon_id = :pokemon_id, item_id = :item_id, count = :count, - expire_timestamp = :expire_timestamp, + expire_timestamp = :expire_timestamp, expire_timestamp_verified = :expire_timestamp_verified, updated = :updated WHERE id = "%d" @@ -196,21 +290,9 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } _ = res } - tappableCache.Set(tappable.Id, *tappable, ttlcache.DefaultTTL) -} - -func hasChangesTappable(old *Tappable, new *Tappable) bool { - return old.Id != new.Id || - old.FortId != new.FortId || - old.SpawnId != new.SpawnId || - old.Type != new.Type || - old.Encounter != new.Encounter || - old.ItemId != new.ItemId || - old.Count != new.Count || - old.ExpireTimestamp != new.ExpireTimestamp || - old.ExpireTimestampVerified != new.ExpireTimestampVerified || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) + tappable.ClearDirty() + tappable.newRecord = false + //tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) } func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { @@ -226,7 +308,7 @@ func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessT } if tappable == nil { - tappable = &Tappable{} + tappable = &Tappable{newRecord: true} } tappable.updateFromProcessTappableProto(ctx, db, tappableDetails, request, timestampMs) diff --git a/decoder/weather.go b/decoder/weather.go index 3e29aa24..8381de26 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -3,18 +3,18 @@ package decoder import ( "context" "database/sql" + "golbat/db" "golbat/pogo" "golbat/webhooks" "github.com/golang/geo/s2" - "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) // Weather struct. -// REMINDER! Keep hasChangesWeather updated after making changes +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Weather struct { Id int64 `db:"id"` Latitude float64 `db:"latitude"` @@ -31,6 +31,17 @@ type Weather struct { Severity null.Int `db:"severity"` WarnWeather null.Bool `db:"warn_weather"` UpdatedMs int64 `db:"updated"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + oldValues WeatherOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// WeatherOldValues holds old field values for webhook comparison +type WeatherOldValues struct { + GameplayCondition null.Int + WarnWeather null.Bool } // CREATE TABLE `weather` ( @@ -52,11 +63,136 @@ type Weather struct { // PRIMARY KEY (`id`) //) +// IsDirty returns true if any field has been modified +func (weather *Weather) IsDirty() bool { + return weather.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (weather *Weather) ClearDirty() { + weather.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (weather *Weather) IsNewRecord() bool { + return weather.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (weather *Weather) snapshotOldValues() { + weather.oldValues = WeatherOldValues{ + GameplayCondition: weather.GameplayCondition, + WarnWeather: weather.WarnWeather, + } +} + +// --- Set methods with dirty tracking --- + +func (weather *Weather) SetId(v int64) { + if weather.Id != v { + weather.Id = v + weather.dirty = true + } +} + +func (weather *Weather) SetLatitude(v float64) { + if !floatAlmostEqual(weather.Latitude, v, floatTolerance) { + weather.Latitude = v + weather.dirty = true + } +} + +func (weather *Weather) SetLongitude(v float64) { + if !floatAlmostEqual(weather.Longitude, v, floatTolerance) { + weather.Longitude = v + weather.dirty = true + } +} + +func (weather *Weather) SetLevel(v null.Int) { + if weather.Level != v { + weather.Level = v + weather.dirty = true + } +} + +func (weather *Weather) SetGameplayCondition(v null.Int) { + if weather.GameplayCondition != v { + weather.GameplayCondition = v + weather.dirty = true + } +} + +func (weather *Weather) SetWindDirection(v null.Int) { + if weather.WindDirection != v { + weather.WindDirection = v + weather.dirty = true + } +} + +func (weather *Weather) SetCloudLevel(v null.Int) { + if weather.CloudLevel != v { + weather.CloudLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetRainLevel(v null.Int) { + if weather.RainLevel != v { + weather.RainLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetWindLevel(v null.Int) { + if weather.WindLevel != v { + weather.WindLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetSnowLevel(v null.Int) { + if weather.SnowLevel != v { + weather.SnowLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetFogLevel(v null.Int) { + if weather.FogLevel != v { + weather.FogLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetSpecialEffectLevel(v null.Int) { + if weather.SpecialEffectLevel != v { + weather.SpecialEffectLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetSeverity(v null.Int) { + if weather.Severity != v { + weather.Severity = v + weather.dirty = true + } +} + +func (weather *Weather) SetWarnWeather(v null.Bool) { + if weather.WarnWeather != v { + weather.WarnWeather = v + weather.dirty = true + } +} + func getWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, error) { inMemoryWeather := weatherCache.Get(weatherId) if inMemoryWeather != nil { weather := inMemoryWeather.Value() - return &weather, nil + weather.snapshotOldValues() + return weather, nil } weather := Weather{} @@ -72,7 +208,7 @@ func getWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*W } weather.UpdatedMs *= 1000 - weatherCache.Set(weatherId, weather, ttlcache.DefaultTTL) + weather.snapshotOldValues() return &weather, nil } @@ -80,46 +216,24 @@ func weatherCellIdFromLatLon(lat, lon float64) int64 { return int64(s2.CellIDFromLatLng(s2.LatLngFromDegrees(lat, lon)).Parent(10)) } -func (weather *Weather) updateWeatherFromClientWeatherProto(clientWeather *pogo.ClientWeatherProto) (oldGameplayCondition null.Int) { - oldGameplayCondition = weather.GameplayCondition - weather.Id = clientWeather.S2CellId +func (weather *Weather) updateWeatherFromClientWeatherProto(clientWeather *pogo.ClientWeatherProto) { + weather.SetId(clientWeather.S2CellId) s2cell := s2.CellFromCellID(s2.CellID(clientWeather.S2CellId)) - weather.Latitude = s2cell.CapBound().RectBound().Center().Lat.Degrees() - weather.Longitude = s2cell.CapBound().RectBound().Center().Lng.Degrees() - weather.Level = null.IntFrom(int64(s2cell.Level())) - weather.GameplayCondition = null.IntFrom(int64(clientWeather.GameplayWeather.GameplayCondition)) - weather.WindDirection = null.IntFrom(int64(clientWeather.DisplayWeather.WindDirection)) - weather.CloudLevel = null.IntFrom(int64(clientWeather.DisplayWeather.CloudLevel)) - weather.RainLevel = null.IntFrom(int64(clientWeather.DisplayWeather.RainLevel)) - weather.WindLevel = null.IntFrom(int64(clientWeather.DisplayWeather.WindLevel)) - weather.SnowLevel = null.IntFrom(int64(clientWeather.DisplayWeather.SnowLevel)) - weather.FogLevel = null.IntFrom(int64(clientWeather.DisplayWeather.FogLevel)) - weather.SpecialEffectLevel = null.IntFrom(int64(clientWeather.DisplayWeather.SpecialEffectLevel)) + weather.SetLatitude(s2cell.CapBound().RectBound().Center().Lat.Degrees()) + weather.SetLongitude(s2cell.CapBound().RectBound().Center().Lng.Degrees()) + weather.SetLevel(null.IntFrom(int64(s2cell.Level()))) + weather.SetGameplayCondition(null.IntFrom(int64(clientWeather.GameplayWeather.GameplayCondition))) + weather.SetWindDirection(null.IntFrom(int64(clientWeather.DisplayWeather.WindDirection))) + weather.SetCloudLevel(null.IntFrom(int64(clientWeather.DisplayWeather.CloudLevel))) + weather.SetRainLevel(null.IntFrom(int64(clientWeather.DisplayWeather.RainLevel))) + weather.SetWindLevel(null.IntFrom(int64(clientWeather.DisplayWeather.WindLevel))) + weather.SetSnowLevel(null.IntFrom(int64(clientWeather.DisplayWeather.SnowLevel))) + weather.SetFogLevel(null.IntFrom(int64(clientWeather.DisplayWeather.FogLevel))) + weather.SetSpecialEffectLevel(null.IntFrom(int64(clientWeather.DisplayWeather.SpecialEffectLevel))) for _, alert := range clientWeather.Alerts { - weather.Severity = null.IntFrom(int64(alert.Severity)) - weather.WarnWeather = null.BoolFrom(alert.WarnWeather) - } - return -} - -// hasChangesWeather compares two Weather structs -// Float tolerance: Latitude, Longitude -func hasChangesWeather(old *Weather, new *Weather) bool { - return old.Id != new.Id || - old.Level != new.Level || - old.GameplayCondition != new.GameplayCondition || - old.WindDirection != new.WindDirection || - old.CloudLevel != new.CloudLevel || - old.RainLevel != new.RainLevel || - old.WindLevel != new.WindLevel || - old.SnowLevel != new.SnowLevel || - old.FogLevel != new.FogLevel || - old.SpecialEffectLevel != new.SpecialEffectLevel || - old.Severity != new.Severity || - old.WarnWeather != new.WarnWeather || - old.UpdatedMs != new.UpdatedMs || - !floatAlmostEqual(old.Latitude, new.Latitude, floatTolerance) || - !floatAlmostEqual(old.Longitude, new.Longitude, floatTolerance) + weather.SetSeverity(null.IntFrom(int64(alert.Severity))) + weather.SetWarnWeather(null.BoolFrom(alert.WarnWeather)) + } } type WeatherWebhook struct { @@ -140,9 +254,12 @@ type WeatherWebhook struct { Updated int64 `json:"updated"` } -func createWeatherWebhooks(oldWeather *Weather, weather *Weather) { - if oldWeather == nil || oldWeather.GameplayCondition.ValueOrZero() != weather.GameplayCondition.ValueOrZero() || - oldWeather.WarnWeather.ValueOrZero() != weather.WarnWeather.ValueOrZero() { +func createWeatherWebhooks(weather *Weather) { + old := &weather.oldValues + isNew := weather.IsNewRecord() + + if isNew || old.GameplayCondition.ValueOrZero() != weather.GameplayCondition.ValueOrZero() || + old.WarnWeather.ValueOrZero() != weather.WarnWeather.ValueOrZero() { s2cell := s2.CellFromCellID(s2.CellID(weather.Id)) var polygon [4][2]float64 @@ -175,12 +292,12 @@ func createWeatherWebhooks(oldWeather *Weather, weather *Weather) { } func saveWeatherRecord(ctx context.Context, db db.DbDetails, weather *Weather) { - oldWeather, _ := getWeatherRecord(ctx, db, weather.Id) - if oldWeather != nil && !hasChangesWeather(oldWeather, weather) { + // Skip save if not dirty and not new + if !weather.IsDirty() && !weather.IsNewRecord() { return } - if oldWeather == nil { + if weather.IsNewRecord() { res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO weather ("+ "id, latitude, longitude, level, gameplay_condition, wind_direction, cloud_level, rain_level, "+ @@ -221,6 +338,8 @@ func saveWeatherRecord(ctx context.Context, db db.DbDetails, weather *Weather) { } _ = res } - weatherCache.Set(weather.Id, *weather, ttlcache.DefaultTTL) - createWeatherWebhooks(oldWeather, weather) + createWeatherWebhooks(weather) + weather.ClearDirty() + weather.newRecord = false + //weatherCache.Set(weather.Id, weather, ttlcache.DefaultTTL) } From a985123b8e6707c8e7903040a4ca36fe01a17023 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 16:57:04 +0000 Subject: [PATCH 08/78] Generic sharding introduced for pokestop/gym --- decoder/main.go | 55 +++++++++---------- decoder/pokemonRtree.go | 14 ++--- decoder/sharded_cache.go | 116 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 37 deletions(-) create mode 100644 decoder/sharded_cache.go diff --git a/decoder/main.go b/decoder/main.go index e398e5e9..6877aee4 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -58,15 +58,15 @@ type webhooksSenderInterface interface { var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector -var pokestopCache *ttlcache.Cache[string, *Pokestop] +var pokestopCache *ShardedCache[string, *Pokestop] var gymCache *ttlcache.Cache[string, *Gym] var stationCache *ttlcache.Cache[string, *Station] var tappableCache *ttlcache.Cache[uint64, *Tappable] var weatherCache *ttlcache.Cache[int64, *Weather] var weatherConsensusCache *ttlcache.Cache[int64, *WeatherConsensusState] var s2CellCache *ttlcache.Cache[uint64, *S2Cell] -var spawnpointCache *ttlcache.Cache[int64, *Spawnpoint] -var pokemonCache []*ttlcache.Cache[uint64, *Pokemon] +var spawnpointCache *ShardedCache[int64, *Spawnpoint] +var pokemonCache *ShardedCache[uint64, *Pokemon] var incidentCache *ttlcache.Cache[string, *Incident] var playerCache *ttlcache.Cache[string, *Player] var routeCache *ttlcache.Cache[string, *Route] @@ -101,27 +101,25 @@ func (cl *gohbemLogger) Print(message string) { log.Info("Gohbem - ", message) } -func getPokemonCache(key uint64) *ttlcache.Cache[uint64, *Pokemon] { - return pokemonCache[key%uint64(len(pokemonCache))] -} - func setPokemonCache(key uint64, value *Pokemon, ttl time.Duration) { - getPokemonCache(key).Set(key, value, ttl) + pokemonCache.Set(key, value, ttl) } func getPokemonFromCache(key uint64) *ttlcache.Item[uint64, *Pokemon] { - return getPokemonCache(key).Get(key) + return pokemonCache.Get(key) } func deletePokemonFromCache(key uint64) { - getPokemonCache(key).Delete(key) + pokemonCache.Delete(key) } func initDataCache() { - pokestopCache = ttlcache.New[string, *Pokestop]( - ttlcache.WithTTL[string, *Pokestop](60 * time.Minute), - ) - go pokestopCache.Start() + // Sharded caches for high-concurrency tables + pokestopCache = NewShardedCache(ShardedCacheConfig[string, *Pokestop]{ + NumShards: runtime.NumCPU(), + TTL: 60 * time.Minute, + KeyToShard: StringKeyToShard, + }) gymCache = ttlcache.New[string, *Gym]( ttlcache.WithTTL[string, *Gym](60 * time.Minute), @@ -153,21 +151,20 @@ func initDataCache() { ) go s2CellCache.Start() - spawnpointCache = ttlcache.New[int64, *Spawnpoint]( - ttlcache.WithTTL[int64, *Spawnpoint](60 * time.Minute), - ) - go spawnpointCache.Start() - - // pokemon is the most active table. Use an array of caches to increase concurrency for querying ttlcache, which places a global lock for each Get/Set operation - // Initialize pokemon cache array: by picking it to be nproc, we should expect ~nproc*(1-1/e) ~ 63% concurrency - pokemonCache = make([]*ttlcache.Cache[uint64, *Pokemon], runtime.NumCPU()) - for i := 0; i < len(pokemonCache); i++ { - pokemonCache[i] = ttlcache.New[uint64, *Pokemon]( - ttlcache.WithTTL[uint64, *Pokemon](60*time.Minute), - ttlcache.WithDisableTouchOnHit[uint64, *Pokemon](), // Pokemon will last 60 mins from when we first see them not last see them - ) - go pokemonCache[i].Start() - } + spawnpointCache = NewShardedCache(ShardedCacheConfig[int64, *Spawnpoint]{ + NumShards: runtime.NumCPU(), + TTL: 60 * time.Minute, + KeyToShard: Int64KeyToShard, + }) + + // Pokemon cache: sharded for high concurrency + // By picking NumShards to be nproc, we should expect ~nproc*(1-1/e) ~ 63% concurrency + pokemonCache = NewShardedCache(ShardedCacheConfig[uint64, *Pokemon]{ + NumShards: runtime.NumCPU(), + TTL: 60 * time.Minute, + KeyToShard: Uint64KeyToShard, + DisableTouchOnHit: true, // Pokemon will last 60 mins from when we first see them not last see them + }) initPokemonRtree() initFortRtree() diff --git a/decoder/pokemonRtree.go b/decoder/pokemonRtree.go index 02037386..48aadc01 100644 --- a/decoder/pokemonRtree.go +++ b/decoder/pokemonRtree.go @@ -50,14 +50,12 @@ var pokemonTree rtree.RTreeG[uint64] func initPokemonRtree() { pokemonLookupCache = xsync.NewMapOf[uint64, PokemonLookupCacheItem]() - // Set up OnEviction callbacks for each cache in the array - for i := range pokemonCache { - pokemonCache[i].OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, *Pokemon]) { - pokemon := v.Value() - removePokemonFromTree(pokemon.Id, pokemon.Lat, pokemon.Lon) - // Rely on the pokemon pvp lookup caches to remove themselves rather than trying to synchronise - }) - } + // Set up OnEviction callback on all shards + pokemonCache.OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, *Pokemon]) { + pokemon := v.Value() + removePokemonFromTree(pokemon.Id, pokemon.Lat, pokemon.Lon) + // Rely on the pokemon pvp lookup caches to remove themselves rather than trying to synchronise + }) } func pokemonRtreeUpdatePokemonOnGet(pokemon *Pokemon) { diff --git a/decoder/sharded_cache.go b/decoder/sharded_cache.go new file mode 100644 index 00000000..b0bdecab --- /dev/null +++ b/decoder/sharded_cache.go @@ -0,0 +1,116 @@ +package decoder + +import ( + "context" + "hash/fnv" + "time" + + "github.com/jellydator/ttlcache/v3" +) + +// ShardedCache is a generic sharded cache for improved concurrency. +// It distributes entries across multiple ttlcache instances to reduce lock contention. +type ShardedCache[K comparable, V any] struct { + shards []*ttlcache.Cache[K, V] + keyToShard func(K) uint64 +} + +// ShardedCacheConfig holds configuration for creating a ShardedCache +type ShardedCacheConfig[K comparable, V any] struct { + NumShards int + TTL time.Duration + KeyToShard func(K) uint64 + DisableTouchOnHit bool +} + +// NewShardedCache creates a new sharded cache with the given configuration. +// The keyToShard function converts keys to uint64 for shard selection. +func NewShardedCache[K comparable, V any](config ShardedCacheConfig[K, V]) *ShardedCache[K, V] { + sc := &ShardedCache[K, V]{ + shards: make([]*ttlcache.Cache[K, V], config.NumShards), + keyToShard: config.KeyToShard, + } + + for i := 0; i < config.NumShards; i++ { + opts := []ttlcache.Option[K, V]{ + ttlcache.WithTTL[K, V](config.TTL), + } + if config.DisableTouchOnHit { + opts = append(opts, ttlcache.WithDisableTouchOnHit[K, V]()) + } + sc.shards[i] = ttlcache.New[K, V](opts...) + go sc.shards[i].Start() + } + + return sc +} + +// getShard returns the cache shard for the given key +func (sc *ShardedCache[K, V]) getShard(key K) *ttlcache.Cache[K, V] { + return sc.shards[sc.keyToShard(key)%uint64(len(sc.shards))] +} + +// Get retrieves an item from the appropriate shard +func (sc *ShardedCache[K, V]) Get(key K) *ttlcache.Item[K, V] { + return sc.getShard(key).Get(key) +} + +// Set stores an item in the appropriate shard +func (sc *ShardedCache[K, V]) Set(key K, value V, ttl time.Duration) { + sc.getShard(key).Set(key, value, ttl) +} + +// Delete removes an item from the appropriate shard +func (sc *ShardedCache[K, V]) Delete(key K) { + sc.getShard(key).Delete(key) +} + +// Range iterates over all items in all shards. +// The callback should return true to continue iteration or false to stop. +func (sc *ShardedCache[K, V]) Range(fn func(*ttlcache.Item[K, V]) bool) { + for _, shard := range sc.shards { + shard.Range(fn) + } +} + +// OnEviction sets an eviction callback on all shards +func (sc *ShardedCache[K, V]) OnEviction(fn func(context.Context, ttlcache.EvictionReason, *ttlcache.Item[K, V])) { + for _, shard := range sc.shards { + shard.OnEviction(fn) + } +} + +// Len returns the total number of items across all shards +func (sc *ShardedCache[K, V]) Len() int { + total := 0 + for _, shard := range sc.shards { + total += shard.Len() + } + return total +} + +// DeleteAll removes all items from all shards +func (sc *ShardedCache[K, V]) DeleteAll() { + for _, shard := range sc.shards { + shard.DeleteAll() + } +} + +// --- Key conversion helpers --- + +// Uint64KeyToShard is the identity function for uint64 keys +func Uint64KeyToShard(key uint64) uint64 { + return key +} + +// Int64KeyToShard converts int64 keys to uint64 for sharding +func Int64KeyToShard(key int64) uint64 { + return uint64(key) +} + +// StringKeyToShard hashes string keys to uint64 for sharding using FNV-1a +func StringKeyToShard(key string) uint64 { + h := fnv.New64a() + h.Write([]byte(key)) + return h.Sum64() +} From bb8210cba40177064e2ac07c332f314889349aac Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 21:12:36 +0000 Subject: [PATCH 09/78] Database change tracing --- decoder/db_debug.go | 21 ++++++ decoder/db_debug_off.go | 14 ++++ decoder/gym.go | 131 ++++++++++++++++++++++++++++++++++- decoder/player.go | 6 +- decoder/pokemon.go | 101 ++++++++++++++++++++++++++- decoder/pokestop.go | 146 +++++++++++++++++++++++++++++++++++++++- decoder/routes.go | 80 ++++++++++++++++++++-- decoder/s2cell.go | 13 ++++ decoder/spawnpoint.go | 11 +-- decoder/station.go | 99 +++++++++++++++++++++++++-- decoder/tappable.go | 51 ++++++++++++-- decoder/weather.go | 7 +- 12 files changed, 652 insertions(+), 28 deletions(-) create mode 100644 decoder/db_debug.go create mode 100644 decoder/db_debug_off.go diff --git a/decoder/db_debug.go b/decoder/db_debug.go new file mode 100644 index 00000000..89498a3f --- /dev/null +++ b/decoder/db_debug.go @@ -0,0 +1,21 @@ +//go:build dbdebug + +package decoder + +import ( + "strings" + + log "github.com/sirupsen/logrus" +) + +// dbDebugEnabled is true when built with -tags dbdebug +const dbDebugEnabled = true + +// dbDebugLog logs a database operation with changed fields +func dbDebugLog(reason, entityType, id string, changedFields []string) { + fields := "" + if len(changedFields) > 0 { + fields = " changed=[" + strings.Join(changedFields, ", ") + "]" + } + log.Debugf("[DB_%s] %s id=%s%s", reason, entityType, id, fields) +} diff --git a/decoder/db_debug_off.go b/decoder/db_debug_off.go new file mode 100644 index 00000000..b5a32b77 --- /dev/null +++ b/decoder/db_debug_off.go @@ -0,0 +1,14 @@ +//go:build !dbdebug + +package decoder + +// dbDebugEnabled is false when dbdebug build tag is not set. +// The compiler will eliminate dead code in if statements checking this const. +const dbDebugEnabled = false + +// dbDebugLog is a no-op stub when dbdebug build tag is not set. +// This function is never called at runtime due to const-folding of dbDebugEnabled. +func dbDebugLog(reason, entityType, id string, changedFields []string) { + // No-op: this function exists only to satisfy the compiler. + // It will never be called because dbDebugEnabled is false. +} diff --git a/decoder/gym.go b/decoder/gym.go index dba5fb34..b3bdcaf6 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -68,8 +68,9 @@ type Gym struct { Defenders null.String `db:"defenders" json:"defenders"` Rsvps null.String `db:"rsvps" json:"rsvps"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues GymOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -177,6 +178,9 @@ func (gym *Gym) SetId(v string) { if gym.Id != v { gym.Id = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Id") + } } } @@ -184,6 +188,9 @@ func (gym *Gym) SetLat(v float64) { if !floatAlmostEqual(gym.Lat, v, floatTolerance) { gym.Lat = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Lat") + } } } @@ -191,6 +198,9 @@ func (gym *Gym) SetLon(v float64) { if !floatAlmostEqual(gym.Lon, v, floatTolerance) { gym.Lon = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Lon") + } } } @@ -198,6 +208,9 @@ func (gym *Gym) SetName(v null.String) { if gym.Name != v { gym.Name = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Name") + } } } @@ -205,6 +218,9 @@ func (gym *Gym) SetUrl(v null.String) { if gym.Url != v { gym.Url = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Url") + } } } @@ -212,6 +228,9 @@ func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { if gym.LastModifiedTimestamp != v { gym.LastModifiedTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "LastModifiedTimestamp") + } } } @@ -219,6 +238,9 @@ func (gym *Gym) SetRaidEndTimestamp(v null.Int) { if gym.RaidEndTimestamp != v { gym.RaidEndTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidEndTimestamp") + } } } @@ -226,6 +248,9 @@ func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { if gym.RaidSpawnTimestamp != v { gym.RaidSpawnTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidSpawnTimestamp") + } } } @@ -233,6 +258,9 @@ func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { if gym.RaidBattleTimestamp != v { gym.RaidBattleTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidBattleTimestamp") + } } } @@ -240,6 +268,9 @@ func (gym *Gym) SetRaidPokemonId(v null.Int) { if gym.RaidPokemonId != v { gym.RaidPokemonId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonId") + } } } @@ -247,6 +278,9 @@ func (gym *Gym) SetGuardingPokemonId(v null.Int) { if gym.GuardingPokemonId != v { gym.GuardingPokemonId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "GuardingPokemonId") + } } } @@ -254,6 +288,9 @@ func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { if gym.GuardingPokemonDisplay != v { gym.GuardingPokemonDisplay = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "GuardingPokemonDisplay") + } } } @@ -261,6 +298,9 @@ func (gym *Gym) SetAvailableSlots(v null.Int) { if gym.AvailableSlots != v { gym.AvailableSlots = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "AvailableSlots") + } } } @@ -268,6 +308,9 @@ func (gym *Gym) SetTeamId(v null.Int) { if gym.TeamId != v { gym.TeamId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "TeamId") + } } } @@ -275,6 +318,9 @@ func (gym *Gym) SetRaidLevel(v null.Int) { if gym.RaidLevel != v { gym.RaidLevel = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidLevel") + } } } @@ -282,6 +328,9 @@ func (gym *Gym) SetEnabled(v null.Int) { if gym.Enabled != v { gym.Enabled = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Enabled") + } } } @@ -289,6 +338,9 @@ func (gym *Gym) SetExRaidEligible(v null.Int) { if gym.ExRaidEligible != v { gym.ExRaidEligible = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "ExRaidEligible") + } } } @@ -304,6 +356,9 @@ func (gym *Gym) SetRaidPokemonMove1(v null.Int) { if gym.RaidPokemonMove1 != v { gym.RaidPokemonMove1 = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonMove1") + } } } @@ -311,6 +366,9 @@ func (gym *Gym) SetRaidPokemonMove2(v null.Int) { if gym.RaidPokemonMove2 != v { gym.RaidPokemonMove2 = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonMove2") + } } } @@ -318,6 +376,9 @@ func (gym *Gym) SetRaidPokemonForm(v null.Int) { if gym.RaidPokemonForm != v { gym.RaidPokemonForm = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonForm") + } } } @@ -325,6 +386,9 @@ func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { if gym.RaidPokemonAlignment != v { gym.RaidPokemonAlignment = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonAlignment") + } } } @@ -332,6 +396,9 @@ func (gym *Gym) SetRaidPokemonCp(v null.Int) { if gym.RaidPokemonCp != v { gym.RaidPokemonCp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonCp") + } } } @@ -339,6 +406,9 @@ func (gym *Gym) SetRaidIsExclusive(v null.Int) { if gym.RaidIsExclusive != v { gym.RaidIsExclusive = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidIsExclusive") + } } } @@ -346,6 +416,9 @@ func (gym *Gym) SetCellId(v null.Int) { if gym.CellId != v { gym.CellId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "CellId") + } } } @@ -353,6 +426,9 @@ func (gym *Gym) SetDeleted(v bool) { if gym.Deleted != v { gym.Deleted = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Deleted") + } } } @@ -360,6 +436,9 @@ func (gym *Gym) SetTotalCp(v null.Int) { if gym.TotalCp != v { gym.TotalCp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "TotalCp") + } } } @@ -367,6 +446,9 @@ func (gym *Gym) SetRaidPokemonGender(v null.Int) { if gym.RaidPokemonGender != v { gym.RaidPokemonGender = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonGender") + } } } @@ -374,6 +456,9 @@ func (gym *Gym) SetSponsorId(v null.Int) { if gym.SponsorId != v { gym.SponsorId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "SponsorId") + } } } @@ -381,6 +466,9 @@ func (gym *Gym) SetPartnerId(v null.String) { if gym.PartnerId != v { gym.PartnerId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "PartnerId") + } } } @@ -388,6 +476,9 @@ func (gym *Gym) SetRaidPokemonCostume(v null.Int) { if gym.RaidPokemonCostume != v { gym.RaidPokemonCostume = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonCostume") + } } } @@ -395,6 +486,9 @@ func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { if gym.RaidPokemonEvolution != v { gym.RaidPokemonEvolution = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonEvolution") + } } } @@ -402,6 +496,9 @@ func (gym *Gym) SetArScanEligible(v null.Int) { if gym.ArScanEligible != v { gym.ArScanEligible = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "ArScanEligible") + } } } @@ -409,6 +506,9 @@ func (gym *Gym) SetPowerUpLevel(v null.Int) { if gym.PowerUpLevel != v { gym.PowerUpLevel = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "PowerUpLevel") + } } } @@ -416,6 +516,9 @@ func (gym *Gym) SetPowerUpPoints(v null.Int) { if gym.PowerUpPoints != v { gym.PowerUpPoints = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "PowerUpPoints") + } } } @@ -423,6 +526,9 @@ func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { if gym.PowerUpEndTimestamp != v { gym.PowerUpEndTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "PowerUpEndTimestamp") + } } } @@ -430,6 +536,9 @@ func (gym *Gym) SetDescription(v null.String) { if gym.Description != v { gym.Description = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Description") + } } } @@ -445,6 +554,9 @@ func (gym *Gym) SetRsvps(v null.String) { if gym.Rsvps != v { gym.Rsvps = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Rsvps") + } } } @@ -992,6 +1104,9 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { gym.Updated = now if gym.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) @@ -1003,6 +1118,9 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { _, _ = res, err } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ "lat = :lat, "+ "lon = :lon, "+ @@ -1057,8 +1175,15 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { createGymWebhooks(gym, areas) createGymFortWebhooks(gym) updateRaidStats(gym, areas) - gym.newRecord = false // After saving, it's no longer a new record + if dbDebugEnabled { + gym.changedFields = gym.changedFields[:0] + } + if gym.IsNewRecord() { + gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + gym.newRecord = false + } gym.ClearDirty() + } func updateGymGetMapFortCache(gym *Gym, skipName bool) { diff --git a/decoder/player.go b/decoder/player.go index 721f7d88..0478ae75 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -914,8 +914,10 @@ func savePlayerRecord(db db.DbDetails, player *Player) { } player.ClearDirty() - player.newRecord = false - //playerCache.Set(player.Name, player, ttlcache.DefaultTTL) + if player.IsNewRecord() { + player.newRecord = false + playerCache.Set(player.Name, player, ttlcache.DefaultTTL) + } } func (player *Player) updateFromPublicProfile(publicProfile *pogo.PlayerPublicProfileProto) { diff --git a/decoder/pokemon.go b/decoder/pokemon.go index d18c6d16..9399bd2a 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -77,8 +77,9 @@ type Pokemon struct { internal grpc.PokemonInternal - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues PokemonOldValues `db:"-" json:"-"` // Old values for webhook comparison and stats } @@ -175,6 +176,9 @@ func (pokemon *Pokemon) SetPokestopId(v null.String) { if pokemon.PokestopId != v { pokemon.PokestopId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "PokestopId") + } } } @@ -182,6 +186,9 @@ func (pokemon *Pokemon) SetSpawnId(v null.Int) { if pokemon.SpawnId != v { pokemon.SpawnId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "SpawnId") + } } } @@ -189,6 +196,9 @@ func (pokemon *Pokemon) SetLat(v float64) { if !floatAlmostEqual(pokemon.Lat, v, floatTolerance) { pokemon.Lat = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Lat") + } } } @@ -196,6 +206,9 @@ func (pokemon *Pokemon) SetLon(v float64) { if !floatAlmostEqual(pokemon.Lon, v, floatTolerance) { pokemon.Lon = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Lon") + } } } @@ -203,6 +216,9 @@ func (pokemon *Pokemon) SetPokemonId(v int16) { if pokemon.PokemonId != v { pokemon.PokemonId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "PokemonId") + } } } @@ -210,6 +226,9 @@ func (pokemon *Pokemon) SetForm(v null.Int) { if pokemon.Form != v { pokemon.Form = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Form") + } } } @@ -217,6 +236,9 @@ func (pokemon *Pokemon) SetCostume(v null.Int) { if pokemon.Costume != v { pokemon.Costume = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Costume") + } } } @@ -224,6 +246,9 @@ func (pokemon *Pokemon) SetGender(v null.Int) { if pokemon.Gender != v { pokemon.Gender = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Gender") + } } } @@ -231,6 +256,9 @@ func (pokemon *Pokemon) SetWeather(v null.Int) { if pokemon.Weather != v { pokemon.Weather = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Weather") + } } } @@ -238,6 +266,9 @@ func (pokemon *Pokemon) SetIsStrong(v null.Bool) { if pokemon.IsStrong != v { pokemon.IsStrong = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "IsStrong") + } } } @@ -245,6 +276,9 @@ func (pokemon *Pokemon) SetExpireTimestamp(v null.Int) { if pokemon.ExpireTimestamp != v { pokemon.ExpireTimestamp = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "ExpireTimestamp") + } } } @@ -252,6 +286,9 @@ func (pokemon *Pokemon) SetExpireTimestampVerified(v bool) { if pokemon.ExpireTimestampVerified != v { pokemon.ExpireTimestampVerified = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "ExpireTimestampVerified") + } } } @@ -259,6 +296,9 @@ func (pokemon *Pokemon) SetSeenType(v null.String) { if pokemon.SeenType != v { pokemon.SeenType = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "SeenType") + } } } @@ -273,6 +313,9 @@ func (pokemon *Pokemon) SetCellId(v null.Int) { if pokemon.CellId != v { pokemon.CellId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "CellId") + } } } @@ -280,6 +323,9 @@ func (pokemon *Pokemon) SetIsEvent(v int8) { if pokemon.IsEvent != v { pokemon.IsEvent = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "IsEvent") + } } } @@ -287,6 +333,9 @@ func (pokemon *Pokemon) SetShiny(v null.Bool) { if pokemon.Shiny != v { pokemon.Shiny = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Shiny") + } } } @@ -294,6 +343,9 @@ func (pokemon *Pokemon) SetCp(v null.Int) { if pokemon.Cp != v { pokemon.Cp = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Cp") + } } } @@ -301,6 +353,9 @@ func (pokemon *Pokemon) SetLevel(v null.Int) { if pokemon.Level != v { pokemon.Level = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Level") + } } } @@ -308,6 +363,9 @@ func (pokemon *Pokemon) SetMove1(v null.Int) { if pokemon.Move1 != v { pokemon.Move1 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Move1") + } } } @@ -315,6 +373,9 @@ func (pokemon *Pokemon) SetMove2(v null.Int) { if pokemon.Move2 != v { pokemon.Move2 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Move2") + } } } @@ -322,6 +383,9 @@ func (pokemon *Pokemon) SetHeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Height, v, floatTolerance) { pokemon.Height = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Height") + } } } @@ -329,6 +393,9 @@ func (pokemon *Pokemon) SetWeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Weight, v, floatTolerance) { pokemon.Weight = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Weight") + } } } @@ -336,6 +403,9 @@ func (pokemon *Pokemon) SetSize(v null.Int) { if pokemon.Size != v { pokemon.Size = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Size") + } } } @@ -343,6 +413,9 @@ func (pokemon *Pokemon) SetIsDitto(v bool) { if pokemon.IsDitto != v { pokemon.IsDitto = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "IsDitto") + } } } @@ -350,6 +423,9 @@ func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { if pokemon.DisplayPokemonId != v { pokemon.DisplayPokemonId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "DisplayPokemonId") + } } } @@ -357,6 +433,9 @@ func (pokemon *Pokemon) SetPvp(v null.String) { if pokemon.Pvp != v { pokemon.Pvp = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Pvp") + } } } @@ -364,6 +443,9 @@ func (pokemon *Pokemon) SetCapture1(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture1, v, floatTolerance) { pokemon.Capture1 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Capture1") + } } } @@ -371,6 +453,9 @@ func (pokemon *Pokemon) SetCapture2(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture2, v, floatTolerance) { pokemon.Capture2 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Capture2") + } } } @@ -378,6 +463,9 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture3, v, floatTolerance) { pokemon.Capture3 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Capture3") + } } } @@ -552,6 +640,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } if pokemon.isNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } pvpField, pvpValue := "", "" if changePvpField { pvpField, pvpValue = "pvp, ", ":pvp, " @@ -578,6 +669,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po rows, rowsErr := res.RowsAffected() log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } pvpUpdate := "" if changePvpField { pvpUpdate = "pvp = :pvp, " @@ -650,6 +744,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } updatePokemonStats(pokemon, areas, now) + if dbDebugEnabled { + pokemon.changedFields = pokemon.changedFields[:0] + } pokemon.newRecord = false // After saving, it's no longer a new record pokemon.ClearDirty() diff --git a/decoder/pokestop.go b/decoder/pokestop.go index c42200fc..deb829f6 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -68,8 +68,9 @@ type Pokestop struct { ShowcaseExpiry null.Int `db:"showcase_expiry" json:"showcase_expiry"` ShowcaseRankings null.String `db:"showcase_rankings" json:"showcase_rankings"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues PokestopOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -164,6 +165,9 @@ func (p *Pokestop) SetId(v string) { if p.Id != v { p.Id = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Id") + } } } @@ -171,6 +175,9 @@ func (p *Pokestop) SetLat(v float64) { if !floatAlmostEqual(p.Lat, v, floatTolerance) { p.Lat = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Lat") + } } } @@ -178,6 +185,9 @@ func (p *Pokestop) SetLon(v float64) { if !floatAlmostEqual(p.Lon, v, floatTolerance) { p.Lon = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Lon") + } } } @@ -185,6 +195,9 @@ func (p *Pokestop) SetName(v null.String) { if p.Name != v { p.Name = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Name") + } } } @@ -192,6 +205,9 @@ func (p *Pokestop) SetUrl(v null.String) { if p.Url != v { p.Url = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Url") + } } } @@ -199,6 +215,9 @@ func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { if p.LureExpireTimestamp != v { p.LureExpireTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "LureExpireTimestamp") + } } } @@ -206,6 +225,9 @@ func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { if p.LastModifiedTimestamp != v { p.LastModifiedTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "LastModifiedTimestamp") + } } } @@ -213,6 +235,9 @@ func (p *Pokestop) SetEnabled(v null.Bool) { if p.Enabled != v { p.Enabled = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Enabled") + } } } @@ -220,6 +245,9 @@ func (p *Pokestop) SetQuestType(v null.Int) { if p.QuestType != v { p.QuestType = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestType") + } } } @@ -227,6 +255,9 @@ func (p *Pokestop) SetQuestTimestamp(v null.Int) { if p.QuestTimestamp != v { p.QuestTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestTimestamp") + } } } @@ -234,6 +265,9 @@ func (p *Pokestop) SetQuestTarget(v null.Int) { if p.QuestTarget != v { p.QuestTarget = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestTarget") + } } } @@ -241,6 +275,9 @@ func (p *Pokestop) SetQuestConditions(v null.String) { if p.QuestConditions != v { p.QuestConditions = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestConditions") + } } } @@ -248,6 +285,9 @@ func (p *Pokestop) SetQuestRewards(v null.String) { if p.QuestRewards != v { p.QuestRewards = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestRewards") + } } } @@ -255,6 +295,9 @@ func (p *Pokestop) SetQuestTemplate(v null.String) { if p.QuestTemplate != v { p.QuestTemplate = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestTemplate") + } } } @@ -262,6 +305,9 @@ func (p *Pokestop) SetQuestTitle(v null.String) { if p.QuestTitle != v { p.QuestTitle = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestTitle") + } } } @@ -269,6 +315,9 @@ func (p *Pokestop) SetQuestExpiry(v null.Int) { if p.QuestExpiry != v { p.QuestExpiry = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestExpiry") + } } } @@ -276,6 +325,9 @@ func (p *Pokestop) SetCellId(v null.Int) { if p.CellId != v { p.CellId = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "CellId") + } } } @@ -283,6 +335,9 @@ func (p *Pokestop) SetDeleted(v bool) { if p.Deleted != v { p.Deleted = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Deleted") + } } } @@ -290,6 +345,9 @@ func (p *Pokestop) SetLureId(v int16) { if p.LureId != v { p.LureId = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "LureId") + } } } @@ -297,6 +355,9 @@ func (p *Pokestop) SetFirstSeenTimestamp(v int16) { if p.FirstSeenTimestamp != v { p.FirstSeenTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "FirstSeenTimestamp") + } } } @@ -304,6 +365,9 @@ func (p *Pokestop) SetSponsorId(v null.Int) { if p.SponsorId != v { p.SponsorId = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "SponsorId") + } } } @@ -311,6 +375,9 @@ func (p *Pokestop) SetPartnerId(v null.String) { if p.PartnerId != v { p.PartnerId = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "PartnerId") + } } } @@ -318,6 +385,9 @@ func (p *Pokestop) SetArScanEligible(v null.Int) { if p.ArScanEligible != v { p.ArScanEligible = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ArScanEligible") + } } } @@ -325,6 +395,9 @@ func (p *Pokestop) SetPowerUpLevel(v null.Int) { if p.PowerUpLevel != v { p.PowerUpLevel = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "PowerUpLevel") + } } } @@ -332,6 +405,9 @@ func (p *Pokestop) SetPowerUpPoints(v null.Int) { if p.PowerUpPoints != v { p.PowerUpPoints = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "PowerUpPoints") + } } } @@ -339,6 +415,9 @@ func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { if p.PowerUpEndTimestamp != v { p.PowerUpEndTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "PowerUpEndTimestamp") + } } } @@ -346,6 +425,9 @@ func (p *Pokestop) SetAlternativeQuestType(v null.Int) { if p.AlternativeQuestType != v { p.AlternativeQuestType = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestType") + } } } @@ -353,6 +435,9 @@ func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { if p.AlternativeQuestTimestamp != v { p.AlternativeQuestTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestTimestamp") + } } } @@ -360,6 +445,9 @@ func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { if p.AlternativeQuestTarget != v { p.AlternativeQuestTarget = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestTarget") + } } } @@ -367,6 +455,9 @@ func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { if p.AlternativeQuestConditions != v { p.AlternativeQuestConditions = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestConditions") + } } } @@ -374,6 +465,9 @@ func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { if p.AlternativeQuestRewards != v { p.AlternativeQuestRewards = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestRewards") + } } } @@ -381,6 +475,9 @@ func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { if p.AlternativeQuestTemplate != v { p.AlternativeQuestTemplate = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestTemplate") + } } } @@ -388,6 +485,9 @@ func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { if p.AlternativeQuestTitle != v { p.AlternativeQuestTitle = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestTitle") + } } } @@ -395,6 +495,9 @@ func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { if p.AlternativeQuestExpiry != v { p.AlternativeQuestExpiry = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestExpiry") + } } } @@ -402,6 +505,9 @@ func (p *Pokestop) SetDescription(v null.String) { if p.Description != v { p.Description = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Description") + } } } @@ -409,6 +515,9 @@ func (p *Pokestop) SetShowcaseFocus(v null.String) { if p.ShowcaseFocus != v { p.ShowcaseFocus = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcaseFocus") + } } } @@ -416,6 +525,9 @@ func (p *Pokestop) SetShowcasePokemon(v null.Int) { if p.ShowcasePokemon != v { p.ShowcasePokemon = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcasePokemon") + } } } @@ -423,6 +535,9 @@ func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { if p.ShowcasePokemonForm != v { p.ShowcasePokemonForm = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcasePokemonForm") + } } } @@ -430,6 +545,9 @@ func (p *Pokestop) SetShowcasePokemonType(v null.Int) { if p.ShowcasePokemonType != v { p.ShowcasePokemonType = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcasePokemonType") + } } } @@ -437,6 +555,9 @@ func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { if p.ShowcaseRankingStandard != v { p.ShowcaseRankingStandard = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcaseRankingStandard") + } } } @@ -444,6 +565,9 @@ func (p *Pokestop) SetShowcaseExpiry(v null.Int) { if p.ShowcaseExpiry != v { p.ShowcaseExpiry = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcaseExpiry") + } } } @@ -451,6 +575,9 @@ func (p *Pokestop) SetShowcaseRankings(v null.String) { if p.ShowcaseRankings != v { p.ShowcaseRankings = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcaseRankings") + } } } @@ -1106,6 +1233,9 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop pokestop.Updated = now if pokestop.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO pokestop ( id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, @@ -1138,6 +1268,9 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop _ = res } else { // Existing record - UPDATE + if dbDebugEnabled { + dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE pokestop SET lat = :lat, @@ -1193,8 +1326,15 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop _ = res } //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) - pokestop.newRecord = false // After saving, it's no longer a new record + if dbDebugEnabled { + pokestop.changedFields = pokestop.changedFields[:0] + } + if pokestop.IsNewRecord() { + pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + pokestop.newRecord = false + } pokestop.ClearDirty() + createPokestopWebhooks(pokestop) createPokestopFortWebhooks(pokestop) } diff --git a/decoder/routes.go b/decoder/routes.go index eb7565be..8407e963 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -41,8 +41,9 @@ type Route struct { Version int64 `db:"version"` Waypoints string `db:"waypoints"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) } // IsDirty returns true if any field has been modified @@ -66,6 +67,9 @@ func (r *Route) SetName(v string) { if r.Name != v { r.Name = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Name") + } } } @@ -73,6 +77,9 @@ func (r *Route) SetShortcode(v string) { if r.Shortcode != v { r.Shortcode = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Shortcode") + } } } @@ -80,6 +87,9 @@ func (r *Route) SetDescription(v string) { if r.Description != v { r.Description = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Description") + } } } @@ -87,6 +97,9 @@ func (r *Route) SetDistanceMeters(v int64) { if r.DistanceMeters != v { r.DistanceMeters = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "DistanceMeters") + } } } @@ -94,6 +107,9 @@ func (r *Route) SetDurationSeconds(v int64) { if r.DurationSeconds != v { r.DurationSeconds = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "DurationSeconds") + } } } @@ -101,6 +117,9 @@ func (r *Route) SetEndFortId(v string) { if r.EndFortId != v { r.EndFortId = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "EndFortId") + } } } @@ -108,6 +127,9 @@ func (r *Route) SetEndImage(v string) { if r.EndImage != v { r.EndImage = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "EndImage") + } } } @@ -115,6 +137,9 @@ func (r *Route) SetEndLat(v float64) { if !floatAlmostEqual(r.EndLat, v, floatTolerance) { r.EndLat = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "EndLat") + } } } @@ -122,6 +147,9 @@ func (r *Route) SetEndLon(v float64) { if !floatAlmostEqual(r.EndLon, v, floatTolerance) { r.EndLon = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "EndLon") + } } } @@ -129,6 +157,9 @@ func (r *Route) SetImage(v string) { if r.Image != v { r.Image = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Image") + } } } @@ -136,6 +167,9 @@ func (r *Route) SetImageBorderColor(v string) { if r.ImageBorderColor != v { r.ImageBorderColor = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "ImageBorderColor") + } } } @@ -143,6 +177,9 @@ func (r *Route) SetReversible(v bool) { if r.Reversible != v { r.Reversible = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Reversible") + } } } @@ -150,6 +187,9 @@ func (r *Route) SetStartFortId(v string) { if r.StartFortId != v { r.StartFortId = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "StartFortId") + } } } @@ -157,6 +197,9 @@ func (r *Route) SetStartImage(v string) { if r.StartImage != v { r.StartImage = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "StartImage") + } } } @@ -164,6 +207,9 @@ func (r *Route) SetStartLat(v float64) { if !floatAlmostEqual(r.StartLat, v, floatTolerance) { r.StartLat = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "StartLat") + } } } @@ -171,6 +217,9 @@ func (r *Route) SetStartLon(v float64) { if !floatAlmostEqual(r.StartLon, v, floatTolerance) { r.StartLon = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "StartLon") + } } } @@ -178,6 +227,9 @@ func (r *Route) SetTags(v null.String) { if r.Tags != v { r.Tags = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Tags") + } } } @@ -185,6 +237,9 @@ func (r *Route) SetType(v int8) { if r.Type != v { r.Type = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Type") + } } } @@ -192,6 +247,9 @@ func (r *Route) SetVersion(v int64) { if r.Version != v { r.Version = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Version") + } } } @@ -199,6 +257,9 @@ func (r *Route) SetWaypoints(v string) { if r.Waypoints != v { r.Waypoints = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Waypoints") + } } } @@ -241,6 +302,9 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { route.Updated = time.Now().Unix() if route.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Route", route.Id, route.changedFields) + } _, err := db.GeneralDb.NamedExec( ` INSERT INTO route ( @@ -270,6 +334,9 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { return fmt.Errorf("insert route error: %w", err) } } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) + } _, err := db.GeneralDb.NamedExec( ` UPDATE route SET @@ -304,9 +371,14 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { } } + if dbDebugEnabled { + route.changedFields = route.changedFields[:0] + } route.ClearDirty() - route.newRecord = false - //routeCache.Set(route.Id, route, ttlcache.DefaultTTL) + if route.IsNewRecord() { + routeCache.Set(route.Id, route, ttlcache.DefaultTTL) + route.newRecord = false + } return nil } diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 07c9c311..5dca4187 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -2,11 +2,14 @@ package decoder import ( "context" + "strconv" + "strings" "time" "golbat/db" "github.com/golang/geo/s2" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -49,6 +52,8 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { s2Cell.Latitude = mapS2Cell.CapBound().RectBound().Center().Lat.Degrees() s2Cell.Longitude = mapS2Cell.CapBound().RectBound().Center().Lng.Degrees() s2Cell.Level = null.IntFrom(int64(mapS2Cell.Level())) + + s2CellCache.Set(s2Cell.Id, s2Cell, ttlcache.DefaultTTL) } s2Cell.Updated = now @@ -59,6 +64,14 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { return } + if dbDebugEnabled { + var updatedCells []string + for _, s2cell := range outputCellIds { + updatedCells = append(updatedCells, strconv.FormatUint(s2cell.Id, 10)) + } + log.Debugf("[DB_S2CELL] Updated cells: %s", strings.Join(updatedCells, ",")) + } + // run bulk query _, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO s2cell (id, center_lat, center_lon, level, updated) diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index e4a70fcb..7639f4aa 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -3,11 +3,12 @@ package decoder import ( "context" "database/sql" - "golbat/db" - "golbat/pogo" "strconv" "time" + "golbat/db" + "golbat/pogo" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" @@ -194,8 +195,10 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi } spawnpoint.ClearDirty() - spawnpoint.newRecord = false - spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) + if spawnpoint.IsNewRecord() { + spawnpoint.newRecord = false + spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) + } } func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { diff --git a/decoder/station.go b/decoder/station.go index ccbaef43..2dcb6815 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -13,6 +13,7 @@ import ( "golbat/util" "golbat/webhooks" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -50,8 +51,9 @@ type Station struct { TotalStationedGmax null.Int `db:"total_stationed_gmax"` StationedPokemon null.String `db:"stationed_pokemon"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues StationOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -102,6 +104,9 @@ func (station *Station) SetId(v string) { if station.Id != v { station.Id = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Id") + } } } @@ -109,6 +114,9 @@ func (station *Station) SetLat(v float64) { if !floatAlmostEqual(station.Lat, v, floatTolerance) { station.Lat = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Lat") + } } } @@ -116,6 +124,9 @@ func (station *Station) SetLon(v float64) { if !floatAlmostEqual(station.Lon, v, floatTolerance) { station.Lon = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Lon") + } } } @@ -123,6 +134,9 @@ func (station *Station) SetName(v string) { if station.Name != v { station.Name = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Name") + } } } @@ -130,6 +144,9 @@ func (station *Station) SetCellId(v int64) { if station.CellId != v { station.CellId = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "CellId") + } } } @@ -137,6 +154,9 @@ func (station *Station) SetStartTime(v int64) { if station.StartTime != v { station.StartTime = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "StartTime") + } } } @@ -144,6 +164,9 @@ func (station *Station) SetEndTime(v int64) { if station.EndTime != v { station.EndTime = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "EndTime") + } } } @@ -151,6 +174,9 @@ func (station *Station) SetCooldownComplete(v int64) { if station.CooldownComplete != v { station.CooldownComplete = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "CooldownComplete") + } } } @@ -158,6 +184,9 @@ func (station *Station) SetIsBattleAvailable(v bool) { if station.IsBattleAvailable != v { station.IsBattleAvailable = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "IsBattleAvailable") + } } } @@ -165,6 +194,9 @@ func (station *Station) SetIsInactive(v bool) { if station.IsInactive != v { station.IsInactive = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "IsInactive") + } } } @@ -172,6 +204,9 @@ func (station *Station) SetBattleLevel(v null.Int) { if station.BattleLevel != v { station.BattleLevel = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattleLevel") + } } } @@ -179,6 +214,9 @@ func (station *Station) SetBattleStart(v null.Int) { if station.BattleStart != v { station.BattleStart = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattleStart") + } } } @@ -186,6 +224,9 @@ func (station *Station) SetBattleEnd(v null.Int) { if station.BattleEnd != v { station.BattleEnd = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattleEnd") + } } } @@ -193,6 +234,9 @@ func (station *Station) SetBattlePokemonId(v null.Int) { if station.BattlePokemonId != v { station.BattlePokemonId = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonId") + } } } @@ -200,6 +244,9 @@ func (station *Station) SetBattlePokemonForm(v null.Int) { if station.BattlePokemonForm != v { station.BattlePokemonForm = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonForm") + } } } @@ -207,6 +254,9 @@ func (station *Station) SetBattlePokemonCostume(v null.Int) { if station.BattlePokemonCostume != v { station.BattlePokemonCostume = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonCostume") + } } } @@ -214,6 +264,9 @@ func (station *Station) SetBattlePokemonGender(v null.Int) { if station.BattlePokemonGender != v { station.BattlePokemonGender = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonGender") + } } } @@ -221,6 +274,9 @@ func (station *Station) SetBattlePokemonAlignment(v null.Int) { if station.BattlePokemonAlignment != v { station.BattlePokemonAlignment = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonAlignment") + } } } @@ -228,6 +284,9 @@ func (station *Station) SetBattlePokemonBreadMode(v null.Int) { if station.BattlePokemonBreadMode != v { station.BattlePokemonBreadMode = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonBreadMode") + } } } @@ -235,6 +294,9 @@ func (station *Station) SetBattlePokemonMove1(v null.Int) { if station.BattlePokemonMove1 != v { station.BattlePokemonMove1 = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonMove1") + } } } @@ -242,6 +304,9 @@ func (station *Station) SetBattlePokemonMove2(v null.Int) { if station.BattlePokemonMove2 != v { station.BattlePokemonMove2 = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonMove2") + } } } @@ -249,6 +314,9 @@ func (station *Station) SetBattlePokemonStamina(v null.Int) { if station.BattlePokemonStamina != v { station.BattlePokemonStamina = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonStamina") + } } } @@ -256,6 +324,9 @@ func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { if !nullFloatAlmostEqual(station.BattlePokemonCpMultiplier, v, floatTolerance) { station.BattlePokemonCpMultiplier = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonCpMultiplier") + } } } @@ -263,6 +334,9 @@ func (station *Station) SetTotalStationedPokemon(v null.Int) { if station.TotalStationedPokemon != v { station.TotalStationedPokemon = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "TotalStationedPokemon") + } } } @@ -270,6 +344,9 @@ func (station *Station) SetTotalStationedGmax(v null.Int) { if station.TotalStationedGmax != v { station.TotalStationedGmax = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "TotalStationedGmax") + } } } @@ -277,6 +354,9 @@ func (station *Station) SetStationedPokemon(v null.String) { if station.StationedPokemon != v { station.StationedPokemon = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "StationedPokemon") + } } } @@ -343,6 +423,9 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { station.Updated = now if station.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Station", station.Id, station.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) @@ -356,6 +439,9 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { } _, _ = res, err } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Station", station.Id, station.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE station SET @@ -395,9 +481,14 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { _, _ = res, err } + if dbDebugEnabled { + station.changedFields = station.changedFields[:0] + } station.ClearDirty() - station.newRecord = false - //stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + if station.IsNewRecord() { + stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + station.newRecord = false + } createStationWebhooks(station) } diff --git a/decoder/tappable.go b/decoder/tappable.go index 95c8da4b..c497e06b 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -11,6 +11,7 @@ import ( "golbat/db" "golbat/pogo" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -31,8 +32,9 @@ type Tappable struct { ExpireTimestampVerified bool `db:"expire_timestamp_verified" json:"expire_timestamp_verified"` Updated int64 `db:"updated" json:"updated"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) } // IsDirty returns true if any field has been modified @@ -56,6 +58,9 @@ func (ta *Tappable) SetLat(v float64) { if !floatAlmostEqual(ta.Lat, v, floatTolerance) { ta.Lat = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Lat") + } } } @@ -63,6 +68,9 @@ func (ta *Tappable) SetLon(v float64) { if !floatAlmostEqual(ta.Lon, v, floatTolerance) { ta.Lon = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Lon") + } } } @@ -70,6 +78,9 @@ func (ta *Tappable) SetFortId(v null.String) { if ta.FortId != v { ta.FortId = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "FortId") + } } } @@ -77,6 +88,9 @@ func (ta *Tappable) SetSpawnId(v null.Int) { if ta.SpawnId != v { ta.SpawnId = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "SpawnId") + } } } @@ -84,6 +98,9 @@ func (ta *Tappable) SetType(v string) { if ta.Type != v { ta.Type = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Type") + } } } @@ -91,6 +108,9 @@ func (ta *Tappable) SetEncounter(v null.Int) { if ta.Encounter != v { ta.Encounter = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Encounter") + } } } @@ -98,6 +118,9 @@ func (ta *Tappable) SetItemId(v null.Int) { if ta.ItemId != v { ta.ItemId = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "ItemId") + } } } @@ -105,6 +128,9 @@ func (ta *Tappable) SetCount(v null.Int) { if ta.Count != v { ta.Count = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Count") + } } } @@ -112,6 +138,9 @@ func (ta *Tappable) SetExpireTimestamp(v null.Int) { if ta.ExpireTimestamp != v { ta.ExpireTimestamp = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "ExpireTimestamp") + } } } @@ -119,6 +148,9 @@ func (ta *Tappable) SetExpireTimestampVerified(v bool) { if ta.ExpireTimestampVerified != v { ta.ExpireTimestampVerified = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "ExpireTimestampVerified") + } } } @@ -254,6 +286,9 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap tappable.Updated = now if tappable.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` INSERT INTO tappable ( id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated @@ -268,6 +303,9 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } _ = res } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` UPDATE tappable SET lat = :lat, @@ -290,9 +328,14 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } _ = res } + if dbDebugEnabled { + tappable.changedFields = tappable.changedFields[:0] + } tappable.ClearDirty() - tappable.newRecord = false - //tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) + if tappable.IsNewRecord() { + tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) + tappable.newRecord = false + } } func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { diff --git a/decoder/weather.go b/decoder/weather.go index 8381de26..15001a5e 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -9,6 +9,7 @@ import ( "golbat/webhooks" "github.com/golang/geo/s2" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -340,6 +341,8 @@ func saveWeatherRecord(ctx context.Context, db db.DbDetails, weather *Weather) { } createWeatherWebhooks(weather) weather.ClearDirty() - weather.newRecord = false - //weatherCache.Set(weather.Id, weather, ttlcache.DefaultTTL) + if weather.IsNewRecord() { + weatherCache.Set(weather.Id, weather, ttlcache.DefaultTTL) + weather.newRecord = false + } } From 33e6e2e4acd2223a06723b907834bc27b4655074 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 21:41:50 +0000 Subject: [PATCH 10/78] Add to cache on get --- decoder/spawnpoint.go | 2 ++ decoder/station.go | 1 + decoder/tappable.go | 2 ++ 3 files changed, 5 insertions(+) diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 7639f4aa..9d411a2f 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -124,6 +124,8 @@ func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int6 return &Spawnpoint{Id: spawnpointId}, err } + spawnpointCache.Set(spawnpointId, &spawnpoint, ttlcache.DefaultTTL) + return &spawnpoint, nil } diff --git a/decoder/station.go b/decoder/station.go index 2dcb6815..68a76ce5 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -406,6 +406,7 @@ func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (* if err != nil { return nil, err } + stationCache.Set(stationId, &station, ttlcache.DefaultTTL) station.snapshotOldValues() return &station, nil } diff --git a/decoder/tappable.go b/decoder/tappable.go index c497e06b..cf089a71 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -273,6 +273,8 @@ func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappab if err != nil { return nil, err } + + tappableCache.Set(id, &tappable, ttlcache.DefaultTTL) return &tappable, nil } From 7db7d36fcfe00bf733789d527bf9d3d68a5d368c Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 22:02:51 +0000 Subject: [PATCH 11/78] Add to cache on get incident --- decoder/incident.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/decoder/incident.go b/decoder/incident.go index dbc0b067..8eb65949 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -5,6 +5,7 @@ import ( "database/sql" "time" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" null "gopkg.in/guregu/null.v4" @@ -231,6 +232,7 @@ func getIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string) return nil, err } + incidentCache.Set(incidentId, &incident, ttlcache.DefaultTTL) incident.snapshotOldValues() return &incident, nil } From af8b3ef287914702d5e310526b0b30031793539b Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 29 Jan 2026 07:11:36 +0000 Subject: [PATCH 12/78] Add weather to cache on read --- decoder/weather.go | 1 + 1 file changed, 1 insertion(+) diff --git a/decoder/weather.go b/decoder/weather.go index 15001a5e..7aab48a5 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -210,6 +210,7 @@ func getWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*W weather.UpdatedMs *= 1000 weather.snapshotOldValues() + weatherCache.Set(weatherId, &weather, ttlcache.DefaultTTL) return &weather, nil } From f89eab36d4bbc26c7e3809b43ef287828638439d Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 08:39:10 +0000 Subject: [PATCH 13/78] copilot comments --- decoder/gym.go | 142 +++++++++++++++++++++++--------------------- decoder/pokestop.go | 7 ++- decoder/station.go | 2 +- 3 files changed, 80 insertions(+), 71 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index b3bdcaf6..d89ff26a 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -68,7 +68,8 @@ type Gym struct { Defenders null.String `db:"defenders" json:"defenders"` Rsvps null.String `db:"rsvps" json:"rsvps"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving (to db) + internalDirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving (in memory only) newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) @@ -143,9 +144,15 @@ func (gym *Gym) IsDirty() bool { return gym.dirty } +// IsInternalDirty returns true if any field has been modified for in-memory +func (gym *Gym) IsInternalDirty() bool { + return gym.internalDirty +} + // ClearDirty resets the dirty flag (call after saving to DB) func (gym *Gym) ClearDirty() { gym.dirty = false + gym.internalDirty = false } // IsNewRecord returns true if this is a new record (not yet in DB) @@ -348,7 +355,7 @@ func (gym *Gym) SetInBattle(v null.Int) { if gym.InBattle != v { gym.InBattle = v //Do not set to dirty, as don't trigger an update - //gym.dirty = true + gym.internalDirty = true } } @@ -546,7 +553,7 @@ func (gym *Gym) SetDefenders(v null.String) { if gym.Defenders != v { gym.Defenders = v //Do not set to dirty, as don't trigger an update - //gym.dirty = true + gym.internalDirty = true } } @@ -1095,7 +1102,7 @@ func createGymWebhooks(gym *Gym, areas []geo.AreaName) { func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { now := time.Now().Unix() - if !gym.IsNewRecord() && !gym.IsDirty() { + if !gym.IsNewRecord() && !gym.IsDirty() && !gym.IsInternalDirty() { if gym.Updated > now-900 { // if a gym is unchanged, but we did see it again after 15 minutes, then save again return @@ -1103,71 +1110,73 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { } gym.Updated = now - if gym.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ - "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) + if gym.IsDirty() { + if gym.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ + "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) - statsCollector.IncDbQuery("insert gym", err) - if err != nil { - log.Errorf("insert gym: %s", err) - return - } + statsCollector.IncDbQuery("insert gym", err) + if err != nil { + log.Errorf("insert gym: %s", err) + return + } - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ - "lat = :lat, "+ - "lon = :lon, "+ - "name = :name, "+ - "url = :url, "+ - "last_modified_timestamp = :last_modified_timestamp, "+ - "raid_end_timestamp = :raid_end_timestamp, "+ - "raid_spawn_timestamp = :raid_spawn_timestamp, "+ - "raid_battle_timestamp = :raid_battle_timestamp, "+ - "updated = :updated, "+ - "raid_pokemon_id = :raid_pokemon_id, "+ - "guarding_pokemon_id = :guarding_pokemon_id, "+ - "guarding_pokemon_display = :guarding_pokemon_display, "+ - "available_slots = :available_slots, "+ - "team_id = :team_id, "+ - "raid_level = :raid_level, "+ - "enabled = :enabled, "+ - "ex_raid_eligible = :ex_raid_eligible, "+ - "in_battle = :in_battle, "+ - "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ - "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ - "raid_pokemon_form = :raid_pokemon_form, "+ - "raid_pokemon_alignment = :raid_pokemon_alignment, "+ - "raid_pokemon_cp = :raid_pokemon_cp, "+ - "raid_is_exclusive = :raid_is_exclusive, "+ - "cell_id = :cell_id, "+ - "deleted = :deleted, "+ - "total_cp = :total_cp, "+ - "raid_pokemon_gender = :raid_pokemon_gender, "+ - "sponsor_id = :sponsor_id, "+ - "partner_id = :partner_id, "+ - "raid_pokemon_costume = :raid_pokemon_costume, "+ - "raid_pokemon_evolution = :raid_pokemon_evolution, "+ - "ar_scan_eligible = :ar_scan_eligible, "+ - "power_up_level = :power_up_level, "+ - "power_up_points = :power_up_points, "+ - "power_up_end_timestamp = :power_up_end_timestamp,"+ - "description = :description,"+ - "defenders = :defenders,"+ - "rsvps = :rsvps "+ - "WHERE id = :id", gym, - ) - statsCollector.IncDbQuery("update gym", err) - if err != nil { - log.Errorf("Update gym %s", err) + _, _ = res, err + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ + "lat = :lat, "+ + "lon = :lon, "+ + "name = :name, "+ + "url = :url, "+ + "last_modified_timestamp = :last_modified_timestamp, "+ + "raid_end_timestamp = :raid_end_timestamp, "+ + "raid_spawn_timestamp = :raid_spawn_timestamp, "+ + "raid_battle_timestamp = :raid_battle_timestamp, "+ + "updated = :updated, "+ + "raid_pokemon_id = :raid_pokemon_id, "+ + "guarding_pokemon_id = :guarding_pokemon_id, "+ + "guarding_pokemon_display = :guarding_pokemon_display, "+ + "available_slots = :available_slots, "+ + "team_id = :team_id, "+ + "raid_level = :raid_level, "+ + "enabled = :enabled, "+ + "ex_raid_eligible = :ex_raid_eligible, "+ + "in_battle = :in_battle, "+ + "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ + "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ + "raid_pokemon_form = :raid_pokemon_form, "+ + "raid_pokemon_alignment = :raid_pokemon_alignment, "+ + "raid_pokemon_cp = :raid_pokemon_cp, "+ + "raid_is_exclusive = :raid_is_exclusive, "+ + "cell_id = :cell_id, "+ + "deleted = :deleted, "+ + "total_cp = :total_cp, "+ + "raid_pokemon_gender = :raid_pokemon_gender, "+ + "sponsor_id = :sponsor_id, "+ + "partner_id = :partner_id, "+ + "raid_pokemon_costume = :raid_pokemon_costume, "+ + "raid_pokemon_evolution = :raid_pokemon_evolution, "+ + "ar_scan_eligible = :ar_scan_eligible, "+ + "power_up_level = :power_up_level, "+ + "power_up_points = :power_up_points, "+ + "power_up_end_timestamp = :power_up_end_timestamp,"+ + "description = :description,"+ + "defenders = :defenders,"+ + "rsvps = :rsvps "+ + "WHERE id = :id", gym, + ) + statsCollector.IncDbQuery("update gym", err) + if err != nil { + log.Errorf("Update gym %s", err) + } + _, _ = res, err } - _, _ = res, err } //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) @@ -1183,7 +1192,6 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { gym.newRecord = false } gym.ClearDirty() - } func updateGymGetMapFortCache(gym *Gym, skipName bool) { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index deb829f6..b5139d98 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1207,7 +1207,7 @@ func createPokestopWebhooks(stop *Pokestop) { ArScanEligible: stop.ArScanEligible.ValueOrZero(), PowerUpLevel: stop.PowerUpLevel.ValueOrZero(), PowerUpPoints: stop.PowerUpPoints.ValueOrZero(), - PowerUpEndTimestamp: stop.PowerUpPoints.ValueOrZero(), + PowerUpEndTimestamp: stop.PowerUpEndTimestamp.ValueOrZero(), Updated: stop.Updated, ShowcaseFocus: stop.ShowcaseFocus, ShowcasePokemonId: stop.ShowcasePokemon, @@ -1329,14 +1329,15 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop if dbDebugEnabled { pokestop.changedFields = pokestop.changedFields[:0] } + + createPokestopWebhooks(pokestop) + createPokestopFortWebhooks(pokestop) if pokestop.IsNewRecord() { pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) pokestop.newRecord = false } pokestop.ClearDirty() - createPokestopWebhooks(pokestop) - createPokestopFortWebhooks(pokestop) } func updatePokestopGetMapFortCache(pokestop *Pokestop) { diff --git a/decoder/station.go b/decoder/station.go index 68a76ce5..029fe168 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -486,11 +486,11 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { station.changedFields = station.changedFields[:0] } station.ClearDirty() + createStationWebhooks(station) if station.IsNewRecord() { stationCache.Set(station.Id, station, ttlcache.DefaultTTL) station.newRecord = false } - createStationWebhooks(station) } func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, cellId uint64) *Station { From f4b37140da783757513edba2f2ebcfbfdff4f208 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 14:08:34 +0000 Subject: [PATCH 14/78] switch to internal pokemon lock --- decoder/api_pokemon.go | 42 ++++++---- decoder/api_pokemon_scan_v1.go | 6 +- decoder/api_pokemon_scan_v2.go | 40 +++++----- decoder/api_pokemon_scan_v3.go | 42 +++++----- decoder/main.go | 104 +++++++++++------------- decoder/pending_pokemon.go | 8 +- decoder/pokemon.go | 142 ++++++++++++++++++++++----------- decoder/sharded_cache.go | 10 +++ decoder/weather_iv.go | 20 ++--- 9 files changed, 230 insertions(+), 184 deletions(-) diff --git a/decoder/api_pokemon.go b/decoder/api_pokemon.go index c41393af..7e017fdd 100644 --- a/decoder/api_pokemon.go +++ b/decoder/api_pokemon.go @@ -83,9 +83,9 @@ func haversine(start, end geo.Location) float64 { return earthRadiusKm * c } -func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { +func SearchPokemon(request ApiPokemonSearch) ([]*ApiPokemonResult, error) { start := time.Now() - results := make([]*Pokemon, 0, request.Limit) + results := make([]uint64, 0, request.Limit) pokemonMatched := 0 if request.SearchIds == nil { @@ -109,6 +109,7 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { if maxDistance == 0 { maxDistance = 10 } + pokemonTree2.Nearby( rtree.BoxDist[float64, uint64]([2]float64{request.Center.Longitude, request.Center.Latitude}, [2]float64{request.Center.Longitude, request.Center.Latitude}, nil), func(min, max [2]float64, pokemonId uint64, dist float64) bool { @@ -128,15 +129,12 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { found := slices.Contains(request.SearchIds, pokemonLookupItem.PokemonLookup.PokemonId) if found { - if pokemonCacheEntry := getPokemonFromCache(pokemonId); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - results = append(results, pokemon) - pokemonMatched++ - - if pokemonMatched > maxPokemon { - log.Infof("SearchPokemon - result would exceed maximum size (%d), stopping scan", maxPokemon) - return false - } + results = append(results, pokemonId) + pokemonMatched++ + + if pokemonMatched > maxPokemon { + log.Infof("SearchPokemon - result would exceed maximum size (%d), stopping scan", maxPokemon) + return false } } @@ -145,15 +143,29 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { ) log.Infof("SearchPokemon - scanned %d pokemon, total time %s, %d returned", pokemonScanned, time.Since(start), len(results)) - return results, nil + + apiResults := make([]*ApiPokemonResult, 0, len(results)) + + for _, encounterId := range results { + pokemon, unlock, _ := peekPokemonRecordReadOnly(encounterId) + if pokemon != nil { + apiPokemon := buildApiPokemonResult(pokemon) + apiResults = append(apiResults, &apiPokemon) + unlock() + } + } + pokemonMatched++ + + return apiResults, nil } // Get one result func GetOnePokemon(pokemonId uint64) *ApiPokemonResult { - if item := getPokemonFromCache(pokemonId); item != nil { - pokemon := item.Value() - apiPokemon := buildApiPokemonResult(pokemon) + item, unlock, _ := peekPokemonRecordReadOnly(pokemonId) + if item != nil { + apiPokemon := buildApiPokemonResult(item) + defer unlock() return &apiPokemon } return nil diff --git a/decoder/api_pokemon_scan_v1.go b/decoder/api_pokemon_scan_v1.go index c4baf317..a3de7d4b 100644 --- a/decoder/api_pokemon_scan_v1.go +++ b/decoder/api_pokemon_scan_v1.go @@ -227,10 +227,10 @@ func GetPokemonInArea(retrieveParameters ApiPokemonScan) []*ApiPokemonResult { results := make([]*ApiPokemonResult, 0, len(returnKeys)) for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { apiPokemon := buildApiPokemonResult(pokemon) + unlock() results = append(results, &apiPokemon) } } diff --git a/decoder/api_pokemon_scan_v2.go b/decoder/api_pokemon_scan_v2.go index 726d0f3f..ae029c90 100644 --- a/decoder/api_pokemon_scan_v2.go +++ b/decoder/api_pokemon_scan_v2.go @@ -103,16 +103,14 @@ func GetPokemonInArea2(retrieveParameters ApiPokemonScan2) []*ApiPokemonResult { startUnix := start.Unix() for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - - if pokemon.ExpireTimestamp.ValueOrZero() < startUnix { - continue + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { + if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { + apiPokemon := buildApiPokemonResult(pokemon) + results = append(results, &apiPokemon) } + unlock() - apiPokemon := buildApiPokemonResult(pokemon) - - results = append(results, &apiPokemon) } } @@ -185,22 +183,20 @@ func GrpcGetPokemonInArea2(retrieveParameters *pb.PokemonScanRequest) []*pb.Poke startUnix := start.Unix() for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - - if pokemon.ExpireTimestamp.ValueOrZero() < startUnix { - continue - } - - apiPokemon := pb.PokemonDetails{ - Id: pokemon.Id, - PokestopId: pokemon.PokestopId.Ptr(), - SpawnId: pokemon.SpawnId.Ptr(), - Lat: pokemon.Lat, - Lon: pokemon.Lon, + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { + if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { + apiPokemon := pb.PokemonDetails{ + Id: pokemon.Id, + PokestopId: pokemon.PokestopId.Ptr(), + SpawnId: pokemon.SpawnId.Ptr(), + Lat: pokemon.Lat, + Lon: pokemon.Lon, + } + results = append(results, &apiPokemon) } - results = append(results, &apiPokemon) + unlock() } } diff --git a/decoder/api_pokemon_scan_v3.go b/decoder/api_pokemon_scan_v3.go index a7c3b14d..6cb5e220 100644 --- a/decoder/api_pokemon_scan_v3.go +++ b/decoder/api_pokemon_scan_v3.go @@ -110,17 +110,13 @@ func GetPokemonInArea3(retrieveParameters ApiPokemonScan3) *PokemonScan3Result { startUnix := start.Unix() for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - - if pokemon.ExpireTimestamp.ValueOrZero() < startUnix { - examined-- - continue + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { + if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { + apiPokemon := buildApiPokemonResult(pokemon) + results = append(results, &apiPokemon) } - - apiPokemon := buildApiPokemonResult(pokemon) - - results = append(results, &apiPokemon) + unlock() } } @@ -204,22 +200,20 @@ func GrpcGetPokemonInArea3(retrieveParameters *pb.PokemonScanRequestV3) ([]*pb.P startUnix := start.Unix() for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - - if pokemon.ExpireTimestamp.ValueOrZero() < startUnix { - continue - } - - apiPokemon := pb.PokemonDetails{ - Id: pokemon.Id, - PokestopId: pokemon.PokestopId.Ptr(), - SpawnId: pokemon.SpawnId.Ptr(), - Lat: pokemon.Lat, - Lon: pokemon.Lon, + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { + if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { + apiPokemon := pb.PokemonDetails{ + Id: pokemon.Id, + PokestopId: pokemon.PokestopId.Ptr(), + SpawnId: pokemon.SpawnId.Ptr(), + Lat: pokemon.Lat, + Lon: pokemon.Lon, + } + results = append(results, &apiPokemon) } - results = append(results, &apiPokemon) + unlock() } } diff --git a/decoder/main.go b/decoder/main.go index 6877aee4..897a7361 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -78,7 +78,6 @@ var pokestopStripedMutex = stripedmutex.New(1103) var stationStripedMutex = stripedmutex.New(1103) var tappableStripedMutex = intstripedmutex.New(563) var incidentStripedMutex = stripedmutex.New(157) -var pokemonStripedMutex = intstripedmutex.New(1103) var weatherStripedMutex = intstripedmutex.New(157) var routeStripedMutex = stripedmutex.New(157) @@ -101,18 +100,6 @@ func (cl *gohbemLogger) Print(message string) { log.Info("Gohbem - ", message) } -func setPokemonCache(key uint64, value *Pokemon, ttl time.Duration) { - pokemonCache.Set(key, value, ttl) -} - -func getPokemonFromCache(key uint64) *ttlcache.Item[uint64, *Pokemon] { - return pokemonCache.Get(key) -} - -func deletePokemonFromCache(key uint64) { - pokemonCache.Delete(key) -} - func initDataCache() { // Sharded caches for high-concurrency tables pokestopCache = NewShardedCache(ShardedCacheConfig[string, *Pokestop]{ @@ -399,76 +386,81 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca for _, wild := range wildPokemonList { encounterId := wild.Data.EncounterId - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() + // spawnpointUpdateFromWild doesn't need Pokemon lock spawnpointUpdateFromWild(ctx, db, wild.Data, wild.Timestamp) if scanParameters.ProcessWild { - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + // Use read-only getter - we're only checking if update is needed, then queuing + pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) if err != nil { - log.Errorf("getOrCreatePokemonRecord: %s", err) - } else { - updateTime := wild.Timestamp / 1000 - if pokemon.isNewRecord() || pokemon.wildSignificantUpdate(wild.Data, updateTime) { - // The sweeper will process it after timeout if no encounter arrives - pending := &PendingPokemon{ - EncounterId: encounterId, - WildPokemon: wild.Data, - CellId: int64(wild.Cell), - TimestampMs: wild.Timestamp, - UpdateTime: updateTime, - WeatherLookup: weatherLookup, - Username: username, - } - pokemonPendingQueue.AddPending(pending) + log.Errorf("getPokemonRecordReadOnly: %s", err) + continue + } + + updateTime := wild.Timestamp / 1000 + shouldQueue := pokemon == nil || pokemon.wildSignificantUpdate(wild.Data, updateTime) + + if unlock != nil { + unlock() + } + + if shouldQueue { + // The sweeper will process it after timeout if no encounter arrives + pending := &PendingPokemon{ + EncounterId: encounterId, + WildPokemon: wild.Data, + CellId: int64(wild.Cell), + TimestampMs: wild.Timestamp, + UpdateTime: updateTime, + WeatherLookup: weatherLookup, + Username: username, } + pokemonPendingQueue.AddPending(pending) } } - pokemonMutex.Unlock() } if scanParameters.ProcessNearby { for _, nearby := range nearbyPokemonList { encounterId := nearby.Data.EncounterId - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Printf("getOrCreatePokemonRecord: %s", err) - } else { - updateTime := nearby.Timestamp / 1000 - if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { - pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) - } + continue + } + + updateTime := nearby.Timestamp / 1000 + if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { + pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) } - pokemonMutex.Unlock() + unlock() } } for _, mapPokemon := range mapPokemonList { encounterId := mapPokemon.Data.EncounterId - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Printf("getOrCreatePokemonRecord: %s", err) - } else { - pokemon.updateFromMap(ctx, db, mapPokemon.Data, int64(mapPokemon.Cell), weatherLookup, mapPokemon.Timestamp, username) - storedDiskEncounter := diskEncounterCache.Get(encounterId) - if storedDiskEncounter != nil { - diskEncounter := storedDiskEncounter.Value() - diskEncounterCache.Delete(encounterId) - pokemon.updatePokemonFromDiskEncounterProto(ctx, db, diskEncounter, username) - //log.Infof("Processed stored disk encounter") - } - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, mapPokemon.Timestamp/1000) + continue } - pokemonMutex.Unlock() + + pokemon.updateFromMap(ctx, db, mapPokemon.Data, int64(mapPokemon.Cell), weatherLookup, mapPokemon.Timestamp, username) + storedDiskEncounter := diskEncounterCache.Get(encounterId) + if storedDiskEncounter != nil { + diskEncounter := storedDiskEncounter.Value() + diskEncounterCache.Delete(encounterId) + pokemon.updatePokemonFromDiskEncounterProto(ctx, db, diskEncounter, username) + //log.Infof("Processed stored disk encounter") + } + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, mapPokemon.Timestamp/1000) + + unlock() } } diff --git a/decoder/pending_pokemon.go b/decoder/pending_pokemon.go index 4a22f070..60c1153f 100644 --- a/decoder/pending_pokemon.go +++ b/decoder/pending_pokemon.go @@ -124,16 +124,12 @@ func (q *PokemonPendingQueue) StartSweeper(ctx context.Context, interval time.Du // processExpired handles pokemon that didn't receive an encounter within the timeout func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.DbDetails, expired []*PendingPokemon) { for _, p := range expired { - pokemonMutex, _ := pokemonStripedMutex.GetLock(p.EncounterId) - pokemonMutex.Lock() - processCtx, cancel := context.WithTimeout(ctx, 3*time.Second) - pokemon, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) if err != nil { log.Errorf("getOrCreatePokemonRecord in sweeper: %s", err) cancel() - pokemonMutex.Unlock() continue } @@ -145,8 +141,8 @@ func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.D savePokemonRecordAsAtTime(processCtx, dbDetails, pokemon, false, true, true, p.UpdateTime) } + unlock() cancel() - pokemonMutex.Unlock() } } diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9399bd2a..222b2fed 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -35,6 +35,8 @@ import ( // // FirstSeenTimestamp: This field is used in IsNewRecord. It should only be set in savePokemonRecord. type Pokemon struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id uint64 `db:"id" json:"id,string"` PokestopId null.String `db:"pokestop_id" json:"pokestop_id"` SpawnId null.Int `db:"spawn_id" json:"spawn_id"` @@ -170,6 +172,16 @@ func (pokemon *Pokemon) snapshotOldValues() { } } +// Lock acquires the Pokemon's mutex +func (pokemon *Pokemon) Lock() { + pokemon.mu.Lock() +} + +// Unlock releases the Pokemon's mutex +func (pokemon *Pokemon) Unlock() { + pokemon.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (pokemon *Pokemon) SetPokestopId(v null.String) { @@ -469,54 +481,90 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { } } -func getPokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, error) { - if db.UsePokemonCache { - inMemoryPokemon := getPokemonFromCache(encounterId) - if inMemoryPokemon != nil { - pokemon := inMemoryPokemon.Value() - pokemon.snapshotOldValues() // Snapshot for webhook comparison - return pokemon, nil - } +// peekPokemonRecordReadOnly acquires lock, does NOT take snapshot. +// Use for read-only checks which will not cause a backing database lookup +// Caller must use returned unlock function +func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { + if item := pokemonCache.Get(encounterId); item != nil { + pokemon := item.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil } + + return nil, nil, nil +} + +// getPokemonRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks, but will cause a backing database lookup +// Caller MUST call returned unlock function. +func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + // If we are in-memory only, this is identical to peek if config.Config.PokemonMemoryOnly { - return nil, nil + return peekPokemonRecordReadOnly(encounterId) } - pokemon := Pokemon{} - err := db.PokemonDb.GetContext(ctx, &pokemon, + // Check cache first + if item := pokemonCache.Get(encounterId); item != nil { + pokemon := item.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil + } + + dbPokemon := Pokemon{} + err := db.PokemonDb.GetContext(ctx, &dbPokemon, "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) - statsCollector.IncDbQuery("select pokemon", err) - if err == sql.ErrNoRows { - return nil, nil + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil } - if err != nil { - return nil, err + return nil, nil, err } - pokemon.snapshotOldValues() // Snapshot for webhook comparison - if db.UsePokemonCache { - setPokemonCache(encounterId, &pokemon, ttlcache.DefaultTTL) + // Atomically cache the loaded Pokemon - if another goroutine raced us, + // we'll get their Pokemon and use that instead (ensuring same mutex) + pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + // Only called if key doesn't exist - our Pokemon wins + pokemonRtreeUpdatePokemonOnGet(&dbPokemon) + return &dbPokemon + }, ttlcache.DefaultTTL) + + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil +} + +// getPokemonRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Pokemon. +// Caller MUST call returned unlock function. +func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) + if err != nil || pokemon == nil { + return nil, nil, err } - pokemonRtreeUpdatePokemonOnGet(&pokemon) - return &pokemon, nil + pokemon.snapshotOldValues() + return pokemon, unlock, nil } -func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, error) { - pokemon, err := getPokemonRecord(ctx, db, encounterId) +// getOrCreatePokemonRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) if pokemon != nil || err != nil { - return pokemon, err + return pokemon, unlock, err } - pokemon = &Pokemon{Id: encounterId, newRecord: true} - if db.UsePokemonCache { - setPokemonCache(encounterId, pokemon, ttlcache.DefaultTTL) - } - return pokemon, nil + + // Create new Pokemon atomically - function only called if key doesn't exist + pokemon = pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + return &Pokemon{Id: encounterId, newRecord: true} + }, ttlcache.DefaultTTL) + + pokemon.Lock() + pokemon.snapshotOldValues() + return pokemon, func() { pokemon.Unlock() }, nil } // hasChangesPokemon compares two Pokemon structs @@ -662,7 +710,8 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po if err != nil { log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) log.Errorf("Full structure: %+v", pokemon) - deletePokemonFromCache(pokemon.Id) // Force reload of pokemon from database + pokemonCache.Delete(pokemon.Id) + // Force reload of pokemon from database return } @@ -718,7 +767,8 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po if err != nil { log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) log.Errorf("Full structure: %+v", pokemon) - deletePokemonFromCache(pokemon.Id) // Force reload of pokemon from database + pokemonCache.Delete(pokemon.Id) + // Force reload of pokemon from database return } @@ -753,7 +803,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache if db.UsePokemonCache { - setPokemonCache(pokemon.Id, pokemon, pokemon.remainingDuration(now)) + pokemonCache.Set(pokemon.Id, pokemon, pokemon.remainingDuration(now)) } } @@ -1817,15 +1867,12 @@ func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, pokemonPendingQueue.Remove(encounterId) } - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - defer pokemonMutex.Unlock() - - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Errorf("Error pokemon [%d]: %s", encounterId, err) return fmt.Sprintf("Error finding pokemon %s", err) } + defer unlock() pokemon.updatePokemonFromEncounterProto(ctx, db, encounter, username, timestamp) savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, timestamp/1000) @@ -1843,21 +1890,22 @@ func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDeta encounterId := uint64(encounter.Pokemon.PokemonDisplay.DisplayId) - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - defer pokemonMutex.Unlock() - - pokemon, err := getPokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) if err != nil { log.Errorf("Error pokemon [%d]: %s", encounterId, err) return fmt.Sprintf("Error finding pokemon %s", err) } if pokemon == nil || pokemon.isNewRecord() { - // No pokemon found + // No pokemon found - unlock not set when pokemon is nil + if unlock != nil { + unlock() + } diskEncounterCache.Set(encounterId, encounter, ttlcache.DefaultTTL) return fmt.Sprintf("%d Disk encounter without previous GMO - Pokemon stored for later", encounterId) } + defer unlock() + pokemon.updatePokemonFromDiskEncounterProto(ctx, db, encounter, username) savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) // updateEncounterStats() should only be called for encounters, and called @@ -1870,15 +1918,13 @@ func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDeta func UpdatePokemonRecordWithTappableEncounter(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounter *pogo.TappableEncounterProto, username string, timestampMs int64) string { encounterId := request.GetEncounterId() - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - defer pokemonMutex.Unlock() - - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Errorf("Error pokemon [%d]: %s", encounterId, err) return fmt.Sprintf("Error finding pokemon %s", err) } + defer unlock() + pokemon.updatePokemonFromTappableEncounterProto(ctx, db, request, encounter, username, timestampMs) savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) // updateEncounterStats() should only be called for encounters, and called diff --git a/decoder/sharded_cache.go b/decoder/sharded_cache.go index b0bdecab..4ae9ea9f 100644 --- a/decoder/sharded_cache.go +++ b/decoder/sharded_cache.go @@ -96,6 +96,16 @@ func (sc *ShardedCache[K, V]) DeleteAll() { } } +// GetOrSetFunc atomically gets an existing item or creates and sets a new one. +// If key exists, returns existing value (createFunc NOT called). +// If key doesn't exist, calls createFunc to create value, sets it, and returns it. +// This prevents race conditions and avoids creating objects unnecessarily. +func (sc *ShardedCache[K, V]) GetOrSetFunc(key K, createFunc func() V, ttl time.Duration) V { + shard := sc.getShard(key) + item, _ := shard.GetOrSetFunc(key, createFunc, ttlcache.WithTTL[K, V](ttl)) + return item.Value() +} + // --- Key conversion helpers --- // Uint64KeyToShard is the identity function for uint64 keys diff --git a/decoder/weather_iv.go b/decoder/weather_iv.go index 8560c3ec..7a85d256 100644 --- a/decoder/weather_iv.go +++ b/decoder/weather_iv.go @@ -4,13 +4,14 @@ import ( "context" "encoding/json" "errors" - "golbat/db" - "golbat/pogo" "net/http" "os" "reflect" "time" + "golbat/db" + "golbat/pogo" + "github.com/golang/geo/s2" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" @@ -203,7 +204,7 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath pokemonLocked := 0 pokemonUpdated := 0 pokemonCpUpdated := 0 - var pokemon *Pokemon + //var pokemon *Pokemon pokemonTree2.Search([2]float64{cellLo.Lng.Degrees(), cellLo.Lat.Degrees()}, [2]float64{cellHi.Lng.Degrees(), cellHi.Lat.Degrees()}, func(min, max [2]float64, pokemonId uint64) bool { if !weatherCell.ContainsPoint(s2.PointFromLatLng(s2.LatLngFromDegrees(min[1], min[0]))) { return true @@ -224,13 +225,12 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath if int8(newWeather) == pokemonLookup.PokemonLookup.Weather { return true } - pokemonMutex, _ := pokemonStripedMutex.GetLock(pokemonId) - pokemonMutex.Lock() - pokemonLocked++ - pokemonEntry := getPokemonFromCache(pokemonId) - if pokemonEntry != nil { - pokemon = pokemonEntry.Value() + + pokemon, unlock, _ := peekPokemonRecordReadOnly(pokemonId) + if pokemon != nil { + pokemonLocked++ if pokemonLookup.PokemonLookup.PokemonId == pokemon.PokemonId && (pokemon.IsDitto || int64(pokemonLookup.PokemonLookup.Form) == pokemon.Form.ValueOrZero()) && int64(newWeather) != pokemon.Weather.ValueOrZero() && pokemon.ExpireTimestamp.ValueOrZero() >= startUnix && pokemon.Updated.ValueOrZero() < timestamp { + pokemon.snapshotOldValues() pokemon.repopulateIv(int64(newWeather), pokemon.IsStrong.ValueOrZero()) if !pokemon.Cp.Valid { pokemon.Weather = null.IntFrom(int64(newWeather)) @@ -244,8 +244,8 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath } } } + unlock() } - pokemonMutex.Unlock() return true }) if pokemonCpUpdated > 0 { From 9660a71d4e07b04544a2303a1a2fcf4039c62700 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 14:51:49 +0000 Subject: [PATCH 15/78] optimise locking in getOrCreatePokemonRecord --- decoder/pokemon.go | 48 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 222b2fed..5bb753bb 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -494,6 +494,18 @@ func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { return nil, nil, nil } +func loadPokemonFromDatabase(ctx context.Context, db db.DbDetails, encounterId uint64, pokemon *Pokemon) error { + err := db.PokemonDb.GetContext(ctx, pokemon, + "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ + "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ + "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ + "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ + "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) + statsCollector.IncDbQuery("select pokemon", err) + + return err +} + // getPokemonRecordReadOnly acquires lock but does NOT take snapshot. // Use for read-only checks, but will cause a backing database lookup // Caller MUST call returned unlock function. @@ -511,13 +523,7 @@ func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId } dbPokemon := Pokemon{} - err := db.PokemonDb.GetContext(ctx, &dbPokemon, - "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ - "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ - "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ - "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ - "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) - statsCollector.IncDbQuery("select pokemon", err) + err := loadPokemonFromDatabase(ctx, db, encounterId, &dbPokemon) if errors.Is(err, sql.ErrNoRows) { return nil, nil, nil } @@ -552,17 +558,31 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId // getOrCreatePokemonRecord gets existing or creates new, locked with snapshot. // Caller MUST call returned unlock function. func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { - pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) - if pokemon != nil || err != nil { - return pokemon, unlock, err - } - // Create new Pokemon atomically - function only called if key doesn't exist - pokemon = pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { return &Pokemon{Id: encounterId, newRecord: true} }, ttlcache.DefaultTTL) - pokemon.Lock() + + if config.Config.PokemonMemoryOnly { + pokemon.snapshotOldValues() + return pokemon, func() { pokemon.Unlock() }, nil + } + + if pokemon.newRecord { + // We should attempt to load from database + err := loadPokemonFromDatabase(ctx, db, encounterId, pokemon) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + pokemon.Unlock() + return nil, nil, err + } + } else { + // We loaded + pokemon.newRecord = false + } + } + pokemon.snapshotOldValues() return pokemon, func() { pokemon.Unlock() }, nil } From 9e24751c2be0b072be1ad9e62821730e3cd252e3 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 15:03:54 +0000 Subject: [PATCH 16/78] optimise locking in getOrCreatePokemonRecord --- decoder/pokemon.go | 6 ++++-- decoder/sharded_cache.go | 2 +- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 5bb753bb..30789340 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -560,9 +560,10 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { // Create new Pokemon atomically - function only called if key doesn't exist pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - return &Pokemon{Id: encounterId, newRecord: true} + p := &Pokemon{Id: encounterId, newRecord: true} + p.Lock() + return p }, ttlcache.DefaultTTL) - pokemon.Lock() if config.Config.PokemonMemoryOnly { pokemon.snapshotOldValues() @@ -580,6 +581,7 @@ func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId } else { // We loaded pokemon.newRecord = false + pokemonRtreeUpdatePokemonOnGet(pokemon) } } diff --git a/decoder/sharded_cache.go b/decoder/sharded_cache.go index 4ae9ea9f..dced7c27 100644 --- a/decoder/sharded_cache.go +++ b/decoder/sharded_cache.go @@ -121,6 +121,6 @@ func Int64KeyToShard(key int64) uint64 { // StringKeyToShard hashes string keys to uint64 for sharding using FNV-1a func StringKeyToShard(key string) uint64 { h := fnv.New64a() - h.Write([]byte(key)) + _, _ = h.Write([]byte(key)) return h.Sum64() } diff --git a/go.mod b/go.mod index 66ce0646..83952183 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/rtree v1.10.0 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f google.golang.org/grpc v1.74.2 - google.golang.org/protobuf v1.36.7 + google.golang.org/protobuf v1.36.11 gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index 9f2f8889..955a7b27 100644 --- a/go.sum +++ b/go.sum @@ -447,6 +447,8 @@ google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9x google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From ca4f0d6bbe0aced319d0763436b473a304470321 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 15:24:53 +0000 Subject: [PATCH 17/78] optimise locking in getOrCreatePokemonRecord --- decoder/pokemon.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 30789340..7e1651b2 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -560,11 +560,11 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { // Create new Pokemon atomically - function only called if key doesn't exist pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - p := &Pokemon{Id: encounterId, newRecord: true} - p.Lock() - return p + return &Pokemon{Id: encounterId, newRecord: true} }, ttlcache.DefaultTTL) + pokemon.Lock() + if config.Config.PokemonMemoryOnly { pokemon.snapshotOldValues() return pokemon, func() { pokemon.Unlock() }, nil From 08254456e5cfacf46580a1ad27fa317f1d766897 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 16:04:16 +0000 Subject: [PATCH 18/78] Pokestop to new locking model --- decoder/fort.go | 23 +++--- decoder/fortRtree.go | 5 +- decoder/fort_tracker.go | 10 ++- decoder/incident.go | 37 ++++++---- decoder/main.go | 17 ++--- decoder/pokemon.go | 9 ++- decoder/pokestop.go | 157 +++++++++++++++++++++++++++------------- routes.go | 9 ++- 8 files changed, 166 insertions(+), 101 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index 2855d149..9c308a66 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -97,8 +97,6 @@ func InitWebHookFortFromPokestop(stop *Pokestop) *FortWebhook { } func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []string, fortType FortType, change FortChange) { - var gyms []Gym - var stops []Pokestop if fortType == GYM { for _, id := range ids { gym, err := GetGymRecord(ctx, dbDetails, id) @@ -108,29 +106,26 @@ func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []strin if gym == nil { continue } - gyms = append(gyms, *gym) + fort := InitWebHookFortFromGym(gym) + CreateFortWebHooks(fort, &FortWebhook{}, change) } } if fortType == POKESTOP { for _, id := range ids { - stop, err := GetPokestopRecord(ctx, dbDetails, id) + stop, unlock, err := getPokestopRecordReadOnly(ctx, dbDetails, id) if err != nil { continue } if stop == nil { continue } - stops = append(stops, *stop) + + fort := InitWebHookFortFromPokestop(stop) + unlock() + + CreateFortWebHooks(fort, &FortWebhook{}, change) } } - for _, gym := range gyms { - fort := InitWebHookFortFromGym(&gym) - CreateFortWebHooks(fort, &FortWebhook{}, change) - } - for _, stop := range stops { - fort := InitWebHookFortFromPokestop(&stop) - CreateFortWebHooks(fort, &FortWebhook{}, change) - } } func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { @@ -149,7 +144,7 @@ func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { Old: old, } webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) - statsCollector.UpdateFortCount(areas, new.Type, "removal") + statsCollector.UpdateFortCount(areas, old.Type, "removal") } else if change == EDIT { areas := MatchStatsGeofence(new.Location.Latitude, new.Location.Longitude) var editTypes []string diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index ce949d00..8a52792b 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -46,7 +46,10 @@ func LoadAllPokestops(details db.DbDetails) { if err != nil { log.Fatalln(err) } - GetPokestopRecord(context.Background(), details, place.Id) + _, unlock, _ := getPokestopRecordReadOnly(context.Background(), details, place.Id) + if unlock != nil { + unlock() + } } log.Infof("Loaded %d pokestops [finished]", count) } diff --git a/decoder/fort_tracker.go b/decoder/fort_tracker.go index d9839e67..36103b5a 100644 --- a/decoder/fort_tracker.go +++ b/decoder/fort_tracker.go @@ -431,11 +431,13 @@ func clearGymWithLock(ctx context.Context, dbDetails db.DbDetails, gymId string, } } -// clearPokestopWithLock marks a pokestop as deleted while holding the striped mutex +// clearPokestopWithLock marks a pokestop as deleted while holding the object-level mutex func clearPokestopWithLock(ctx context.Context, dbDetails db.DbDetails, stopId string, cellId uint64, removeFromTracker bool) { - pokestopMutex, _ := pokestopStripedMutex.GetLock(stopId) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() + // Lock the pokestop if it exists in cache + pokestop, unlock, _ := PeekPokestopRecord(stopId) + if pokestop != nil { + defer unlock() + } pokestopCache.Delete(stopId) if err := db.ClearOldPokestops(ctx, dbDetails, []string{stopId}); err != nil { diff --git a/decoder/incident.go b/decoder/incident.go index 8eb65949..a432f2e5 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -281,12 +281,14 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident createIncidentWebhooks(ctx, db, incident) - stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId) - if stop == nil { - stop = &Pokestop{} + var stopLat, stopLon float64 + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + stopLat, stopLon = stop.Lat, stop.Lon + unlock() } - areas := MatchStatsGeofence(stop.Lat, stop.Lon) + areas := MatchStatsGeofence(stopLat, stopLon) updateIncidentStats(incident, areas) incident.ClearDirty() @@ -299,14 +301,19 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Inci isNew := incident.IsNewRecord() if isNew || (old.ExpirationTime != incident.ExpirationTime || old.Character != incident.Character || old.Confirmed != incident.Confirmed || old.Slot1PokemonId != incident.Slot1PokemonId) { - stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId) - if stop == nil { - stop = &Pokestop{} + var pokestopName, stopUrl string + var stopLat, stopLon float64 + var stopEnabled bool + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + pokestopName = stop.Name.ValueOrZero() + stopLat, stopLon = stop.Lat, stop.Lon + stopUrl = stop.Url.ValueOrZero() + stopEnabled = stop.Enabled.ValueOrZero() + unlock() } - - pokestopName := "Unknown" - if stop.Name.Valid { - pokestopName = stop.Name.String + if pokestopName == "" { + pokestopName = "Unknown" } var lineup []webhookLineup @@ -333,11 +340,11 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Inci incidentHook := IncidentWebhook{ Id: incident.Id, PokestopId: incident.PokestopId, - Latitude: stop.Lat, - Longitude: stop.Lon, + Latitude: stopLat, + Longitude: stopLon, PokestopName: pokestopName, - Url: stop.Url.ValueOrZero(), - Enabled: stop.Enabled.ValueOrZero(), + Url: stopUrl, + Enabled: stopEnabled, Start: incident.StartTime, IncidentExpireTimestamp: incident.ExpirationTime, Expiration: incident.ExpirationTime, diff --git a/decoder/main.go b/decoder/main.go index 897a7361..4d2324a6 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -74,7 +74,6 @@ var diskEncounterCache *ttlcache.Cache[uint64, *pogo.DiskEncounterOutProto] var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto] var gymStripedMutex = stripedmutex.New(1103) -var pokestopStripedMutex = stripedmutex.New(1103) var stationStripedMutex = stripedmutex.New(1103) var tappableStripedMutex = intstripedmutex.New(563) var incidentStripedMutex = stripedmutex.New(157) @@ -271,19 +270,12 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa for _, fort := range p { fortId := fort.Data.FortId if fort.Data.FortType == pogo.FortType_CHECKPOINT && scanParameters.ProcessPokestops { - pokestopMutex, _ := pokestopStripedMutex.GetLock(fortId) - - pokestopMutex.Lock() - pokestop, err := GetPokestopRecord(ctx, db, fortId) // should check error + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fortId) if err != nil { - log.Errorf("getPokestopRecord: %s", err) - pokestopMutex.Unlock() + log.Errorf("getOrCreatePokestopRecord: %s", err) continue } - if pokestop == nil { - pokestop = &Pokestop{newRecord: true} - } pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) // If this is a new pokestop, check if it was converted from a gym and copy shared fields @@ -295,6 +287,7 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa } savePokestopRecord(ctx, db, pokestop) + unlock() incidents := fort.Data.PokestopDisplays if incidents == nil && fort.Data.PokestopDisplay != nil { @@ -324,7 +317,6 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa incidentMutex.Unlock() } } - pokestopMutex.Unlock() } if fort.Data.FortType == pogo.FortType_GYM && scanParameters.ProcessGyms { @@ -346,9 +338,10 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa // If this is a new gym, check if it was converted from a pokestop and copy shared fields if gym.IsNewRecord() { - pokestop, _ := GetPokestopRecord(ctx, db, fortId) + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, fortId) if pokestop != nil { gym.copySharedFieldsFrom(pokestop) + unlock() } } diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 7e1651b2..e4c079bb 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -884,10 +884,11 @@ func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemo var pokestopName *string if pokemon.PokestopId.Valid { - pokestop, _ := GetPokestopRecord(ctx, db, pokemon.PokestopId.String) + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokemon.PokestopId.String) name := "Unknown" if pokestop != nil { name = pokestop.Name.ValueOrZero() + unlock() } pokestopName = &name } @@ -1086,7 +1087,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP spawnpointId := mapPokemon.SpawnpointId - pokestop, _ := GetPokestopRecord(ctx, db, spawnpointId) + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, spawnpointId) if pokestop == nil { // Unrecognised pokestop return @@ -1095,6 +1096,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP pokemon.SetLat(pokestop.Lat) pokemon.SetLon(pokestop.Lon) pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) + unlock() if mapPokemon.PokemonDisplay != nil { pokemon.setPokemonDisplay(int16(mapPokemon.PokedexTypeId), mapPokemon.PokemonDisplay) @@ -1149,7 +1151,7 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n default: return } - pokestop, _ := GetPokestopRecord(ctx, db, pokestopId) + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokestopId) if pokestop == nil { // Unrecognised pokestop, rollback changes overrideLatLon = pokemon.isNewRecord() @@ -1158,6 +1160,7 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n pokemon.SetPokestopId(null.StringFrom(pokestopId)) lat, lon = pokestop.Lat, pokestop.Lon useCellLatLon = false + unlock() } } if useCellLatLon { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index b5139d98..c0fd7141 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" "github.com/jellydator/ttlcache/v3" @@ -24,6 +25,8 @@ import ( // Pokestop struct. type Pokestop struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` Lon float64 `db:"lon" json:"lon"` @@ -159,6 +162,16 @@ func (p *Pokestop) snapshotOldValues() { } } +// Lock acquires the Pokestop's mutex +func (p *Pokestop) Lock() { + p.mu.Lock() +} + +// Unlock releases the Pokestop's mutex +func (p *Pokestop) Unlock() { + p.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (p *Pokestop) SetId(v string) { @@ -622,16 +635,8 @@ type PokestopWebhook struct { ShowcaseRankings json.RawMessage `json:"showcase_rankings"` } -func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, error) { - stop := pokestopCache.Get(fortId) - if stop != nil { - //log.Debugf("GetPokestopRecord %s (from cache)", fortId) - pokestop := stop.Value() - pokestop.snapshotOldValues() // Snapshot for webhook comparison - return pokestop, nil - } - pokestop := Pokestop{} - err := db.GeneralDb.GetContext(ctx, &pokestop, +func loadPokestopFromDatabase(ctx context.Context, db db.DbDetails, fortId string, pokestop *Pokestop) error { + err := db.GeneralDb.GetContext(ctx, pokestop, `SELECT pokestop.id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, pokestop.updated, quest_type, quest_timestamp, quest_target, quest_conditions, quest_rewards, quest_template, quest_title, @@ -643,22 +648,96 @@ func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Po showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings FROM pokestop WHERE pokestop.id = ? `, fortId) - //log.Debugf("GetPokestopRecord %s (from db)", fortId) - statsCollector.IncDbQuery("select pokestop", err) + return err +} + +// PeekPokestopRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekPokestopRecord(fortId string) (*Pokestop, func(), error) { + if item := pokestopCache.Get(fortId); item != nil { + pokestop := item.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil + } + return nil, nil, nil +} + +// getPokestopRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + // Check cache first + if item := pokestopCache.Get(fortId); item != nil { + pokestop := item.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil + } + + dbPokestop := Pokestop{} + err := loadPokestopFromDatabase(ctx, db, fortId, &dbPokestop) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil, nil } if err != nil { - return nil, err + return nil, nil, err } - pokestop.snapshotOldValues() // Snapshot for webhook comparison - pokestopCache.Set(fortId, &pokestop, ttlcache.DefaultTTL) - if config.Config.TestFortInMemory { - fortRtreeUpdatePokestopOnGet(&pokestop) + // Atomically cache the loaded Pokestop - if another goroutine raced us, + // we'll get their Pokestop and use that instead (ensuring same mutex) + pokestop := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + // Only called if key doesn't exist - our Pokestop wins + if config.Config.TestFortInMemory { + fortRtreeUpdatePokestopOnGet(&dbPokestop) + } + return &dbPokestop + }, ttlcache.DefaultTTL) + + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil +} + +// getPokestopRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Pokestop. +// Caller MUST call returned unlock function if non-nil. +func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + pokestop, unlock, err := getPokestopRecordReadOnly(ctx, db, fortId) + if err != nil || pokestop == nil { + return nil, nil, err } - return &pokestop, nil + pokestop.snapshotOldValues() + return pokestop, unlock, nil +} + +// getOrCreatePokestopRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + // Create new Pokestop atomically - function only called if key doesn't exist + pokestop := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + return &Pokestop{Id: fortId, newRecord: true} + }, ttlcache.DefaultTTL) + + pokestop.Lock() + + if pokestop.newRecord { + // We should attempt to load from database + err := loadPokestopFromDatabase(ctx, db, fortId, pokestop) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + pokestop.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + pokestop.newRecord = false + if config.Config.TestFortInMemory { + fortRtreeUpdatePokestopOnGet(pokestop) + } + } + } + + pokestop.snapshotOldValues() + return pokestop, func() { pokestop.Unlock() }, nil } var LureTime int64 = 1800 @@ -1351,19 +1430,13 @@ func updatePokestopGetMapFortCache(pokestop *Pokestop) { } func UpdatePokestopRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { - pokestopMutex, _ := pokestopStripedMutex.GetLock(fort.Id) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - - pokestop, err := GetPokestopRecord(ctx, db, fort.Id) // should check error + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fort.Id) if err != nil { log.Printf("Update pokestop %s", err) return fmt.Sprintf("Error %s", err) } + defer unlock() - if pokestop == nil { - pokestop = &Pokestop{newRecord: true} - } pokestop.updatePokestopFromFortDetailsProto(fort) updatePokestopGetMapFortCache(pokestop) @@ -1383,19 +1456,14 @@ func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.F } statsCollector.IncDecodeQuest("ok", haveArStr) - pokestopMutex, _ := pokestopStripedMutex.GetLock(quest.FortId) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - pokestop, err := GetPokestopRecord(ctx, db, quest.FortId) + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, quest.FortId) if err != nil { log.Printf("Update quest %s", err) return fmt.Sprintf("error %s", err) } + defer unlock() - if pokestop == nil { - pokestop = &Pokestop{newRecord: true} - } questTitle := pokestop.updatePokestopFromQuestProto(quest, haveAr) updatePokestopGetMapFortCache(pokestop) @@ -1428,11 +1496,7 @@ func GetQuestStatusWithGeofence(dbDetails db.DbDetails, geofence *geojson.Featur } func UpdatePokestopRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { - pokestopMutex, _ := pokestopStripedMutex.GetLock(mapFort.Id) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - - pokestop, err := GetPokestopRecord(ctx, db, mapFort.Id) + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, mapFort.Id) if err != nil { log.Printf("Update pokestop %s", err) return false, fmt.Sprintf("Error %s", err) @@ -1441,6 +1505,7 @@ func UpdatePokestopRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDe if pokestop == nil { return false, "" } + defer unlock() pokestop.updatePokestopFromGetMapFortsOutProto(mapFort) savePokestopRecord(ctx, db, pokestop) @@ -1474,11 +1539,7 @@ func UpdatePokestopWithContestData(ctx context.Context, db db.DbDetails, request contest := contestData.ContestIncident.Contests[0] - pokestopMutex, _ := pokestopStripedMutex.GetLock(fortId) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - - pokestop, err := GetPokestopRecord(ctx, db, fortId) + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) if err != nil { log.Printf("Get pokestop %s", err) return "Error getting pokestop" @@ -1488,6 +1549,7 @@ func UpdatePokestopWithContestData(ctx context.Context, db db.DbDetails, request log.Infof("Contest data for pokestop %s not found", fortId) return fmt.Sprintf("Contest data for pokestop %s not found", fortId) } + defer unlock() pokestop.updatePokestopFromGetContestDataOutProto(contest) savePokestopRecord(ctx, db, pokestop) @@ -1502,11 +1564,7 @@ func getFortIdFromContest(id string) string { func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDetails, request *pogo.GetPokemonSizeLeaderboardEntryProto, contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) string { fortId := getFortIdFromContest(request.GetContestId()) - pokestopMutex, _ := pokestopStripedMutex.GetLock(fortId) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - - pokestop, err := GetPokestopRecord(ctx, db, fortId) + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) if err != nil { log.Printf("Get pokestop %s", err) return "Error getting pokestop" @@ -1516,6 +1574,7 @@ func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDeta log.Infof("Contest data for pokestop %s not found", fortId) return fmt.Sprintf("Contest data for pokestop %s not found", fortId) } + defer unlock() pokestop.updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData) savePokestopRecord(ctx, db, pokestop) diff --git a/routes.go b/routes.go index ca840480..ff1fda73 100644 --- a/routes.go +++ b/routes.go @@ -489,9 +489,12 @@ func GetPokestopPositions(c *gin.Context) { func GetPokestop(c *gin.Context) { fortId := c.Param("fort_id") - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - pokestop, err := decoder.GetPokestopRecord(ctx, dbDetails, fortId) - cancel() + //ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + pokestop, unlock, err := decoder.PeekPokestopRecord(fortId) + if unlock != nil { + defer unlock() + } + //cancel() if err != nil { log.Warnf("GET /api/pokestop/id/:fort_id/ Error during post retrieve %v", err) c.Status(http.StatusInternalServerError) From 642eb3288d5d842daa9e867202b4a4ca54ff4eef Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 16:39:22 +0000 Subject: [PATCH 19/78] Gym locking model changes --- decoder/fort.go | 5 +- decoder/fortRtree.go | 5 +- decoder/fort_tracker.go | 10 ++- decoder/gym.go | 175 ++++++++++++++++++++++++++++----------- decoder/main.go | 18 ++-- decoder/pokemon.go | 10 ++- decoder/pokestop.go | 10 ++- decoder/sharded_cache.go | 15 ++-- 8 files changed, 165 insertions(+), 83 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index 9c308a66..979bc2e1 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -99,14 +99,17 @@ func InitWebHookFortFromPokestop(stop *Pokestop) *FortWebhook { func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []string, fortType FortType, change FortChange) { if fortType == GYM { for _, id := range ids { - gym, err := GetGymRecord(ctx, dbDetails, id) + gym, unlock, err := getGymRecordReadOnly(ctx, dbDetails, id) if err != nil { continue } if gym == nil { continue } + fort := InitWebHookFortFromGym(gym) + unlock() + CreateFortWebHooks(fort, &FortWebhook{}, change) } } diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 8a52792b..14222171 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -71,7 +71,10 @@ func LoadAllGyms(details db.DbDetails) { if err != nil { log.Fatalln(err) } - GetGymRecord(context.Background(), details, place.Id) + _, unlock, _ := getGymRecordReadOnly(context.Background(), details, place.Id) + if unlock != nil { + unlock() + } } log.Infof("Loaded %d gyms [finished]", count) } diff --git a/decoder/fort_tracker.go b/decoder/fort_tracker.go index 36103b5a..b3a2ae0d 100644 --- a/decoder/fort_tracker.go +++ b/decoder/fort_tracker.go @@ -409,11 +409,13 @@ func GetFortTracker() *FortTracker { return fortTracker } -// clearGymWithLock marks a gym as deleted while holding the striped mutex +// clearGymWithLock marks a gym as deleted while holding the object-level mutex func clearGymWithLock(ctx context.Context, dbDetails db.DbDetails, gymId string, cellId uint64, removeFromTracker bool) { - gymMutex, _ := gymStripedMutex.GetLock(gymId) - gymMutex.Lock() - defer gymMutex.Unlock() + // Lock the gym if it exists in cache + gym, unlock, _ := PeekGymRecord(gymId) + if gym != nil { + defer unlock() + } gymCache.Delete(gymId) if err := db.ClearOldGyms(ctx, dbDetails, []string{gymId}); err != nil { diff --git a/decoder/gym.go b/decoder/gym.go index d89ff26a..261cfff6 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -5,9 +5,11 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "slices" "strings" + "sync" "time" "golbat/geo" @@ -26,6 +28,8 @@ import ( // Gym struct. // REMINDER! Keep hasChangesGym updated after making changes type Gym struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` Lon float64 `db:"lon" json:"lon"` @@ -179,6 +183,16 @@ func (gym *Gym) snapshotOldValues() { } } +// Lock acquires the Gym's mutex +func (gym *Gym) Lock() { + gym.mu.Lock() +} + +// Unlock releases the Gym's mutex +func (gym *Gym) Unlock() { + gym.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (gym *Gym) SetId(v string) { @@ -567,31 +581,116 @@ func (gym *Gym) SetRsvps(v null.String) { } } -func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, error) { - inMemoryGym := gymCache.Get(fortId) - if inMemoryGym != nil { - gym := inMemoryGym.Value() - gym.snapshotOldValues() // Snapshot for webhook comparison - return gym, nil +func loadGymFromDatabase(ctx context.Context, db db.DbDetails, fortId string, gym *Gym) error { + err := db.GeneralDb.GetContext(ctx, gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) + statsCollector.IncDbQuery("select gym", err) + return err +} + +// PeekGymRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekGymRecord(fortId string) (*Gym, func(), error) { + if item := gymCache.Get(fortId); item != nil { + gym := item.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil } - gym := Gym{} - err := db.GeneralDb.GetContext(ctx, &gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) + return nil, nil, nil +} - statsCollector.IncDbQuery("select gym", err) - if err == sql.ErrNoRows { - return nil, nil +// getGymRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + // Check cache first + if item := gymCache.Get(fortId); item != nil { + gym := item.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil } + dbGym := Gym{} + err := loadGymFromDatabase(ctx, db, fortId, &dbGym) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return nil, err + return nil, nil, err + } + + // Atomically cache the loaded Gym - if another goroutine raced us, + // we'll get their Gym and use that instead (ensuring same mutex) + existingGym, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { + // Only called if key doesn't exist - our Pokestop wins + if config.Config.TestFortInMemory { + fortRtreeUpdateGymOnGet(&dbGym) + } + return &dbGym + }) + + gym := existingGym.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil +} + +// getGymRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Gym. +// Caller MUST call returned unlock function if non-nil. +func getGymRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + gym, unlock, err := getGymRecordReadOnly(ctx, db, fortId) + if err != nil || gym == nil { + return nil, nil, err + } + gym.snapshotOldValues() + return gym, unlock, nil +} + +// getOrCreateGymRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + // Create new Gym atomically - function only called if key doesn't exist + gymItem, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { + return &Gym{Id: fortId, newRecord: true} + }) + + gym := gymItem.Value() + gym.Lock() + + if gym.newRecord { + // We should attempt to load from database + err := loadGymFromDatabase(ctx, db, fortId, gym) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + gym.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + gym.newRecord = false + if config.Config.TestFortInMemory { + fortRtreeUpdateGymOnGet(gym) + } + } } - gym.snapshotOldValues() // Snapshot for webhook comparison - gymCache.Set(fortId, &gym, ttlcache.DefaultTTL) - if config.Config.TestFortInMemory { - fortRtreeUpdateGymOnGet(&gym) + gym.snapshotOldValues() + return gym, func() { gym.Unlock() }, nil +} + +// GetGymRecord returns a copy of the Gym for external/API use. +// For internal use, prefer getGymRecordReadOnly or getGymRecordForUpdate. +func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, error) { + gym, unlock, err := getGymRecordReadOnly(ctx, db, fortId) + if err != nil { + return nil, err + } + if gym == nil { + return nil, nil } - return &gym, nil + // Make a copy for safe external use + gymCopy := *gym + unlock() + return &gymCopy, nil } func escapeLike(s string) string { @@ -1205,18 +1304,12 @@ func updateGymGetMapFortCache(gym *Gym, skipName bool) { } func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { - gymMutex, _ := gymStripedMutex.GetLock(fort.Id) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, fort.Id) // should check error + gym, unlock, err := getOrCreateGymRecord(ctx, db, fort.Id) if err != nil { return err.Error() } + defer unlock() - if gym == nil { - gym = &Gym{newRecord: true} - } gym.updateGymFromFortProto(fort) updateGymGetMapFortCache(gym, true) @@ -1226,18 +1319,12 @@ func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails } func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymInfo *pogo.GymGetInfoOutProto) string { - gymMutex, _ := gymStripedMutex.GetLock(gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) // should check error + gym, unlock, err := getOrCreateGymRecord(ctx, db, gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) if err != nil { return err.Error() } + defer unlock() - if gym == nil { - gym = &Gym{newRecord: true} - } gym.updateGymFromGymInfoOutProto(gymInfo) updateGymGetMapFortCache(gym, true) @@ -1246,11 +1333,7 @@ func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymIn } func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { - gymMutex, _ := gymStripedMutex.GetLock(mapFort.Id) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, mapFort.Id) + gym, unlock, err := getGymRecordForUpdate(ctx, db, mapFort.Id) if err != nil { return false, err.Error() } @@ -1259,6 +1342,7 @@ func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails if gym == nil { return false, "" } + defer unlock() gym.updateGymFromGetMapFortsOutProto(mapFort, false) saveGymRecord(ctx, db, gym) @@ -1266,11 +1350,7 @@ func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails } func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pogo.RaidDetails, resp *pogo.GetEventRsvpsOutProto) string { - gymMutex, _ := gymStripedMutex.GetLock(req.FortId) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, req.FortId) + gym, unlock, err := getGymRecordForUpdate(ctx, db, req.FortId) if err != nil { return err.Error() } @@ -1279,6 +1359,8 @@ func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pog // Do not add RSVP details to unknown gyms return fmt.Sprintf("%s Gym not present", req.FortId) } + defer unlock() + gym.updateGymFromRsvpProto(resp) saveGymRecord(ctx, db, gym) @@ -1287,11 +1369,7 @@ func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pog } func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { - gymMutex, _ := gymStripedMutex.GetLock(fortId) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, fortId) + gym, unlock, err := getGymRecordForUpdate(ctx, db, fortId) if err != nil { return err.Error() } @@ -1300,6 +1378,7 @@ func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { // Do not add RSVP details to unknown gyms return fmt.Sprintf("%s Gym not present", fortId) } + defer unlock() if gym.Rsvps.Valid { gym.SetRsvps(null.NewString("", false)) diff --git a/decoder/main.go b/decoder/main.go index 4d2324a6..c66c8ea7 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -73,7 +73,6 @@ var routeCache *ttlcache.Cache[string, *Route] var diskEncounterCache *ttlcache.Cache[uint64, *pogo.DiskEncounterOutProto] var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto] -var gymStripedMutex = stripedmutex.New(1103) var stationStripedMutex = stripedmutex.New(1103) var tappableStripedMutex = intstripedmutex.New(563) var incidentStripedMutex = stripedmutex.New(157) @@ -280,9 +279,10 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa // If this is a new pokestop, check if it was converted from a gym and copy shared fields if pokestop.IsNewRecord() { - gym, _ := GetGymRecord(ctx, db, fortId) + gym, gymUnlock, _ := getGymRecordReadOnly(ctx, db, fortId) if gym != nil { pokestop.copySharedFieldsFrom(gym) + gymUnlock() } } @@ -320,20 +320,12 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa } if fort.Data.FortType == pogo.FortType_GYM && scanParameters.ProcessGyms { - gymMutex, _ := gymStripedMutex.GetLock(fortId) - - gymMutex.Lock() - gym, err := GetGymRecord(ctx, db, fortId) + gym, gymUnlock, err := getOrCreateGymRecord(ctx, db, fortId) if err != nil { - log.Errorf("GetGymRecord: %s", err) - gymMutex.Unlock() + log.Errorf("getOrCreateGymRecord: %s", err) continue } - if gym == nil { - gym = &Gym{newRecord: true} - } - gym.updateGymFromFort(fort.Data, fort.Cell) // If this is a new gym, check if it was converted from a pokestop and copy shared fields @@ -346,7 +338,7 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa } saveGymRecord(ctx, db, gym) - gymMutex.Unlock() + gymUnlock() } } } diff --git a/decoder/pokemon.go b/decoder/pokemon.go index e4c079bb..2518a237 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -533,12 +533,13 @@ func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId // Atomically cache the loaded Pokemon - if another goroutine raced us, // we'll get their Pokemon and use that instead (ensuring same mutex) - pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + existingPokemon, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { // Only called if key doesn't exist - our Pokemon wins pokemonRtreeUpdatePokemonOnGet(&dbPokemon) return &dbPokemon - }, ttlcache.DefaultTTL) + }) + pokemon := existingPokemon.Value() pokemon.Lock() return pokemon, func() { pokemon.Unlock() }, nil } @@ -559,10 +560,11 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId // Caller MUST call returned unlock function. func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { // Create new Pokemon atomically - function only called if key doesn't exist - pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + pokemonItem, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { return &Pokemon{Id: encounterId, newRecord: true} - }, ttlcache.DefaultTTL) + }) + pokemon := pokemonItem.Value() pokemon.Lock() if config.Config.PokemonMemoryOnly { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index c0fd7141..d4afe735 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -685,14 +685,15 @@ func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId stri // Atomically cache the loaded Pokestop - if another goroutine raced us, // we'll get their Pokestop and use that instead (ensuring same mutex) - pokestop := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { // Only called if key doesn't exist - our Pokestop wins if config.Config.TestFortInMemory { fortRtreeUpdatePokestopOnGet(&dbPokestop) } return &dbPokestop - }, ttlcache.DefaultTTL) + }) + pokestop := existingPokestop.Value() pokestop.Lock() return pokestop, func() { pokestop.Unlock() }, nil } @@ -713,10 +714,11 @@ func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId str // Caller MUST call returned unlock function. func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { // Create new Pokestop atomically - function only called if key doesn't exist - pokestop := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + pokestopItem, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { return &Pokestop{Id: fortId, newRecord: true} - }, ttlcache.DefaultTTL) + }) + pokestop := pokestopItem.Value() pokestop.Lock() if pokestop.newRecord { diff --git a/decoder/sharded_cache.go b/decoder/sharded_cache.go index dced7c27..84777838 100644 --- a/decoder/sharded_cache.go +++ b/decoder/sharded_cache.go @@ -96,14 +96,13 @@ func (sc *ShardedCache[K, V]) DeleteAll() { } } -// GetOrSetFunc atomically gets an existing item or creates and sets a new one. -// If key exists, returns existing value (createFunc NOT called). -// If key doesn't exist, calls createFunc to create value, sets it, and returns it. -// This prevents race conditions and avoids creating objects unnecessarily. -func (sc *ShardedCache[K, V]) GetOrSetFunc(key K, createFunc func() V, ttl time.Duration) V { - shard := sc.getShard(key) - item, _ := shard.GetOrSetFunc(key, createFunc, ttlcache.WithTTL[K, V](ttl)) - return item.Value() +// GetOrSetFunc retrieves an item from the cache by the provided key. +// If the element is not found, it is created by executing the fn function +// with the provided options and then returned. +// The bool return value is true if the item was found, false if created +// during the execution of the method. +func (sc *ShardedCache[K, V]) GetOrSetFunc(key K, createFunc func() V, opts ...ttlcache.Option[K, V]) (*ttlcache.Item[K, V], bool) { + return sc.getShard(key).GetOrSetFunc(key, createFunc, opts...) } // --- Key conversion helpers --- From c56b3baafb314f093c2b0954f4c4c23826c42282 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 17:26:53 +0000 Subject: [PATCH 20/78] copilot review suggestion fixes --- decoder/api_pokemon.go | 1 - decoder/incident.go | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/decoder/api_pokemon.go b/decoder/api_pokemon.go index 7e017fdd..6a414eb6 100644 --- a/decoder/api_pokemon.go +++ b/decoder/api_pokemon.go @@ -154,7 +154,6 @@ func SearchPokemon(request ApiPokemonSearch) ([]*ApiPokemonResult, error) { unlock() } } - pokemonMatched++ return apiResults, nil } diff --git a/decoder/incident.go b/decoder/incident.go index a432f2e5..c94dd598 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -292,8 +292,10 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident updateIncidentStats(incident, areas) incident.ClearDirty() - incident.newRecord = false - //incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) + if incident.IsNewRecord() { + incident.newRecord = false + incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) + } } func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { From a28a03979798f51cab71ae8e85cae99915367559 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 18:10:17 +0000 Subject: [PATCH 21/78] update modules --- go.mod | 78 +++++++------ go.sum | 362 ++++++++++++++++++++------------------------------------- 2 files changed, 163 insertions(+), 277 deletions(-) diff --git a/go.mod b/go.mod index 83952183..fc946286 100644 --- a/go.mod +++ b/go.mod @@ -5,32 +5,32 @@ go 1.25 toolchain go1.25.0 require ( - github.com/Depado/ginprom v1.8.1 + github.com/Depado/ginprom v1.8.2 github.com/UnownHash/gohbem v0.12.0 - github.com/getsentry/sentry-go v0.35.1 - github.com/gin-gonic/gin v1.10.1 + github.com/getsentry/sentry-go v0.42.0 + github.com/gin-gonic/gin v1.11.0 github.com/go-sql-driver/mysql v1.9.3 github.com/goccy/go-json v0.10.5 - github.com/golang-migrate/migrate/v4 v4.18.3 - github.com/golang/geo v0.0.0-20250813021530-247f39904721 - github.com/grafana/pyroscope-go v1.2.4 + github.com/golang-migrate/migrate/v4 v4.19.1 + github.com/golang/geo v0.0.0-20260129164528-943061e2742c + github.com/grafana/pyroscope-go v1.2.7 github.com/jellydator/ttlcache/v3 v3.4.0 github.com/jmoiron/sqlx v1.4.0 github.com/knadh/koanf/maps v0.1.2 github.com/knadh/koanf/parsers/toml v0.1.0 - github.com/knadh/koanf/providers/file v1.2.0 + github.com/knadh/koanf/providers/file v1.2.1 github.com/knadh/koanf/providers/structs v1.0.0 - github.com/knadh/koanf/v2 v2.2.2 + github.com/knadh/koanf/v2 v2.3.2 github.com/nmvalera/striped-mutex v0.1.0 - github.com/paulmach/orb v0.11.1 - github.com/prometheus/client_golang v1.23.0 + github.com/paulmach/orb v0.12.0 + github.com/prometheus/client_golang v1.23.2 github.com/puzpuzpuz/xsync/v3 v3.5.1 - github.com/ringsaturn/tzf v1.0.0 - github.com/ringsaturn/tzf-rel v0.0.2025-b - github.com/sirupsen/logrus v1.9.3 + github.com/ringsaturn/tzf v1.0.3 + github.com/ringsaturn/tzf-rel v0.0.2025-c + github.com/sirupsen/logrus v1.9.4 github.com/tidwall/rtree v1.10.0 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f - google.golang.org/grpc v1.74.2 + google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -39,23 +39,23 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.14.0 // indirect - github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.27.0 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -68,22 +68,24 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.65.0 // indirect - github.com/prometheus/procfs v0.17.0 // indirect - github.com/ringsaturn/tzf-rel-lite v0.0.2025-b // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/ringsaturn/tzf-rel-lite v0.0.2025-c // indirect github.com/tidwall/geoindex v1.7.0 // indirect - github.com/tidwall/geojson v1.4.5 // indirect + github.com/tidwall/geojson v1.4.6 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twpayne/go-polyline v1.1.1 // indirect - github.com/ugorji/go/codec v1.3.0 // indirect - go.mongodb.org/mongo-driver v1.17.4 // indirect - go.uber.org/atomic v1.11.0 // indirect - golang.org/x/arch v0.20.0 // indirect - golang.org/x/crypto v0.41.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.mongodb.org/mongo-driver v1.17.8 // indirect + go.uber.org/mock v0.6.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/text v0.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect ) diff --git a/go.sum b/go.sum index 955a7b27..a16aadf3 100644 --- a/go.sum +++ b/go.sum @@ -2,46 +2,40 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Depado/ginprom v1.8.1 h1:lrQTddbRqlHq1j6SpJDySDumJlR7FEybzdX0PS3HXPc= -github.com/Depado/ginprom v1.8.1/go.mod h1:9Z+ahPJLSeMndDfnDTfiuBn2SKVAuL2yvihApWzof9A= +github.com/Depado/ginprom v1.8.2 h1:H3sXqXlHfXpoUHciuWSbod1jzc9OyaZ4edM5oYL/nUI= +github.com/Depado/ginprom v1.8.2/go.mod h1:uq9dl4TqwBr0OpkvswJURh5fmjZcbrrMoDiDFHN8dMw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/UnownHash/gohbem v0.12.0 h1:eSyioEWJSU/81i6wf5x4XaiRZBXm6dW/KuYiHKjcELI= github.com/UnownHash/gohbem v0.12.0/go.mod h1:PUeicvRH6HyjTgkuaivjYHzDUzErf2QlsXZ24m0DaNU= -github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= -github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= +github.com/appleboy/gofight/v2 v2.2.0 h1:uqQ3wzTlF1ma+r4jRCQ4cygCjrGZyZEBMBCjT/t9zRw= +github.com/appleboy/gofight/v2 v2.2.0/go.mod h1:USTV3UbA5kHBs4I91EsPi+6PIVZAx3KLorYjvtON91A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bytedance/sonic v1.12.9 h1:Od1BvK55NnewtGaJsTDeAOSnLVO2BTSLOe0+ooKokmQ= -github.com/bytedance/sonic v1.12.9/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= -github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= -github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= -github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= -github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.3 h1:yctD0Q3v2NOGfSWPLPvG2ggA2kV6TS6s4wioyEqssH0= -github.com/bytedance/sonic/loader v0.2.3/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= -github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= -github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= -github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dhui/dktest v0.4.4 h1:+I4s6JRE1yGuqflzwqG+aIaMdgXIorCf5P98JnaAWa8= -github.com/dhui/dktest v0.4.4/go.mod h1:4+22R4lgsdAXrDyaH4Nqx2JEz2hLp49MqQmm9HLCQhM= -github.com/dhui/dktest v0.4.5 h1:uUfYBIVREmj/Rw6MvgmqNAYzTiKOHJak+enB5Di73MM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4= -github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -51,30 +45,20 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= -github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= -github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= -github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4= -github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= -github.com/getsentry/sentry-go v0.35.1 h1:iopow6UVLE2aXu46xKVIs8Z9D/YZkJrHkgozrxa+tOQ= -github.com/getsentry/sentry-go v0.35.1/go.mod h1:C55omcY9ChRQIUcVcGcs+Zdy4ZpQGvNJ7JYHIoSWOtE= -github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= -github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8= +github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= -github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= +github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -83,64 +67,38 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8= -github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= -github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= -github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= -github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= -github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-sql-driver/mysql v1.9.0 h1:Y0zIbQXhQKmQgTp44Y1dp3wTXcn804QoTptLZT1vtvo= -github.com/go-sql-driver/mysql v1.9.0/go.mod h1:pDetrLJeA3oMujJuvXc8RJoasr589B6A9fwzD3QMrqw= -github.com/go-sql-driver/mysql v1.9.1 h1:FrjNGn/BsJQjVRuSa8CBrM5BWA9BWoXXat3KrtSb/iI= -github.com/go-sql-driver/mysql v1.9.1/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-migrate/migrate/v4 v4.18.2 h1:2VSCMz7x7mjyTXx3m2zPokOY82LTRgxK1yQYKo6wWQ8= -github.com/golang-migrate/migrate/v4 v4.18.2/go.mod h1:2CM6tJvn2kqPXwnXO/d3rAQYiyoIm180VsO8PRX6Rpk= -github.com/golang-migrate/migrate/v4 v4.18.3 h1:EYGkoOsvgHHfm5U/naS1RP/6PL/Xv3S4B/swMiAmDLs= -github.com/golang-migrate/migrate/v4 v4.18.3/go.mod h1:99BKpIi6ruaaXRM1A77eqZ+FWPQ3cfRa+ZVy5bmWMaY= -github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I= -github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U= -github.com/golang/geo v0.0.0-20250328065203-0b6e08c212fb h1:eqdj1jSZjgmPdSl2lr3rAwJykSe9jHxPN1zLuasKVh0= -github.com/golang/geo v0.0.0-20250328065203-0b6e08c212fb/go.mod h1:J+F9/3Ofc8ysEOY2/cNjxTMl2eB1gvPIywEHUplPgDA= -github.com/golang/geo v0.0.0-20250813021530-247f39904721 h1:Hlto+T7Ba4CJM4SN8WiA9mw3MdMUboxWsWBaUzRuJuA= -github.com/golang/geo v0.0.0-20250813021530-247f39904721/go.mod h1:AN0OjM34c3PbjAsX+QNma1nYtJtRxl+s9MZNV7S+efw= +github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= +github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= +github.com/golang/geo v0.0.0-20260129164528-943061e2742c h1:ysO2h2Odnl1AJM1I2Lm/fa6JvO0pECMSt2CwBaa+ITo= +github.com/golang/geo v0.0.0-20260129164528-943061e2742c/go.mod h1:Mymr9kRGDc64JPr03TSZmuIBODZ3KyswLzm1xL0HFA8= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= -github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= -github.com/grafana/pyroscope-go v1.2.1 h1:ewi38pE6XMnoHlZYhGxS3uH5TGKA7vDhkT1T3RVkjq0= -github.com/grafana/pyroscope-go v1.2.1/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= -github.com/grafana/pyroscope-go v1.2.4 h1:B22GMXz+O0nWLatxLuaP7o7L9dvP0clLvIpmeEQQM0Q= -github.com/grafana/pyroscope-go v1.2.4/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= -github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= -github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/jellydator/ttlcache/v3 v3.3.0 h1:BdoC9cE81qXfrxeb9eoJi9dWrdhSuwXMAnHTbnBm4Wc= -github.com/jellydator/ttlcache/v3 v3.3.0/go.mod h1:bj2/e0l4jRnQdrnSTaGTsh4GSXvMjQcy41i7th0GVGw= +github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= +github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= @@ -150,34 +108,20 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= -github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/parsers/toml v0.1.0 h1:S2hLqS4TgWZYj4/7mI5m1CQQcWurxUz6ODgOub/6LCI= github.com/knadh/koanf/parsers/toml v0.1.0/go.mod h1:yUprhq6eo3GbyVXFFMdbfZSo928ksS+uo0FFqNMnO18= -github.com/knadh/koanf/providers/file v1.1.2 h1:aCC36YGOgV5lTtAFz2qkgtWdeQsgfxUkxDOe+2nQY3w= -github.com/knadh/koanf/providers/file v1.1.2/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI= -github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U= -github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= -github.com/knadh/koanf/providers/structs v0.1.0 h1:wJRteCNn1qvLtE5h8KQBvLJovidSdntfdyIbbCzEyE0= -github.com/knadh/koanf/providers/structs v0.1.0/go.mod h1:sw2YZ3txUcqA3Z27gPlmmBzWn1h8Nt9O6EP/91MkcWE= +github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= +github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= github.com/knadh/koanf/providers/structs v1.0.0 h1:DznjB7NQykhqCar2LvNug3MuxEQsZ5KvfgMbio+23u4= github.com/knadh/koanf/providers/structs v1.0.0/go.mod h1:kjo5TFtgpaZORlpoJqcbeLowM2cINodv8kX+oFAeQ1w= -github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ= -github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo= -github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A= -github.com/knadh/koanf/v2 v2.2.2/go.mod h1:abWQc0cBXLSF/PSOMCB/SK+T13NXDsPvOksbpi5e/9Q= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/knadh/koanf/v2 v2.3.2 h1:Ee6tuzQYFwcZXQpc2MiVeC6qHMandf5SMUJJNoFp/c4= +github.com/knadh/koanf/v2 v2.3.2/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -191,8 +135,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/loov/hrtime v1.0.3 h1:LiWKU3B9skJwRPUf0Urs9+0+OE3TxdMuiRPOTwR0gcU= -github.com/loov/hrtime v1.0.3/go.mod h1:yDY3Pwv2izeY4sq7YcPX/dtLwzg5NU1AxWuWxKwd0p0= +github.com/loov/hrtime v1.0.4 h1:K0wPQBsd9mWer2Sx8zIfpyAlF4ckZovtkEMUR/l9wpU= +github.com/loov/hrtime v1.0.4/go.mod h1:VbIwDNS2gYTRoo0RjQFdqdDlBjJLXrkDIOgoA7Jvupk= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -223,64 +167,46 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= -github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= +github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= -github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= -github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= -github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= -github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= -github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= -github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= -github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= -github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= -github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= -github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= -github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg= github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= -github.com/ringsaturn/go-cities.json v0.6.6 h1:zXEsYeQBjgJwKdGHMrq1j8d9uVNxmnbVvi2fAQAyfNM= -github.com/ringsaturn/go-cities.json v0.6.6/go.mod h1:RWApnQPG6nU558XXbY1try5mi9u9Hd667J6vr948VBo= -github.com/ringsaturn/go-cities.json v0.6.8 h1:PsbjBrEANywxKSMMbagv4qnLSg4Rz+xWeoVKY3pkWnA= -github.com/ringsaturn/tzf v0.17.0 h1:hGHBPBzJfSLmCAKg0khA7yxx9kqPPNYkggdodpKNaKI= -github.com/ringsaturn/tzf v0.17.0/go.mod h1:4Z139lkC4Btg5tse9qdQ78gWmj7X0VwJjDxxO4ydAiU= -github.com/ringsaturn/tzf v1.0.0 h1:z0M2wKJNkyCsNFv/D1cveh6F5jhGwm8aysl0GLh2rnY= -github.com/ringsaturn/tzf v1.0.0/go.mod h1:H/Fl+lPWq+5oD72UZQzFXQnYXcWs3nnyGq6PIYEN8YY= -github.com/ringsaturn/tzf-rel v0.0.2025-a h1:OLJe11vif6JMDtECkS71hAOdIzDHV+uBVgWUW3V/dZQ= -github.com/ringsaturn/tzf-rel v0.0.2025-a/go.mod h1:R6HljjIcUQclZ6bOzWDa8llkEb/x+dmMWU60jXx2BEI= -github.com/ringsaturn/tzf-rel v0.0.2025-b h1:R/YuE6HKj/IwSbb6LilwhwsHoXEkUtnHrtz29Mnkdfw= -github.com/ringsaturn/tzf-rel v0.0.2025-b/go.mod h1:p5HyM9AThIOOr5ZCyheMFAdg236o/HMMnz/DObWdnys= -github.com/ringsaturn/tzf-rel-lite v0.0.2025-a h1:V+B0brHLxYqSyAeEWim/0BhM7Hnz587ZLGe3+8YgJ6E= -github.com/ringsaturn/tzf-rel-lite v0.0.2025-a/go.mod h1:nVMtMUFC40aMfjXzc+fj3CCXrHxD8sRUXLEVfxhFVAs= -github.com/ringsaturn/tzf-rel-lite v0.0.2025-b h1:YYuKav8cpkRtDZ9yFF0kBTO3bU/TjtcawjKI91GWCa4= -github.com/ringsaturn/tzf-rel-lite v0.0.2025-b/go.mod h1:SyVF6OU+Le0vKajtTA7PvYabdYCJsDlmplHuXeCZDrw= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/ringsaturn/go-cities.json v0.6.13 h1:p5afPcJ/tEE6uzFCOzLSHJYXgWnGdPmwZB9KBrEASxc= +github.com/ringsaturn/go-cities.json v0.6.13/go.mod h1:VtklT4Sod9i6kvXXNZV63sfjeCX9l11OQfaAvPu+p4M= +github.com/ringsaturn/tzf v1.0.3 h1:DdGcCiHpS6kg0Fo0XK+YlwNGIRXK3rs+KvFAd6b5XfQ= +github.com/ringsaturn/tzf v1.0.3/go.mod h1:8wWHQjIYklMR3uG9cIkzc/otIBhit3vtAX/D78cXNpY= +github.com/ringsaturn/tzf-rel v0.0.2025-c h1:hx2KHcZzMnO2VLg/GXKKJ6vMubmPwimcen9Gf/t1KzY= +github.com/ringsaturn/tzf-rel v0.0.2025-c/go.mod h1:p5HyM9AThIOOr5ZCyheMFAdg236o/HMMnz/DObWdnys= +github.com/ringsaturn/tzf-rel-lite v0.0.2025-c h1:CUs4l73ApN87MhlAhp1UtcRe3E5UFMQnl9d9XtJiHvg= +github.com/ringsaturn/tzf-rel-lite v0.0.2025-c/go.mod h1:SyVF6OU+Le0vKajtTA7PvYabdYCJsDlmplHuXeCZDrw= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -292,21 +218,21 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/cities v0.1.0 h1:CVNkmMf7NEC9Bvokf5GoSsArHCKRMTgLuubRTHnH0mE= github.com/tidwall/cities v0.1.0/go.mod h1:lV/HDp2gCcRcHJWqgt6Di54GiDrTZwh1aG2ZUPNbqa4= github.com/tidwall/geoindex v1.4.4/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= github.com/tidwall/geoindex v1.7.0 h1:jtk41sfgwIt8MEDyC3xyKSj75iXXf6rjReJGDNPtR5o= github.com/tidwall/geoindex v1.7.0/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= -github.com/tidwall/geojson v1.4.5 h1:BFVb5Pr7WZJMqFXy1LVudt5hPEWR3g4uhjk5Ezc3GzA= -github.com/tidwall/geojson v1.4.5/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg= +github.com/tidwall/geojson v1.4.6 h1:HpEGer4tc5ieFn8Ts8aTG9fo+hgFJkqfql4O9cgphmg= +github.com/tidwall/geojson v1.4.6/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= -github.com/tidwall/lotsa v1.0.3 h1:lFAp3PIsS58FPmz+LzhE1mcZ67tBBCRPv5j66g6y7sg= -github.com/tidwall/lotsa v1.0.3/go.mod h1:cPF+z88hamDNDjvE+u3suxCtRMVw24Gvze9eeWGYook= +github.com/tidwall/lotsa v1.0.4 h1:7jF9n2JVRuI42E4AqBlbAcjF6ACyI+8v46/CYQY47ZI= +github.com/tidwall/lotsa v1.0.4/go.mod h1:cPF+z88hamDNDjvE+u3suxCtRMVw24Gvze9eeWGYook= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -320,10 +246,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twpayne/go-polyline v1.1.1 h1:/tSF1BR7rN4HWj4XKqvRUNrCiYVMCvywxTFVofvDV0w= github.com/twpayne/go-polyline v1.1.1/go.mod h1:ybd9IWWivW/rlXPXuuckeKUyF3yrIim+iqA7kSl4NFY= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= -github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= @@ -331,46 +255,36 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.17.2 h1:gvZyk8352qSfzyZ2UMWcpDpMSGEr1eqE4T793SqyhzM= -go.mongodb.org/mongo-driver v1.17.2/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= -go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= -go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.mongodb.org/mongo-driver v1.17.8 h1:BDP3+U3Y8K0vTrpqDJIRaXNhb/bKyoVeg6tIJsW5EhM= +go.mongodb.org/mongo-driver v1.17.8/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= -golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= -golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= -golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= -golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= +golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -378,47 +292,30 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -427,26 +324,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250219182151-9fdb1cabc7b2 h1:DMTIbak9GhdaSxEjvVzAeNZvyc03I61duqNbnm3SU0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250219182151-9fdb1cabc7b2/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a h1:tPE/Kp+x9dMSwUm/uM0JKK0IfdiJkwAbSMSeZBXXJXc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= -google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= -google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -461,4 +346,3 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= From 03c897ad5b4cf0cdaa42f4a7790ca62836e0102a Mon Sep 17 00:00:00 2001 From: Fabio1988 Date: Thu, 29 Jan 2026 18:30:54 +0100 Subject: [PATCH 22/78] fix: increase spawnpoint updates --- decoder/spawnpoint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 9d411a2f..09632856 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -213,7 +213,8 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { spawnpoint := inMemorySpawnpoint.Value() now := time.Now().Unix() - if now-spawnpoint.LastSeen > 3600 { + // update at least every 6 hours + if now-spawnpoint.LastSeen > 21600 { spawnpoint.LastSeen = now _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ From d2ba3b7d771e15e1b49f9d2fe2351965506f0485 Mon Sep 17 00:00:00 2001 From: Fabio1988 Date: Fri, 30 Jan 2026 16:55:18 +0100 Subject: [PATCH 23/78] feat: reduce updates config --- config.toml.example | 6 ++++++ config/config.go | 1 + config/reader.go | 1 + decoder/gym.go | 5 +++-- decoder/main.go | 10 ++++++++++ decoder/pokestop.go | 9 +++++---- decoder/routes.go | 2 +- decoder/s2cell.go | 2 +- decoder/spawnpoint.go | 4 ++-- decoder/station.go | 2 +- 10 files changed, 31 insertions(+), 11 deletions(-) diff --git a/config.toml.example b/config.toml.example index 561410c4..2c018fcb 100644 --- a/config.toml.example +++ b/config.toml.example @@ -3,6 +3,12 @@ port = 9001 # Listening port for golbat raw_bearer = "" # Raw bearer (password) required api_secret = "golbat" # Golbat secret required on api calls (blank for none) +# When enabled, reduce_updates will make fort update debounce windows much longer +# to reduce database churn. Specifically, gym/pokestop/station debounce will be +# extended from 15 minutes (900s) to 12 hours (43200s) and spawnpoint last_seen +# will be updated every 12 hours instead of the default 6 hours. +reduce_updates = false + pokemon_memory_only = false # Use in-memory storage for pokemon only [koji] diff --git a/config/config.go b/config/config.go index 09463131..fb66aa37 100644 --- a/config/config.go +++ b/config/config.go @@ -27,6 +27,7 @@ type configDefinition struct { Weather weather `koanf:"weather"` ScanRules []scanRule `koanf:"scan_rules"` MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` + ReduceUpdates bool `koanf:"reduce_updates"` } func (configDefinition configDefinition) GetWebhookInterval() time.Duration { diff --git a/config/reader.go b/config/reader.go index 270edc93..e0daa36c 100644 --- a/config/reader.go +++ b/config/reader.go @@ -62,6 +62,7 @@ func ReadConfig() (configDefinition, error) { LevelCaps: []int{50, 51}, }, MaxConcurrentProactiveIVSwitch: 6, + ReduceUpdates: false, }, "koanf"), nil) if defaultErr != nil { fmt.Println(fmt.Errorf("failed to load default config: %w", defaultErr)) diff --git a/decoder/gym.go b/decoder/gym.go index 261cfff6..f43c48f5 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -1202,8 +1202,9 @@ func createGymWebhooks(gym *Gym, areas []geo.AreaName) { func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { now := time.Now().Unix() if !gym.IsNewRecord() && !gym.IsDirty() && !gym.IsInternalDirty() { - if gym.Updated > now-900 { - // if a gym is unchanged, but we did see it again after 15 minutes, then save again + // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. + if gym.Updated > now-GetUpdateThreshold(900) { + // if a gym is unchanged and was seen recently, skip saving return } } diff --git a/decoder/main.go b/decoder/main.go index c66c8ea7..a9a99a89 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -551,3 +551,13 @@ func SetWebhooksSender(whSender webhooksSenderInterface) { func SetStatsCollector(collector stats_collector.StatsCollector) { statsCollector = collector } + +// GetUpdateThreshold returns the number of seconds that should be used as a +// debounce/last-seen threshold. Pass the default seconds for normal operation +// If ReduceUpdates is enabled in the loaded config.Config, this returns 43200 (12 hours). +func GetUpdateThreshold(defaultSeconds int64) int64 { + if config.Config.ReduceUpdates { + return 43200 // 12 hours + } + return defaultSeconds +} diff --git a/decoder/pokestop.go b/decoder/pokestop.go index d4afe735..024ef7a6 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1306,7 +1306,8 @@ func createPokestopWebhooks(stop *Pokestop) { func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { now := time.Now().Unix() if !pokestop.IsNewRecord() && !pokestop.IsDirty() { - if pokestop.Updated > now-900 { + // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. + if pokestop.Updated > now-GetUpdateThreshold(900) { // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again return } @@ -1343,12 +1344,12 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop statsCollector.IncDbQuery("insert pokestop", err) //log.Debugf("Insert pokestop %s %+v", pokestop.Id, pokestop) if err != nil { - log.Errorf("insert pokestop %s: %s", pokestop.Id, err) + log.Errorf("insert pokestop: %s", err) return } - _ = res + + _, _ = res, err } else { - // Existing record - UPDATE if dbDebugEnabled { dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) } diff --git a/decoder/routes.go b/decoder/routes.go index 8407e963..3ac86969 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -293,7 +293,7 @@ func getRouteRecord(db db.DbDetails, id string) (*Route, error) { func saveRouteRecord(db db.DbDetails, route *Route) error { // Skip save if not dirty and not new, unless 15-minute debounce expired if !route.IsDirty() && !route.IsNewRecord() { - if route.Updated > time.Now().Unix()-900 { + if route.Updated > time.Now().Unix()-GetUpdateThreshold(900) { // if a route is unchanged, but we did see it again after 15 minutes, then save again return nil } diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 5dca4187..4f59ebaf 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -41,7 +41,7 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { if c := s2CellCache.Get(cellId); c != nil { cachedCell := c.Value() - if cachedCell.Updated > now-900 { + if cachedCell.Updated > now-GetUpdateThreshold(900) { continue } s2Cell = cachedCell diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 09632856..176eac2b 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -213,8 +213,8 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { spawnpoint := inMemorySpawnpoint.Value() now := time.Now().Unix() - // update at least every 6 hours - if now-spawnpoint.LastSeen > 21600 { + // update at least every 6 hours (21600s). If reduce_updates is enabled, use 12 hours. + if now-spawnpoint.LastSeen > GetUpdateThreshold(21600) { spawnpoint.LastSeen = now _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ diff --git a/decoder/station.go b/decoder/station.go index 029fe168..b2517bfe 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -416,7 +416,7 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { // Skip save if not dirty and was updated recently (15-min debounce) if !station.IsDirty() && !station.IsNewRecord() { - if station.Updated > now-900 { + if station.Updated > now-GetUpdateThreshold(900) { return } } From f4976ac238204fd40301471e85b5318b86a68913 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 21:04:12 +0000 Subject: [PATCH 24/78] more locking update --- decoder/gym.go | 2 + decoder/incident.go | 113 ++++++++++++++++++++++++++----- decoder/main.go | 96 +++++++------------------- decoder/pokemon.go | 14 +++- decoder/pokestop.go | 3 + decoder/routes.go | 160 +++++++++++++++++++++++++++++++++----------- decoder/station.go | 126 +++++++++++++++++++++++++++------- decoder/tappable.go | 123 +++++++++++++++++++++++++++------- decoder/weather.go | 112 ++++++++++++++++++++++++++----- main.go | 6 +- 10 files changed, 557 insertions(+), 198 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index f43c48f5..5fc58626 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -617,6 +617,7 @@ func getGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) ( if err != nil { return nil, nil, err } + dbGym.ClearDirty() // Atomically cache the loaded Gym - if another goroutine raced us, // we'll get their Gym and use that instead (ensuring same mutex) @@ -667,6 +668,7 @@ func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) ( } else { // We loaded from DB gym.newRecord = false + gym.ClearDirty() if config.Config.TestFortInMemory { fortRtreeUpdateGymOnGet(gym) } diff --git a/decoder/incident.go b/decoder/incident.go index c94dd598..38b5a992 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -3,6 +3,8 @@ package decoder import ( "context" "database/sql" + "errors" + "sync" "time" "github.com/jellydator/ttlcache/v3" @@ -17,6 +19,8 @@ import ( // Incident struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Incident struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id"` PokestopId string `db:"pokestop_id"` StartTime int64 `db:"start"` @@ -98,6 +102,16 @@ func (incident *Incident) IsNewRecord() bool { return incident.newRecord } +// Lock acquires the Incident's mutex +func (incident *Incident) Lock() { + incident.mu.Lock() +} + +// Unlock releases the Incident's mutex +func (incident *Incident) Unlock() { + incident.mu.Unlock() +} + // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (incident *Incident) snapshotOldValues() { @@ -210,31 +224,96 @@ func (incident *Incident) SetSlot3Form(v null.Int) { } } -func getIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, error) { - inMemoryIncident := incidentCache.Get(incidentId) - if inMemoryIncident != nil { - incident := inMemoryIncident.Value() - incident.snapshotOldValues() - return incident, nil - } - - incident := Incident{} - err := db.GeneralDb.GetContext(ctx, &incident, +func loadIncidentFromDatabase(ctx context.Context, db db.DbDetails, incidentId string, incident *Incident) error { + err := db.GeneralDb.GetContext(ctx, incident, "SELECT id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form "+ - "FROM incident "+ - "WHERE incident.id = ? ", incidentId) + "FROM incident WHERE incident.id = ?", incidentId) statsCollector.IncDbQuery("select incident", err) - if err == sql.ErrNoRows { - return nil, nil + return err +} + +// peekIncidentRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekIncidentRecord(incidentId string) (*Incident, func(), error) { + if item := incidentCache.Get(incidentId); item != nil { + incident := item.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil } + return nil, nil, nil +} +// getIncidentRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getIncidentRecordReadOnly(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { + // Check cache first + if item := incidentCache.Get(incidentId); item != nil { + incident := item.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil + } + + dbIncident := Incident{} + err := loadIncidentFromDatabase(ctx, db, incidentId, &dbIncident) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return nil, err + return nil, nil, err + } + dbIncident.ClearDirty() + + // Atomically cache the loaded Incident - if another goroutine raced us, + // we'll get their Incident and use that instead (ensuring same mutex) + existingIncident, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + return &dbIncident + }) + + incident := existingIncident.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil +} + +// getIncidentRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getIncidentRecordForUpdate(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { + incident, unlock, err := getIncidentRecordReadOnly(ctx, db, incidentId) + if err != nil || incident == nil { + return nil, nil, err + } + incident.snapshotOldValues() + return incident, unlock, nil +} + +// getOrCreateIncidentRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string, pokestopId string) (*Incident, func(), error) { + // Create new Incident atomically - function only called if key doesn't exist + incidentItem, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + return &Incident{Id: incidentId, PokestopId: pokestopId, newRecord: true} + }) + + incident := incidentItem.Value() + incident.Lock() + + if incident.newRecord { + // We should attempt to load from database + err := loadIncidentFromDatabase(ctx, db, incidentId, incident) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + incident.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + incident.newRecord = false + incident.ClearDirty() + } } - incidentCache.Set(incidentId, &incident, ttlcache.DefaultTTL) incident.snapshotOldValues() - return &incident, nil + return incident, func() { incident.Unlock() }, nil } func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident) { diff --git a/decoder/main.go b/decoder/main.go index a9a99a89..21467b7c 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -7,11 +7,8 @@ import ( "runtime" "time" - "golbat/intstripedmutex" - "github.com/UnownHash/gohbem" "github.com/jellydator/ttlcache/v3" - stripedmutex "github.com/nmvalera/striped-mutex" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" @@ -73,12 +70,6 @@ var routeCache *ttlcache.Cache[string, *Route] var diskEncounterCache *ttlcache.Cache[uint64, *pogo.DiskEncounterOutProto] var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto] -var stationStripedMutex = stripedmutex.New(1103) -var tappableStripedMutex = intstripedmutex.New(563) -var incidentStripedMutex = stripedmutex.New(157) -var weatherStripedMutex = intstripedmutex.New(157) -var routeStripedMutex = stripedmutex.New(157) - var ProactiveIVSwitchSem chan bool var ohbem *gohbem.Ohbem @@ -296,25 +287,14 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa if incidents != nil { for _, incidentProto := range incidents { - incidentMutex, _ := incidentStripedMutex.GetLock(incidentProto.IncidentId) - - incidentMutex.Lock() - incident, err := getIncidentRecord(ctx, db, incidentProto.IncidentId) + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, incidentProto.IncidentId, fortId) if err != nil { - log.Errorf("getIncident: %s", err) - incidentMutex.Unlock() + log.Errorf("getOrCreateIncidentRecord: %s", err) continue } - if incident == nil { - incident = &Incident{ - PokestopId: fortId, - newRecord: true, - } - } incident.updateFromPokestopIncidentDisplay(incidentProto) saveIncidentRecord(ctx, db, incident) - - incidentMutex.Unlock() + unlock() } } } @@ -346,20 +326,14 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawStationData) { for _, stationProto := range p { stationId := stationProto.Data.Id - stationMutex, _ := stationStripedMutex.GetLock(stationId) - stationMutex.Lock() - station, err := getStationRecord(ctx, db, stationId) + station, unlock, err := getOrCreateStationRecord(ctx, db, stationId) if err != nil { - log.Errorf("getStationRecord: %s", err) - stationMutex.Unlock() + log.Errorf("getOrCreateStationRecord: %s", err) continue } - if station == nil { - station = &Station{newRecord: true} - } station.updateFromStationProto(stationProto.Data, stationProto.Cell) saveStationRecord(ctx, db, station) - stationMutex.Unlock() + unlock() } } @@ -452,13 +426,13 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.ClientWeatherProto, timestampMs int64, account string) (updates []WeatherUpdate) { hourKey := timestampMs / time.Hour.Milliseconds() for _, weatherProto := range p { - weatherMutex, _ := weatherStripedMutex.GetLock(uint64(weatherProto.S2CellId)) - weatherMutex.Lock() - - weather, err := getWeatherRecord(ctx, db, weatherProto.S2CellId) + weather, unlock, err := getOrCreateWeatherRecord(ctx, db, weatherProto.S2CellId) if err != nil { - log.Printf("getWeatherRecord: %s", err) - } else if weather == nil || timestampMs >= weather.UpdatedMs { + log.Printf("getOrCreateWeatherRecord: %s", err) + continue + } + + if weather.newRecord || timestampMs >= weather.UpdatedMs { state := getWeatherConsensusState(weatherProto.S2CellId, hourKey) if state != nil { publish, publishProto := state.applyObservation(hourKey, account, weatherProto) @@ -466,9 +440,6 @@ func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.Cl if publishProto == nil { publishProto = weatherProto } - if weather == nil { - weather = &Weather{newRecord: true} - } weather.UpdatedMs = timestampMs weather.updateWeatherFromClientWeatherProto(publishProto) saveWeatherRecord(ctx, db, weather) @@ -482,7 +453,7 @@ func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.Cl } } - weatherMutex.Unlock() + unlock() } return updates } @@ -492,55 +463,34 @@ func UpdateClientMapS2CellBatch(ctx context.Context, db db.DbDetails, cellIds [] } func UpdateIncidentLineup(ctx context.Context, db db.DbDetails, protoReq *pogo.OpenInvasionCombatSessionProto, protoRes *pogo.OpenInvasionCombatSessionOutProto) string { - incidentMutex, _ := incidentStripedMutex.GetLock(protoReq.IncidentLookup.IncidentId) - - incidentMutex.Lock() - incident, err := getIncidentRecord(ctx, db, protoReq.IncidentLookup.IncidentId) + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, protoReq.IncidentLookup.IncidentId, protoReq.IncidentLookup.FortId) if err != nil { - incidentMutex.Unlock() - return fmt.Sprintf("getIncident: %s", err) + return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) } - if incident == nil { + defer unlock() + + if incident.newRecord { log.Debugf("Updating lineup before it was saved: %s", protoReq.IncidentLookup.IncidentId) - incident = &Incident{ - Id: protoReq.IncidentLookup.IncidentId, - PokestopId: protoReq.IncidentLookup.FortId, - newRecord: true, - } } incident.updateFromOpenInvasionCombatSessionOut(protoRes) saveIncidentRecord(ctx, db, incident) - incidentMutex.Unlock() return "" } func ConfirmIncident(ctx context.Context, db db.DbDetails, proto *pogo.StartIncidentOutProto) string { - incidentMutex, _ := incidentStripedMutex.GetLock(proto.Incident.IncidentId) - - incidentMutex.Lock() - incident, err := getIncidentRecord(ctx, db, proto.Incident.IncidentId) + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, proto.Incident.IncidentId, proto.Incident.FortId) if err != nil { - incidentMutex.Unlock() - return fmt.Sprintf("getIncident: %s", err) + return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) } - if incident == nil { + defer unlock() + + if incident.newRecord { log.Debugf("Confirming incident before it was saved: %s", proto.Incident.IncidentId) - incident = &Incident{ - Id: proto.Incident.IncidentId, - PokestopId: proto.Incident.FortId, - newRecord: true, - } } incident.updateFromStartIncidentOut(proto) - if incident == nil { - incidentMutex.Unlock() - // I only saw this once during testing but I couldn't reproduce it so just in case - return "Unable to process incident" - } saveIncidentRecord(ctx, db, incident) - incidentMutex.Unlock() return "" } diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 2518a237..1f5494e2 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -530,6 +530,7 @@ func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId if err != nil { return nil, nil, err } + dbPokemon.ClearDirty() // Atomically cache the loaded Pokemon - if another goroutine raced us, // we'll get their Pokemon and use that instead (ensuring same mutex) @@ -583,6 +584,7 @@ func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId } else { // We loaded pokemon.newRecord = false + pokemon.ClearDirty() pokemonRtreeUpdatePokemonOnGet(pokemon) } } @@ -1613,12 +1615,15 @@ func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails Form: int32(proto.PokemonDisplay.Form), } if scan.CellWeather == int32(pogo.GameplayWeatherProto_NONE) { - weather, err := getWeatherRecord(ctx, db, weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon)) - if err != nil || weather == nil || !weather.GameplayCondition.Valid { + weather, unlock, err := peekWeatherRecord(weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon)) + if weather == nil || !weather.GameplayCondition.Valid { log.Warnf("Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) } else { scan.CellWeather = int32(weather.GameplayCondition.Int64) } + if unlock != nil { + unlock() + } } if proto.CpMultiplier < 0.734 { scan.Level = int32((58.215688455154954*proto.CpMultiplier-2.7012478057856497)*proto.CpMultiplier + 1.3220677708486794) @@ -1838,7 +1843,7 @@ func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails cellId := weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon) cellWeather, found := weather[cellId] if !found { - record, err := getWeatherRecord(ctx, db, cellId) + record, unlock, err := getWeatherRecordReadOnly(ctx, db, cellId) if err != nil || record == nil || !record.GameplayCondition.Valid { log.Warnf("[POKEMON] Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) } else { @@ -1846,6 +1851,9 @@ func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails cellWeather = pogo.GameplayWeatherProto_WeatherCondition(record.GameplayCondition.Int64) found = true } + if unlock != nil { + unlock() + } } if found && cellWeather == pogo.GameplayWeatherProto_PARTLY_CLOUDY { shouldOverrideIv = true diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 024ef7a6..8e25200f 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -683,6 +683,8 @@ func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId stri return nil, nil, err } + dbPokestop.ClearDirty() + // Atomically cache the loaded Pokestop - if another goroutine raced us, // we'll get their Pokestop and use that instead (ensuring same mutex) existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { @@ -732,6 +734,7 @@ func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId stri } else { // We loaded from DB pokestop.newRecord = false + pokestop.ClearDirty() if config.Config.TestFortInMemory { fortRtreeUpdatePokestopOnGet(pokestop) } diff --git a/decoder/routes.go b/decoder/routes.go index 3ac86969..e707b806 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -1,9 +1,12 @@ package decoder import ( + "context" "database/sql" "encoding/json" + "errors" "fmt" + "sync" "time" "golbat/db" @@ -18,6 +21,8 @@ import ( // Route struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Route struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id"` Name string `db:"name"` Shortcode string `db:"shortcode"` @@ -44,6 +49,13 @@ type Route struct { dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) + + oldValues RouteOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// RouteOldValues holds old field values for webhook comparison +type RouteOldValues struct { + Version int64 } // IsDirty returns true if any field has been modified @@ -61,6 +73,24 @@ func (r *Route) IsNewRecord() bool { return r.newRecord } +// Lock acquires the Route's mutex +func (r *Route) Lock() { + r.mu.Lock() +} + +// Unlock releases the Route's mutex +func (r *Route) Unlock() { + r.mu.Unlock() +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (r *Route) snapshotOldValues() { + r.oldValues = RouteOldValues{ + Version: r.Version, + } +} + // --- Set methods with dirty tracking --- func (r *Route) SetName(v string) { @@ -263,34 +293,98 @@ func (r *Route) SetWaypoints(v string) { } } -func getRouteRecord(db db.DbDetails, id string) (*Route, error) { - inMemoryRoute := routeCache.Get(id) - if inMemoryRoute != nil { - route := inMemoryRoute.Value() - return route, nil +func loadRouteFromDatabase(ctx context.Context, db db.DbDetails, routeId string, route *Route) error { + err := db.GeneralDb.GetContext(ctx, route, + `SELECT * FROM route WHERE route.id = ?`, routeId) + statsCollector.IncDbQuery("select route", err) + return err +} + +// peekRouteRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekRouteRecord(routeId string) (*Route, func(), error) { + if item := routeCache.Get(routeId); item != nil { + route := item.Value() + route.Lock() + return route, func() { route.Unlock() }, nil } + return nil, nil, nil +} - route := Route{} - err := db.GeneralDb.Get(&route, - ` - SELECT * - FROM route - WHERE route.id = ? - `, - id, - ) - statsCollector.IncDbQuery("select route", err) - if err == sql.ErrNoRows { - return nil, nil - } else if err != nil { - return nil, err +// getRouteRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getRouteRecordReadOnly(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + // Check cache first + if item := routeCache.Get(routeId); item != nil { + route := item.Value() + route.Lock() + return route, func() { route.Unlock() }, nil + } + + dbRoute := Route{} + err := loadRouteFromDatabase(ctx, db, routeId, &dbRoute) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil } + if err != nil { + return nil, nil, err + } + dbRoute.ClearDirty() + + // Atomically cache the loaded Route - if another goroutine raced us, + // we'll get their Route and use that instead (ensuring same mutex) + existingRoute, _ := routeCache.GetOrSetFunc(routeId, func() *Route { + return &dbRoute + }) + + route := existingRoute.Value() + route.Lock() + return route, func() { route.Unlock() }, nil +} + +// getRouteRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getRouteRecordForUpdate(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + route, unlock, err := getRouteRecordReadOnly(ctx, db, routeId) + if err != nil || route == nil { + return nil, nil, err + } + route.snapshotOldValues() + return route, unlock, nil +} + +// getOrCreateRouteRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateRouteRecord(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + // Create new Route atomically - function only called if key doesn't exist + routeItem, _ := routeCache.GetOrSetFunc(routeId, func() *Route { + return &Route{Id: routeId, newRecord: true} + }) - routeCache.Set(id, &route, ttlcache.DefaultTTL) - return &route, nil + route := routeItem.Value() + route.Lock() + + if route.newRecord { + // We should attempt to load from database + err := loadRouteFromDatabase(ctx, db, routeId, route) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + route.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + route.newRecord = false + route.ClearDirty() + } + } + + route.snapshotOldValues() + return route, func() { route.Unlock() }, nil } -func saveRouteRecord(db db.DbDetails, route *Route) error { +func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { // Skip save if not dirty and not new, unless 15-minute debounce expired if !route.IsDirty() && !route.IsNewRecord() { if route.Updated > time.Now().Unix()-GetUpdateThreshold(900) { @@ -305,7 +399,7 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { if dbDebugEnabled { dbDebugLog("INSERT", "Route", route.Id, route.changedFields) } - _, err := db.GeneralDb.NamedExec( + _, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO route ( id, name, shortcode, description, distance_meters, @@ -337,7 +431,7 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { if dbDebugEnabled { dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) } - _, err := db.GeneralDb.NamedExec( + _, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE route SET name = :name, @@ -422,24 +516,14 @@ func (route *Route) updateFromSharedRouteProto(sharedRouteProto *pogo.SharedRout } } -func UpdateRouteRecordWithSharedRouteProto(db db.DbDetails, sharedRouteProto *pogo.SharedRouteProto) error { - routeMutex, _ := routeStripedMutex.GetLock(sharedRouteProto.GetId()) - routeMutex.Lock() - defer routeMutex.Unlock() - - route, err := getRouteRecord(db, sharedRouteProto.GetId()) +func UpdateRouteRecordWithSharedRouteProto(ctx context.Context, db db.DbDetails, sharedRouteProto *pogo.SharedRouteProto) error { + route, unlock, err := getOrCreateRouteRecord(ctx, db, sharedRouteProto.GetId()) if err != nil { return err } - - if route == nil { - route = &Route{ - Id: sharedRouteProto.GetId(), - newRecord: true, - } - } + defer unlock() route.updateFromSharedRouteProto(sharedRouteProto) - saveError := saveRouteRecord(db, route) + saveError := saveRouteRecord(ctx, db, route) return saveError } diff --git a/decoder/station.go b/decoder/station.go index b2517bfe..bbf5c79a 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "sync" "time" "golbat/db" @@ -21,6 +22,8 @@ import ( // Station struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Station struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` @@ -84,6 +87,16 @@ func (station *Station) IsNewRecord() bool { return station.newRecord } +// Lock acquires the Station's mutex +func (station *Station) Lock() { + station.mu.Lock() +} + +// Unlock releases the Station's mutex +func (station *Station) Unlock() { + station.mu.Unlock() +} + // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (station *Station) snapshotOldValues() { @@ -384,31 +397,96 @@ type StationWebhook struct { Updated int64 `json:"updated"` } -func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, error) { - inMemoryStation := stationCache.Get(stationId) - if inMemoryStation != nil { - station := inMemoryStation.Value() - station.snapshotOldValues() - return station, nil - } - station := Station{} - err := db.GeneralDb.GetContext(ctx, &station, - ` - SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon - FROM station WHERE id = ? - `, stationId) +func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId string, station *Station) error { + err := db.GeneralDb.GetContext(ctx, station, + `SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon + FROM station WHERE id = ?`, stationId) statsCollector.IncDbQuery("select station", err) + return err +} - if errors.Is(err, sql.ErrNoRows) { - return nil, nil +// peekStationRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekStationRecord(stationId string) (*Station, func(), error) { + if item := stationCache.Get(stationId); item != nil { + station := item.Value() + station.Lock() + return station, func() { station.Unlock() }, nil + } + return nil, nil, nil +} + +// getStationRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + // Check cache first + if item := stationCache.Get(stationId); item != nil { + station := item.Value() + station.Lock() + return station, func() { station.Unlock() }, nil } + dbStation := Station{} + err := loadStationFromDatabase(ctx, db, stationId, &dbStation) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return nil, err + return nil, nil, err + } + dbStation.ClearDirty() + + // Atomically cache the loaded Station - if another goroutine raced us, + // we'll get their Station and use that instead (ensuring same mutex) + existingStation, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + return &dbStation + }) + + station := existingStation.Value() + station.Lock() + return station, func() { station.Unlock() }, nil +} + +// getStationRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getStationRecordForUpdate(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + station, unlock, err := getStationRecordReadOnly(ctx, db, stationId) + if err != nil || station == nil { + return nil, nil, err + } + station.snapshotOldValues() + return station, unlock, nil +} + +// getOrCreateStationRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + // Create new Station atomically - function only called if key doesn't exist + stationItem, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + return &Station{Id: stationId, newRecord: true} + }) + + station := stationItem.Value() + station.Lock() + + if station.newRecord { + // We should attempt to load from database + err := loadStationFromDatabase(ctx, db, stationId, station) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + station.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + station.newRecord = false + station.ClearDirty() + } } - stationCache.Set(stationId, &station, ttlcache.DefaultTTL) + station.snapshotOldValues() - return &station, nil + return station, func() { station.Unlock() }, nil } func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { @@ -590,11 +668,8 @@ func (station *Station) resetStationedPokemonFromStationDetailsNotFound() *Stati func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto) string { stationId := request.StationId - stationMutex, _ := stationStripedMutex.GetLock(stationId) - stationMutex.Lock() - defer stationMutex.Unlock() - station, err := getStationRecord(ctx, db, stationId) + station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) if err != nil { log.Printf("Get station %s", err) return "Error getting station" @@ -604,6 +679,7 @@ func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db. log.Infof("Stationed pokemon details for station %s not found", stationId) return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) } + defer unlock() station.resetStationedPokemonFromStationDetailsNotFound() saveStationRecord(ctx, db, station) @@ -612,11 +688,8 @@ func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db. func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto, stationDetails *pogo.GetStationedPokemonDetailsOutProto) string { stationId := request.StationId - stationMutex, _ := stationStripedMutex.GetLock(stationId) - stationMutex.Lock() - defer stationMutex.Unlock() - station, err := getStationRecord(ctx, db, stationId) + station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) if err != nil { log.Printf("Get station %s", err) return "Error getting station" @@ -626,6 +699,7 @@ func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, reque log.Infof("Stationed pokemon details for station %s not found", stationId) return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) } + defer unlock() station.updateFromGetStationedPokemonDetailsOutProto(stationDetails) saveStationRecord(ctx, db, station) diff --git a/decoder/tappable.go b/decoder/tappable.go index cf089a71..5454039e 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strconv" + "sync" "time" "golbat/db" @@ -19,6 +20,8 @@ import ( // Tappable struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Tappable struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id uint64 `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` Lon float64 `db:"lon" json:"lon"` @@ -52,6 +55,16 @@ func (ta *Tappable) IsNewRecord() bool { return ta.newRecord } +// Lock acquires the Tappable's mutex +func (ta *Tappable) Lock() { + ta.mu.Lock() +} + +// Unlock releases the Tappable's mutex +func (ta *Tappable) Unlock() { + ta.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (ta *Tappable) SetLat(v float64) { @@ -254,28 +267,98 @@ func (ta *Tappable) setUnknownTimestamp(now int64) { } } -func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, error) { - inMemoryTappable := tappableCache.Get(id) - if inMemoryTappable != nil { - tappable := inMemoryTappable.Value() - return tappable, nil - } - tappable := Tappable{} - err := db.GeneralDb.GetContext(ctx, &tappable, +func loadTappableFromDatabase(ctx context.Context, db db.DbDetails, id uint64, tappable *Tappable) error { + err := db.GeneralDb.GetContext(ctx, tappable, `SELECT id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated - FROM tappable - WHERE id = ?`, strconv.FormatUint(id, 10)) + FROM tappable WHERE id = ?`, strconv.FormatUint(id, 10)) statsCollector.IncDbQuery("select tappable", err) + return err +} + +// peekTappableRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekTappableRecord(id uint64) (*Tappable, func(), error) { + if item := tappableCache.Get(id); item != nil { + tappable := item.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil + } + return nil, nil, nil +} + +// getTappableRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getTappableRecordReadOnly(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { + // Check cache first + if item := tappableCache.Get(id); item != nil { + tappable := item.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil + } + + dbTappable := Tappable{} + err := loadTappableFromDatabase(ctx, db, id, &dbTappable) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbTappable.ClearDirty() + + // Atomically cache the loaded Tappable - if another goroutine raced us, + // we'll get their Tappable and use that instead (ensuring same mutex) + existingTappable, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { + return &dbTappable + }) + + tappable := existingTappable.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil +} + +// getOrCreateTappableRecord gets existing or creates new, locked. +// Caller MUST call returned unlock function. +func getOrCreateTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { + // Create new Tappable atomically - function only called if key doesn't exist + tappableItem, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { + return &Tappable{Id: id, newRecord: true} + }) + + tappable := tappableItem.Value() + tappable.Lock() + + if tappable.newRecord { + // We should attempt to load from database + err := loadTappableFromDatabase(ctx, db, id, tappable) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + tappable.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + tappable.newRecord = false + tappable.ClearDirty() + } } + return tappable, func() { tappable.Unlock() }, nil +} + +// GetTappableRecord is an exported function for API use. +// Returns a tappable if found, nil if not found. +func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, error) { + tappable, unlock, err := getTappableRecordReadOnly(ctx, db, id) if err != nil { return nil, err } - - tappableCache.Set(id, &tappable, ttlcache.DefaultTTL) - return &tappable, nil + if tappable == nil { + return nil, nil + } + defer unlock() + return tappable, nil } func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tappable) { @@ -342,19 +425,13 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { id := request.GetEncounterId() - tappableMutex, _ := tappableStripedMutex.GetLock(id) - tappableMutex.Lock() - defer tappableMutex.Unlock() - tappable, err := GetTappableRecord(ctx, db, id) + tappable, unlock, err := getOrCreateTappableRecord(ctx, db, id) if err != nil { - log.Printf("Get tappable %s", err) + log.Printf("getOrCreateTappableRecord: %s", err) return "Error getting tappable" } - - if tappable == nil { - tappable = &Tappable{newRecord: true} - } + defer unlock() tappable.updateFromProcessTappableProto(ctx, db, tappableDetails, request, timestampMs) saveTappableRecord(ctx, db, tappable) diff --git a/decoder/weather.go b/decoder/weather.go index 7aab48a5..7f346761 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -3,6 +3,8 @@ package decoder import ( "context" "database/sql" + "errors" + "sync" "golbat/db" "golbat/pogo" @@ -17,6 +19,8 @@ import ( // Weather struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Weather struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id int64 `db:"id"` Latitude float64 `db:"latitude"` Longitude float64 `db:"longitude"` @@ -79,6 +83,16 @@ func (weather *Weather) IsNewRecord() bool { return weather.newRecord } +// Lock acquires the Weather's mutex +func (weather *Weather) Lock() { + weather.mu.Lock() +} + +// Unlock releases the Weather's mutex +func (weather *Weather) Unlock() { + weather.mu.Unlock() +} + // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (weather *Weather) snapshotOldValues() { @@ -188,30 +202,98 @@ func (weather *Weather) SetWarnWeather(v null.Bool) { } } -func getWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, error) { - inMemoryWeather := weatherCache.Get(weatherId) - if inMemoryWeather != nil { - weather := inMemoryWeather.Value() - weather.snapshotOldValues() - return weather, nil +func loadWeatherFromDatabase(ctx context.Context, db db.DbDetails, weatherId int64, weather *Weather) error { + err := db.GeneralDb.GetContext(ctx, weather, + "SELECT id, latitude, longitude, level, gameplay_condition, wind_direction, cloud_level, rain_level, wind_level, snow_level, fog_level, special_effect_level, severity, warn_weather, updated FROM weather WHERE id = ?", weatherId) + statsCollector.IncDbQuery("select weather", err) + if err == nil { + weather.UpdatedMs *= 1000 } - weather := Weather{} + return err +} - err := db.GeneralDb.GetContext(ctx, &weather, "SELECT id, latitude, longitude, level, gameplay_condition, wind_direction, cloud_level, rain_level, wind_level, snow_level, fog_level, special_effect_level, severity, warn_weather, updated FROM weather WHERE id = ?", weatherId) +// peekWeatherRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekWeatherRecord(weatherId int64) (*Weather, func(), error) { + if item := weatherCache.Get(weatherId); item != nil { + weather := item.Value() + weather.Lock() + return weather, func() { weather.Unlock() }, nil + } + return nil, nil, nil +} - statsCollector.IncDbQuery("select weather", err) - if err == sql.ErrNoRows { - return nil, nil +// getWeatherRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getWeatherRecordReadOnly(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, func(), error) { + // Check cache first + if item := weatherCache.Get(weatherId); item != nil { + weather := item.Value() + weather.Lock() + return weather, func() { weather.Unlock() }, nil } + dbWeather := Weather{} + err := loadWeatherFromDatabase(ctx, db, weatherId, &dbWeather) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return nil, err + return nil, nil, err + } + dbWeather.ClearDirty() + + // Atomically cache the loaded Weather - if another goroutine raced us, + // we'll get their Weather and use that instead (ensuring same mutex) + existingWeather, _ := weatherCache.GetOrSetFunc(weatherId, func() *Weather { + return &dbWeather + }) + + weather := existingWeather.Value() + weather.Lock() + return weather, func() { weather.Unlock() }, nil +} + +// getWeatherRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getWeatherRecordForUpdate(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, func(), error) { + weather, unlock, err := getWeatherRecordReadOnly(ctx, db, weatherId) + if err != nil || weather == nil { + return nil, nil, err + } + weather.snapshotOldValues() + return weather, unlock, nil +} + +// getOrCreateWeatherRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, func(), error) { + // Create new Weather atomically - function only called if key doesn't exist + weatherItem, _ := weatherCache.GetOrSetFunc(weatherId, func() *Weather { + return &Weather{Id: weatherId, newRecord: true} + }) + + weather := weatherItem.Value() + weather.Lock() + + if weather.newRecord { + // We should attempt to load from database + err := loadWeatherFromDatabase(ctx, db, weatherId, weather) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + weather.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + weather.newRecord = false + weather.ClearDirty() + } } - weather.UpdatedMs *= 1000 weather.snapshotOldValues() - weatherCache.Set(weatherId, &weather, ttlcache.DefaultTTL) - return &weather, nil + return weather, func() { weather.Unlock() }, nil } func weatherCellIdFromLatLon(lat, lon float64) int64 { diff --git a/main.go b/main.go index 209599b8..bf486171 100644 --- a/main.go +++ b/main.go @@ -454,7 +454,7 @@ func decode(ctx context.Context, method int, protoData *ProtoData) { result = decodeGetMapForts(ctx, protoData.Data) processed = true case pogo.Method_METHOD_GET_ROUTES: - result = decodeGetRoutes(protoData.Data) + result = decodeGetRoutes(ctx, protoData.Data) processed = true case pogo.Method_METHOD_GET_CONTEST_DATA: if getScanParameters(protoData).ProcessPokestops { @@ -690,7 +690,7 @@ func decodeGetMapForts(ctx context.Context, sDec []byte) string { return "No forts updated" } -func decodeGetRoutes(payload []byte) string { +func decodeGetRoutes(ctx context.Context, payload []byte) string { getRoutesOutProto := &pogo.GetRoutesOutProto{} if err := proto.Unmarshal(payload, getRoutesOutProto); err != nil { return fmt.Sprintf("failed to decode GetRoutesOutProto %s", err) @@ -711,7 +711,7 @@ func decodeGetRoutes(payload []byte) string { log.Warnf("Non published Route found in GetRoutesOutProto, status: %s", routeSubmissionStatus.String()) continue } - decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(dbDetails, route) + decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(ctx, dbDetails, route) if decodeError != nil { if decodeErrors[route.Id] != true { decodeErrors[route.Id] = true From 0f2bd79065a48bb400ee661181f46c324ce84538 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 21:22:32 +0000 Subject: [PATCH 25/78] spawnpoint locking --- decoder/pokemon.go | 6 +- decoder/spawnpoint.go | 136 +++++++++++++++++++++++++++++++----------- decoder/tappable.go | 6 +- 3 files changed, 111 insertions(+), 37 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 1f5494e2..47096a4d 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -1221,9 +1221,10 @@ func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db } pokemon.ExpireTimestampVerified = false - spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) + spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) if spawnPoint != nil && spawnPoint.DespawnSec.Valid { despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) + unlock() date := time.Unix(timestampMs/1000, 0) secondOfHour := date.Second() + date.Minute()*60 @@ -1235,6 +1236,9 @@ func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) pokemon.SetExpireTimestampVerified(true) } else { + if unlock != nil { + unlock() + } pokemon.setUnknownTimestamp(timestampMs / 1000) } } diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 176eac2b..7e1d8811 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -3,7 +3,9 @@ package decoder import ( "context" "database/sql" + "errors" "strconv" + "sync" "time" "golbat/db" @@ -17,6 +19,8 @@ import ( // Spawnpoint struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Spawnpoint struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id int64 `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` @@ -56,6 +60,16 @@ func (s *Spawnpoint) IsNewRecord() bool { return s.newRecord } +// Lock acquires the Spawnpoint's mutex +func (s *Spawnpoint) Lock() { + s.mu.Lock() +} + +// Unlock releases the Spawnpoint's mutex +func (s *Spawnpoint) Unlock() { + s.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (s *Spawnpoint) SetLat(v float64) { @@ -105,28 +119,82 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { } } -func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, error) { - inMemorySpawnpoint := spawnpointCache.Get(spawnpointId) - if inMemorySpawnpoint != nil { - spawnpoint := inMemorySpawnpoint.Value() - return spawnpoint, nil - } - spawnpoint := Spawnpoint{} +func loadSpawnpointFromDatabase(ctx context.Context, db db.DbDetails, spawnpointId int64, spawnpoint *Spawnpoint) error { + err := db.GeneralDb.GetContext(ctx, spawnpoint, + "SELECT id, lat, lon, updated, last_seen, despawn_sec FROM spawnpoint WHERE id = ?", spawnpointId) + statsCollector.IncDbQuery("select spawnpoint", err) + return err +} - err := db.GeneralDb.GetContext(ctx, &spawnpoint, "SELECT id, lat, lon, updated, last_seen, despawn_sec FROM spawnpoint WHERE id = ?", spawnpointId) +// peekSpawnpointRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekSpawnpointRecord(spawnpointId int64) (*Spawnpoint, func(), error) { + if item := spawnpointCache.Get(spawnpointId); item != nil { + spawnpoint := item.Value() + spawnpoint.Lock() + return spawnpoint, func() { spawnpoint.Unlock() }, nil + } + return nil, nil, nil +} - statsCollector.IncDbQuery("select spawnpoint", err) - if err == sql.ErrNoRows { - return nil, nil +// getSpawnpointRecord acquires lock. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, func(), error) { + // Check cache first + if item := spawnpointCache.Get(spawnpointId); item != nil { + spawnpoint := item.Value() + spawnpoint.Lock() + return spawnpoint, func() { spawnpoint.Unlock() }, nil } + dbSpawnpoint := Spawnpoint{} + err := loadSpawnpointFromDatabase(ctx, db, spawnpointId, &dbSpawnpoint) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return &Spawnpoint{Id: spawnpointId}, err + return nil, nil, err } + dbSpawnpoint.ClearDirty() - spawnpointCache.Set(spawnpointId, &spawnpoint, ttlcache.DefaultTTL) + // Atomically cache the loaded Spawnpoint - if another goroutine raced us, + // we'll get their Spawnpoint and use that instead (ensuring same mutex) + existingSpawnpoint, _ := spawnpointCache.GetOrSetFunc(spawnpointId, func() *Spawnpoint { + return &dbSpawnpoint + }) - return &spawnpoint, nil + spawnpoint := existingSpawnpoint.Value() + spawnpoint.Lock() + return spawnpoint, func() { spawnpoint.Unlock() }, nil +} + +// getOrCreateSpawnpointRecord gets existing or creates new, locked. +// Caller MUST call returned unlock function. +func getOrCreateSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, func(), error) { + // Create new Spawnpoint atomically - function only called if key doesn't exist + spawnpointItem, _ := spawnpointCache.GetOrSetFunc(spawnpointId, func() *Spawnpoint { + return &Spawnpoint{Id: spawnpointId, newRecord: true} + }) + + spawnpoint := spawnpointItem.Value() + spawnpoint.Lock() + + if spawnpoint.newRecord { + // We should attempt to load from database + err := loadSpawnpointFromDatabase(ctx, db, spawnpointId, spawnpoint) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + spawnpoint.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + spawnpoint.newRecord = false + spawnpoint.ClearDirty() + } + } + + return spawnpoint, func() { spawnpoint.Unlock() }, nil } func Abs(x int64) int64 { @@ -148,27 +216,30 @@ func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon date := time.Unix(expireTimeStamp, 0) secondOfHour := date.Second() + date.Minute()*60 - spawnpoint, _ := getSpawnpointRecord(ctx, db, spawnId) - if spawnpoint == nil { - spawnpoint = &Spawnpoint{Id: spawnId, newRecord: true} + spawnpoint, unlock, err := getOrCreateSpawnpointRecord(ctx, db, spawnId) + if err != nil { + log.Errorf("getOrCreateSpawnpointRecord: %s", err) + return } spawnpoint.SetLat(wildPokemon.Latitude) spawnpoint.SetLon(wildPokemon.Longitude) spawnpoint.SetDespawnSec(null.IntFrom(int64(secondOfHour))) spawnpointUpdate(ctx, db, spawnpoint) + unlock() } else { - spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) - if spawnPoint == nil { - spawnpoint := &Spawnpoint{ - Id: spawnId, - Lat: wildPokemon.Latitude, - Lon: wildPokemon.Longitude, - newRecord: true, - } + spawnpoint, unlock, err := getOrCreateSpawnpointRecord(ctx, db, spawnId) + if err != nil { + log.Errorf("getOrCreateSpawnpointRecord: %s", err) + return + } + if spawnpoint.newRecord { + spawnpoint.SetLat(wildPokemon.Latitude) + spawnpoint.SetLon(wildPokemon.Longitude) spawnpointUpdate(ctx, db, spawnpoint) } else { - spawnpointSeen(ctx, db, spawnId) + spawnpointSeen(ctx, db, spawnpoint) } + unlock() } } @@ -203,14 +274,9 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi } } -func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { - inMemorySpawnpoint := spawnpointCache.Get(spawnpointId) - if inMemorySpawnpoint == nil { - // This should never happen, since all routes here have previously created a spawnpoint in the cache - return - } - - spawnpoint := inMemorySpawnpoint.Value() +// spawnpointSeen updates the last_seen timestamp for a spawnpoint. +// The spawnpoint must already be locked by the caller. +func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoint) { now := time.Now().Unix() // update at least every 6 hours (21600s). If reduce_updates is enabled, use 12 hours. @@ -219,7 +285,7 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ "SET last_seen=? "+ - "WHERE id = ? ", now, spawnpointId) + "WHERE id = ? ", now, spawnpoint.Id) statsCollector.IncDbQuery("update spawnpoint", err) if err != nil { log.Printf("Error updating spawnpoint last seen %s", err) diff --git a/decoder/tappable.go b/decoder/tappable.go index 5454039e..3a498ef2 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -235,9 +235,10 @@ func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.Db func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, timestampMs int64) { ta.SetExpireTimestampVerified(false) if spawnId := ta.SpawnId.ValueOrZero(); spawnId != 0 { - spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) + spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) if spawnPoint != nil && spawnPoint.DespawnSec.Valid { despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) + unlock() date := time.Unix(timestampMs/1000, 0) secondOfHour := date.Second() + date.Minute()*60 @@ -249,6 +250,9 @@ func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, tim ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) ta.SetExpireTimestampVerified(true) } else { + if unlock != nil { + unlock() + } ta.setUnknownTimestamp(timestampMs / 1000) } } else if fortId := ta.FortId.ValueOrZero(); fortId != "" { From 07efbe6858e7435e2ab190a5fe3aec015e1ff5e7 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 31 Jan 2026 14:22:56 +0000 Subject: [PATCH 26/78] Tidy source --- decode.go | 738 ++++++++++++++ decoder/api_gym.go | 96 ++ decoder/api_pokemon_common.go | 2 +- decoder/api_pokestop.go | 101 ++ decoder/api_tappable.go | 39 + decoder/fort.go | 18 +- decoder/fortRtree.go | 2 +- decoder/gmo_decode.go | 224 +++++ decoder/gym.go | 832 +-------------- decoder/gym_decode.go | 311 ++++++ decoder/gym_process.go | 97 ++ decoder/gym_state.go | 430 ++++++++ decoder/incident.go | 278 +---- decoder/incident_decode.go | 54 + decoder/incident_process.go | 43 + decoder/incident_state.go | 234 +++++ decoder/main.go | 250 +---- decoder/player.go | 2 +- decoder/pokemon.go | 1512 +--------------------------- decoder/pokemonRtree.go | 2 +- decoder/pokemon_decode.go | 968 ++++++++++++++++++ decoder/pokemon_process.go | 92 ++ decoder/pokemon_state.go | 486 +++++++++ decoder/pokestop.go | 1013 +------------------ decoder/pokestop_decode.go | 475 +++++++++ decoder/pokestop_process.go | 167 +++ decoder/pokestop_showcase.go | 2 +- decoder/pokestop_state.go | 397 ++++++++ decoder/routes.go | 249 +---- decoder/routes_decode.go | 51 + decoder/routes_process.go | 20 + decoder/routes_state.go | 196 ++++ decoder/s2cell.go | 2 +- decoder/spawnpoint.go | 2 +- decoder/station.go | 388 +------ decoder/station_decode.go | 106 ++ decoder/station_process.go | 51 + decoder/station_state.go | 253 +++++ decoder/tappable.go | 288 +----- decoder/tappable_decode.go | 117 +++ decoder/tappable_process.go | 26 + decoder/tappable_state.go | 157 +++ decoder/weather.go | 2 +- decoder/weather_iv.go | 2 +- go.mod | 3 +- go.sum | 6 +- main.go | 743 +------------- routes.go | 60 +- stats_collector/noop.go | 2 +- stats_collector/prometheus.go | 5 +- stats_collector/stats_collector.go | 2 +- 51 files changed, 6016 insertions(+), 5580 deletions(-) create mode 100644 decode.go create mode 100644 decoder/api_pokestop.go create mode 100644 decoder/api_tappable.go create mode 100644 decoder/gmo_decode.go create mode 100644 decoder/gym_decode.go create mode 100644 decoder/gym_process.go create mode 100644 decoder/gym_state.go create mode 100644 decoder/incident_decode.go create mode 100644 decoder/incident_process.go create mode 100644 decoder/incident_state.go create mode 100644 decoder/pokemon_decode.go create mode 100644 decoder/pokemon_process.go create mode 100644 decoder/pokemon_state.go create mode 100644 decoder/pokestop_decode.go create mode 100644 decoder/pokestop_process.go create mode 100644 decoder/pokestop_state.go create mode 100644 decoder/routes_decode.go create mode 100644 decoder/routes_process.go create mode 100644 decoder/routes_state.go create mode 100644 decoder/station_decode.go create mode 100644 decoder/station_process.go create mode 100644 decoder/station_state.go create mode 100644 decoder/tappable_decode.go create mode 100644 decoder/tappable_process.go create mode 100644 decoder/tappable_state.go diff --git a/decode.go b/decode.go new file mode 100644 index 00000000..daab1cb3 --- /dev/null +++ b/decode.go @@ -0,0 +1,738 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" + + "golbat/decoder" + "golbat/pogo" + + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" +) + +func decode(ctx context.Context, method int, protoData *ProtoData) { + getMethodName := func(method int, trimString bool) string { + if val, ok := pogo.Method_name[int32(method)]; ok { + if trimString && strings.HasPrefix(val, "METHOD_") { + return strings.TrimPrefix(val, "METHOD_") + } + return val + } + return fmt.Sprintf("#%d", method) + } + + if method != int(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION) && protoData.Level < 30 { + statsCollector.IncDecodeMethods("error", "low_level", getMethodName(method, true)) + log.Debugf("Insufficient Level %d Did not process hook type %s", protoData.Level, pogo.Method(method)) + return + } + + processed := false + ignore := false + start := time.Now() + result := "" + + switch pogo.Method(method) { + case pogo.Method_METHOD_START_INCIDENT: + result = decodeStartIncident(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_INVASION_OPEN_COMBAT_SESSION: + if protoData.Request != nil { + result = decodeOpenInvasion(ctx, protoData.Request, protoData.Data) + processed = true + } + case pogo.Method_METHOD_FORT_DETAILS: + result = decodeFortDetails(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_GET_MAP_OBJECTS: + result = decodeGMO(ctx, protoData, getScanParameters(protoData)) + processed = true + case pogo.Method_METHOD_GYM_GET_INFO: + result = decodeGetGymInfo(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_ENCOUNTER: + if getScanParameters(protoData).ProcessPokemon { + result = decodeEncounter(ctx, protoData.Data, protoData.Account, protoData.TimestampMs) + } + processed = true + case pogo.Method_METHOD_DISK_ENCOUNTER: + result = decodeDiskEncounter(ctx, protoData.Data, protoData.Account) + processed = true + case pogo.Method_METHOD_FORT_SEARCH: + result = decodeQuest(ctx, protoData.Data, protoData.HaveAr) + processed = true + case pogo.Method_METHOD_GET_PLAYER: + ignore = true + case pogo.Method_METHOD_GET_HOLOHOLO_INVENTORY: + ignore = true + case pogo.Method_METHOD_CREATE_COMBAT_CHALLENGE: + ignore = true + case pogo.Method(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION): + if protoData.Request != nil { + result = decodeSocialActionWithRequest(protoData.Request, protoData.Data) + processed = true + } + case pogo.Method_METHOD_GET_MAP_FORTS: + result = decodeGetMapForts(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_GET_ROUTES: + result = decodeGetRoutes(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_GET_CONTEST_DATA: + if getScanParameters(protoData).ProcessPokestops { + // Request helps, but can be decoded without it + result = decodeGetContestData(ctx, protoData.Request, protoData.Data) + } + processed = true + case pogo.Method_METHOD_GET_POKEMON_SIZE_CONTEST_ENTRY: + // Request is essential to decode this + if protoData.Request != nil { + if getScanParameters(protoData).ProcessPokestops { + result = decodeGetPokemonSizeContestEntry(ctx, protoData.Request, protoData.Data) + } + processed = true + } + case pogo.Method_METHOD_GET_STATION_DETAILS: + if getScanParameters(protoData).ProcessStations { + // Request is essential to decode this + result = decodeGetStationDetails(ctx, protoData.Request, protoData.Data) + } + processed = true + case pogo.Method_METHOD_PROCESS_TAPPABLE: + if getScanParameters(protoData).ProcessTappables { + // Request is essential to decode this + result = decodeTappable(ctx, protoData.Request, protoData.Data, protoData.Account, protoData.TimestampMs) + } + processed = true + case pogo.Method_METHOD_GET_EVENT_RSVPS: + if getScanParameters(protoData).ProcessGyms { + result = decodeGetEventRsvp(ctx, protoData.Request, protoData.Data) + } + processed = true + case pogo.Method_METHOD_GET_EVENT_RSVP_COUNT: + if getScanParameters(protoData).ProcessGyms { + result = decodeGetEventRsvpCount(ctx, protoData.Data) + } + processed = true + default: + log.Debugf("Did not know hook type %s", pogo.Method(method)) + } + if !ignore { + elapsed := time.Since(start) + if processed == true { + statsCollector.IncDecodeMethods("ok", "", getMethodName(method, true)) + log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, result) + } else { + log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, "**Did not process**") + statsCollector.IncDecodeMethods("unprocessed", "", getMethodName(method, true)) + } + } +} + +func getScanParameters(protoData *ProtoData) decoder.ScanParameters { + return decoder.FindScanConfiguration(protoData.ScanContext, protoData.Lat, protoData.Lon) +} + +func decodeQuest(ctx context.Context, sDec []byte, haveAr *bool) string { + if haveAr == nil { + statsCollector.IncDecodeQuest("error", "missing_ar_info") + log.Infoln("Cannot determine AR quest - ignoring") + // We should either assume AR quest, or trace inventory like RDM probably + return "No AR quest info" + } + decodedQuest := &pogo.FortSearchOutProto{} + if err := proto.Unmarshal(sDec, decodedQuest); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeQuest("error", "parse") + return "Parse failure" + } + + if decodedQuest.Result != pogo.FortSearchOutProto_SUCCESS { + statsCollector.IncDecodeQuest("error", "non_success") + res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedQuest.Result, + pogo.FortSearchOutProto_Result_name[int32(decodedQuest.Result)]) + return res + } + + return decoder.UpdatePokestopWithQuest(ctx, dbDetails, decodedQuest, *haveAr) + +} + +func decodeSocialActionWithRequest(request []byte, payload []byte) string { + var proxyRequestProto pogo.ProxyRequestProto + + if err := proto.Unmarshal(request, &proxyRequestProto); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeSocialActionWithRequest("error", "request_parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + var proxyResponseProto pogo.ProxyResponseProto + + if err := proto.Unmarshal(payload, &proxyResponseProto); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeSocialActionWithRequest("error", "response_parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED && proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED_AND_REASSIGNED { + statsCollector.IncDecodeSocialActionWithRequest("error", "non_success") + return fmt.Sprintf("unsuccessful proxyResponseProto response %d %s", int(proxyResponseProto.Status), proxyResponseProto.Status) + } + + switch pogo.InternalSocialAction(proxyRequestProto.GetAction()) { + case pogo.InternalSocialAction_SOCIAL_ACTION_LIST_FRIEND_STATUS: + statsCollector.IncDecodeSocialActionWithRequest("ok", "list_friend_status") + return decodeGetFriendDetails(proxyResponseProto.Payload) + case pogo.InternalSocialAction_SOCIAL_ACTION_SEARCH_PLAYER: + statsCollector.IncDecodeSocialActionWithRequest("ok", "search_player") + return decodeSearchPlayer(&proxyRequestProto, proxyResponseProto.Payload) + + } + + statsCollector.IncDecodeSocialActionWithRequest("ok", "unknown") + return fmt.Sprintf("Did not process %s", pogo.InternalSocialAction(proxyRequestProto.GetAction()).String()) +} + +func decodeGetFriendDetails(payload []byte) string { + var getFriendDetailsOutProto pogo.InternalGetFriendDetailsOutProto + getFriendDetailsError := proto.Unmarshal(payload, &getFriendDetailsOutProto) + + if getFriendDetailsError != nil { + statsCollector.IncDecodeGetFriendDetails("error", "parse") + log.Errorf("Failed to parse %s", getFriendDetailsError) + return fmt.Sprintf("Failed to parse %s", getFriendDetailsError) + } + + if getFriendDetailsOutProto.GetResult() != pogo.InternalGetFriendDetailsOutProto_SUCCESS || getFriendDetailsOutProto.GetFriend() == nil { + statsCollector.IncDecodeGetFriendDetails("error", "non_success") + return fmt.Sprintf("unsuccessful get friends details") + } + + failures := 0 + + for _, friend := range getFriendDetailsOutProto.GetFriend() { + player := friend.GetPlayer() + + updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, "", player.GetPlayerId()) + if updatePlayerError != nil { + failures++ + } + } + + statsCollector.IncDecodeGetFriendDetails("ok", "") + return fmt.Sprintf("%d players decoded on %d", len(getFriendDetailsOutProto.GetFriend())-failures, len(getFriendDetailsOutProto.GetFriend())) +} + +func decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byte) string { + var searchPlayerOutProto pogo.InternalSearchPlayerOutProto + searchPlayerOutError := proto.Unmarshal(payload, &searchPlayerOutProto) + + if searchPlayerOutError != nil { + log.Errorf("Failed to parse %s", searchPlayerOutError) + statsCollector.IncDecodeSearchPlayer("error", "parse") + return fmt.Sprintf("Failed to parse %s", searchPlayerOutError) + } + + if searchPlayerOutProto.GetResult() != pogo.InternalSearchPlayerOutProto_SUCCESS || searchPlayerOutProto.GetPlayer() == nil { + statsCollector.IncDecodeSearchPlayer("error", "non_success") + return fmt.Sprintf("unsuccessful search player response") + } + + var searchPlayerProto pogo.InternalSearchPlayerProto + searchPlayerError := proto.Unmarshal(proxyRequestProto.GetPayload(), &searchPlayerProto) + + if searchPlayerError != nil || searchPlayerProto.GetFriendCode() == "" { + statsCollector.IncDecodeSearchPlayer("error", "parse") + return fmt.Sprintf("Failed to parse %s", searchPlayerError) + } + + player := searchPlayerOutProto.GetPlayer() + updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, searchPlayerProto.GetFriendCode(), "") + if updatePlayerError != nil { + statsCollector.IncDecodeSearchPlayer("error", "update") + return fmt.Sprintf("Failed update player %s", updatePlayerError) + } + + statsCollector.IncDecodeSearchPlayer("ok", "") + return fmt.Sprintf("1 player decoded from SearchPlayerProto") +} + +func decodeFortDetails(ctx context.Context, sDec []byte) string { + decodedFort := &pogo.FortDetailsOutProto{} + if err := proto.Unmarshal(sDec, decodedFort); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeFortDetails("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + switch decodedFort.FortType { + case pogo.FortType_CHECKPOINT: + statsCollector.IncDecodeFortDetails("ok", "pokestop") + return decoder.UpdatePokestopRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) + case pogo.FortType_GYM: + statsCollector.IncDecodeFortDetails("ok", "gym") + return decoder.UpdateGymRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) + } + + statsCollector.IncDecodeFortDetails("ok", "unknown") + return "Unknown fort type" +} + +func decodeGetMapForts(ctx context.Context, sDec []byte) string { + decodedMapForts := &pogo.GetMapFortsOutProto{} + if err := proto.Unmarshal(sDec, decodedMapForts); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeGetMapForts("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedMapForts.Status != pogo.GetMapFortsOutProto_SUCCESS { + statsCollector.IncDecodeGetMapForts("error", "non_success") + res := fmt.Sprintf(`GetMapFortsOutProto: Ignored non-success value %d:%s`, decodedMapForts.Status, + pogo.GetMapFortsOutProto_Status_name[int32(decodedMapForts.Status)]) + return res + } + + statsCollector.IncDecodeGetMapForts("ok", "") + var outputString string + processedForts := 0 + + for _, fort := range decodedMapForts.Fort { + status, output := decoder.UpdateFortRecordWithGetMapFortsOutProto(ctx, dbDetails, fort) + if status { + processedForts += 1 + outputString += output + ", " + } + } + + if processedForts > 0 { + return fmt.Sprintf("Updated %d forts: %s", processedForts, outputString) + } + return "No forts updated" +} + +func decodeGetRoutes(ctx context.Context, payload []byte) string { + getRoutesOutProto := &pogo.GetRoutesOutProto{} + if err := proto.Unmarshal(payload, getRoutesOutProto); err != nil { + return fmt.Sprintf("failed to decode GetRoutesOutProto %s", err) + } + + if getRoutesOutProto.Status != pogo.GetRoutesOutProto_SUCCESS { + return fmt.Sprintf("GetRoutesOutProto: Ignored non-success value %d:%s", getRoutesOutProto.Status, getRoutesOutProto.Status.String()) + } + + decodeSuccesses := map[string]bool{} + decodeErrors := map[string]bool{} + + for _, routeMapCell := range getRoutesOutProto.GetRouteMapCell() { + for _, route := range routeMapCell.GetRoute() { + //TODO we need to check the repeated field, for now access last element + routeSubmissionStatus := route.RouteSubmissionStatus[len(route.RouteSubmissionStatus)-1] + if routeSubmissionStatus != nil && routeSubmissionStatus.Status != pogo.RouteSubmissionStatus_PUBLISHED { + log.Warnf("Non published Route found in GetRoutesOutProto, status: %s", routeSubmissionStatus.String()) + continue + } + decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(ctx, dbDetails, route) + if decodeError != nil { + if decodeErrors[route.Id] != true { + decodeErrors[route.Id] = true + } + log.Errorf("Failed to decode route %s", decodeError) + } else if decodeSuccesses[route.Id] != true { + decodeSuccesses[route.Id] = true + } + } + } + + return fmt.Sprintf( + "Decoded %d routes, failed to decode %d routes, from %d cells", + len(decodeSuccesses), + len(decodeErrors), + len(getRoutesOutProto.GetRouteMapCell()), + ) +} + +func decodeGetGymInfo(ctx context.Context, sDec []byte) string { + decodedGymInfo := &pogo.GymGetInfoOutProto{} + if err := proto.Unmarshal(sDec, decodedGymInfo); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeGetGymInfo("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedGymInfo.Result != pogo.GymGetInfoOutProto_SUCCESS { + statsCollector.IncDecodeGetGymInfo("error", "non_success") + res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedGymInfo.Result, + pogo.GymGetInfoOutProto_Result_name[int32(decodedGymInfo.Result)]) + return res + } + + statsCollector.IncDecodeGetGymInfo("ok", "") + return decoder.UpdateGymRecordWithGymInfoProto(ctx, dbDetails, decodedGymInfo) +} + +func decodeEncounter(ctx context.Context, sDec []byte, username string, timestampMs int64) string { + decodedEncounterInfo := &pogo.EncounterOutProto{} + if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeEncounter("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedEncounterInfo.Status != pogo.EncounterOutProto_ENCOUNTER_SUCCESS { + statsCollector.IncDecodeEncounter("error", "non_success") + res := fmt.Sprintf(`EncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Status, + pogo.EncounterOutProto_Status_name[int32(decodedEncounterInfo.Status)]) + return res + } + + statsCollector.IncDecodeEncounter("ok", "") + return decoder.UpdatePokemonRecordWithEncounterProto(ctx, dbDetails, decodedEncounterInfo, username, timestampMs) +} + +func decodeDiskEncounter(ctx context.Context, sDec []byte, username string) string { + decodedEncounterInfo := &pogo.DiskEncounterOutProto{} + if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeDiskEncounter("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedEncounterInfo.Result != pogo.DiskEncounterOutProto_SUCCESS { + statsCollector.IncDecodeDiskEncounter("error", "non_success") + res := fmt.Sprintf(`DiskEncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Result, + pogo.DiskEncounterOutProto_Result_name[int32(decodedEncounterInfo.Result)]) + return res + } + + statsCollector.IncDecodeDiskEncounter("ok", "") + return decoder.UpdatePokemonRecordWithDiskEncounterProto(ctx, dbDetails, decodedEncounterInfo, username) +} + +func decodeStartIncident(ctx context.Context, sDec []byte) string { + decodedIncident := &pogo.StartIncidentOutProto{} + if err := proto.Unmarshal(sDec, decodedIncident); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeStartIncident("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedIncident.Status != pogo.StartIncidentOutProto_SUCCESS { + statsCollector.IncDecodeStartIncident("error", "non_success") + res := fmt.Sprintf(`GiovanniOutProto: Ignored non-success value %d:%s`, decodedIncident.Status, + pogo.StartIncidentOutProto_Status_name[int32(decodedIncident.Status)]) + return res + } + + statsCollector.IncDecodeStartIncident("ok", "") + return decoder.ConfirmIncident(ctx, dbDetails, decodedIncident) +} + +func decodeOpenInvasion(ctx context.Context, request []byte, payload []byte) string { + decodeOpenInvasionRequest := &pogo.OpenInvasionCombatSessionProto{} + + if err := proto.Unmarshal(request, decodeOpenInvasionRequest); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeOpenInvasion("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + if decodeOpenInvasionRequest.IncidentLookup == nil { + return "Invalid OpenInvasionCombatSessionProto received" + } + + decodedOpenInvasionResponse := &pogo.OpenInvasionCombatSessionOutProto{} + if err := proto.Unmarshal(payload, decodedOpenInvasionResponse); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeOpenInvasion("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedOpenInvasionResponse.Status != pogo.InvasionStatus_SUCCESS { + statsCollector.IncDecodeOpenInvasion("error", "non_success") + res := fmt.Sprintf(`InvasionLineupOutProto: Ignored non-success value %d:%s`, decodedOpenInvasionResponse.Status, + pogo.InvasionStatus_Status_name[int32(decodedOpenInvasionResponse.Status)]) + return res + } + + statsCollector.IncDecodeOpenInvasion("ok", "") + return decoder.UpdateIncidentLineup(ctx, dbDetails, decodeOpenInvasionRequest, decodedOpenInvasionResponse) +} + +func decodeGMO(ctx context.Context, protoData *ProtoData, scanParameters decoder.ScanParameters) string { + decodedGmo := &pogo.GetMapObjectsOutProto{} + + if err := proto.Unmarshal(protoData.Data, decodedGmo); err != nil { + statsCollector.IncDecodeGMO("error", "parse") + log.Errorf("Failed to parse %s", err) + } + + if decodedGmo.Status != pogo.GetMapObjectsOutProto_SUCCESS { + statsCollector.IncDecodeGMO("error", "non_success") + res := fmt.Sprintf(`GetMapObjectsOutProto: Ignored non-success value %d:%s`, decodedGmo.Status, + pogo.GetMapObjectsOutProto_Status_name[int32(decodedGmo.Status)]) + return res + } + + var newForts []decoder.RawFortData + var newStations []decoder.RawStationData + var newWildPokemon []decoder.RawWildPokemonData + var newNearbyPokemon []decoder.RawNearbyPokemonData + var newMapPokemon []decoder.RawMapPokemonData + var newMapCells []uint64 + var cellsToBeCleaned []uint64 + + // track forts per cell for memory-based cleanup (only if tracker enabled) + cellForts := make(map[uint64]*decoder.FortTrackerGMOContents) + + if len(decodedGmo.MapCell) == 0 { + return "Skipping GetMapObjectsOutProto: No map cells found" + } + for _, mapCell := range decodedGmo.MapCell { + // initialize cell forts tracking for every map cell (so empty fort lists are seen as "no forts") + cellForts[mapCell.S2CellId] = &decoder.FortTrackerGMOContents{ + Pokestops: make([]string, 0), + Gyms: make([]string, 0), + Timestamp: mapCell.AsOfTimeMs, + } + // always mark this mapCell to be checked for removed forts. Previously only cells with forts were + // added which meant an empty fort list (all forts removed) was never passed to the tracker. + cellsToBeCleaned = append(cellsToBeCleaned, mapCell.S2CellId) + + if isCellNotEmpty(mapCell) { + newMapCells = append(newMapCells, mapCell.S2CellId) + } + + for _, fort := range mapCell.Fort { + newForts = append(newForts, decoder.RawFortData{Cell: mapCell.S2CellId, Data: fort, Timestamp: mapCell.AsOfTimeMs}) + + // track fort by type for memory-based cleanup (only if tracker enabled) + if cf, ok := cellForts[mapCell.S2CellId]; ok { + switch fort.FortType { + case pogo.FortType_GYM: + cf.Gyms = append(cf.Gyms, fort.FortId) + case pogo.FortType_CHECKPOINT: + cf.Pokestops = append(cf.Pokestops, fort.FortId) + } + } + + if fort.ActivePokemon != nil { + newMapPokemon = append(newMapPokemon, decoder.RawMapPokemonData{Cell: mapCell.S2CellId, Data: fort.ActivePokemon, Timestamp: mapCell.AsOfTimeMs}) + } + } + for _, mon := range mapCell.WildPokemon { + newWildPokemon = append(newWildPokemon, decoder.RawWildPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) + } + for _, mon := range mapCell.NearbyPokemon { + newNearbyPokemon = append(newNearbyPokemon, decoder.RawNearbyPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) + } + for _, station := range mapCell.Stations { + newStations = append(newStations, decoder.RawStationData{Cell: mapCell.S2CellId, Data: station}) + } + } + + if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { + decoder.UpdateFortBatch(ctx, dbDetails, scanParameters, newForts) + } + var weatherUpdates []decoder.WeatherUpdate + if scanParameters.ProcessWeather { + weatherUpdates = decoder.UpdateClientWeatherBatch(ctx, dbDetails, decodedGmo.ClientWeather, decodedGmo.MapCell[0].AsOfTimeMs, protoData.Account) + } + if scanParameters.ProcessPokemon { + decoder.UpdatePokemonBatch(ctx, dbDetails, scanParameters, newWildPokemon, newNearbyPokemon, newMapPokemon, decodedGmo.ClientWeather, protoData.Account) + if scanParameters.ProcessWeather && scanParameters.ProactiveIVSwitching { + for _, weatherUpdate := range weatherUpdates { + go func(weatherUpdate decoder.WeatherUpdate) { + decoder.ProactiveIVSwitchSem <- true + defer func() { <-decoder.ProactiveIVSwitchSem }() + decoder.ProactiveIVSwitch(ctx, dbDetails, weatherUpdate, scanParameters.ProactiveIVSwitchingToDB, decodedGmo.MapCell[0].AsOfTimeMs/1000) + }(weatherUpdate) + } + } + } + if scanParameters.ProcessStations { + decoder.UpdateStationBatch(ctx, dbDetails, scanParameters, newStations) + } + + if scanParameters.ProcessCells { + decoder.UpdateClientMapS2CellBatch(ctx, dbDetails, newMapCells) + } + + if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + decoder.CheckRemovedForts(ctx, dbDetails, cellsToBeCleaned, cellForts) + }() + } + + newFortsLen := len(newForts) + newStationsLen := len(newStations) + newWildPokemonLen := len(newWildPokemon) + newNearbyPokemonLen := len(newNearbyPokemon) + newMapPokemonLen := len(newMapPokemon) + newClientWeatherLen := len(decodedGmo.ClientWeather) + newMapCellsLen := len(newMapCells) + + statsCollector.IncDecodeGMO("ok", "") + statsCollector.AddDecodeGMOType("fort", float64(newFortsLen)) + statsCollector.AddDecodeGMOType("station", float64(newStationsLen)) + statsCollector.AddDecodeGMOType("wild_pokemon", float64(newWildPokemonLen)) + statsCollector.AddDecodeGMOType("nearby_pokemon", float64(newNearbyPokemonLen)) + statsCollector.AddDecodeGMOType("map_pokemon", float64(newMapPokemonLen)) + statsCollector.AddDecodeGMOType("weather", float64(newClientWeatherLen)) + statsCollector.AddDecodeGMOType("cell", float64(newMapCellsLen)) + + return fmt.Sprintf("%d cells containing %d forts %d stations %d mon %d nearby", newMapCellsLen, newFortsLen, newStationsLen, newWildPokemonLen, newNearbyPokemonLen) +} + +func isCellNotEmpty(mapCell *pogo.ClientMapCellProto) bool { + return len(mapCell.Stations) > 0 || len(mapCell.Fort) > 0 || len(mapCell.WildPokemon) > 0 || len(mapCell.NearbyPokemon) > 0 || len(mapCell.CatchablePokemon) > 0 +} + +func cellContainsForts(mapCell *pogo.ClientMapCellProto) bool { + return len(mapCell.Fort) > 0 +} + +func decodeGetContestData(ctx context.Context, request []byte, data []byte) string { + var decodedContestData pogo.GetContestDataOutProto + if err := proto.Unmarshal(data, &decodedContestData); err != nil { + log.Errorf("Failed to parse GetContestDataOutProto %s", err) + return fmt.Sprintf("Failed to parse GetContestDataOutProto %s", err) + } + + var decodedContestDataRequest pogo.GetContestDataProto + if request != nil { + if err := proto.Unmarshal(request, &decodedContestDataRequest); err != nil { + log.Errorf("Failed to parse GetContestDataProto %s", err) + return fmt.Sprintf("Failed to parse GetContestDataProto %s", err) + } + } + return decoder.UpdatePokestopWithContestData(ctx, dbDetails, &decodedContestDataRequest, &decodedContestData) +} + +func decodeGetPokemonSizeContestEntry(ctx context.Context, request []byte, data []byte) string { + var decodedPokemonSizeContestEntry pogo.GetPokemonSizeLeaderboardEntryOutProto + if err := proto.Unmarshal(data, &decodedPokemonSizeContestEntry); err != nil { + log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + } + + if decodedPokemonSizeContestEntry.Status != pogo.GetPokemonSizeLeaderboardEntryOutProto_SUCCESS { + return fmt.Sprintf("Ignored GetPokemonSizeLeaderboardEntryOutProto non-success status %s", decodedPokemonSizeContestEntry.Status) + } + + var decodedPokemonSizeContestEntryRequest pogo.GetPokemonSizeLeaderboardEntryProto + if request != nil { + if err := proto.Unmarshal(request, &decodedPokemonSizeContestEntryRequest); err != nil { + log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + } + } + + return decoder.UpdatePokestopWithPokemonSizeContestEntry(ctx, dbDetails, &decodedPokemonSizeContestEntryRequest, &decodedPokemonSizeContestEntry) +} + +func decodeGetStationDetails(ctx context.Context, request []byte, data []byte) string { + var decodedGetStationDetails pogo.GetStationedPokemonDetailsOutProto + if err := proto.Unmarshal(data, &decodedGetStationDetails); err != nil { + log.Errorf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) + return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) + } + + var decodedGetStationDetailsRequest pogo.GetStationedPokemonDetailsProto + if request != nil { + if err := proto.Unmarshal(request, &decodedGetStationDetailsRequest); err != nil { + log.Errorf("Failed to parse GetStationedPokemonDetailsProto %s", err) + return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsProto %s", err) + } + } + + if decodedGetStationDetails.Result == pogo.GetStationedPokemonDetailsOutProto_STATION_NOT_FOUND { + // station without stationed pokemon found, therefore we need to reset the columns + return decoder.ResetStationedPokemonWithStationDetailsNotFound(ctx, dbDetails, &decodedGetStationDetailsRequest) + } else if decodedGetStationDetails.Result != pogo.GetStationedPokemonDetailsOutProto_SUCCESS { + return fmt.Sprintf("Ignored GetStationedPokemonDetailsOutProto non-success status %s", decodedGetStationDetails.Result) + } + + return decoder.UpdateStationWithStationDetails(ctx, dbDetails, &decodedGetStationDetailsRequest, &decodedGetStationDetails) +} + +func decodeTappable(ctx context.Context, request, data []byte, username string, timestampMs int64) string { + var tappable pogo.ProcessTappableOutProto + if err := proto.Unmarshal(data, &tappable); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse ProcessTappableOutProto %s", err) + } + + var tappableRequest pogo.ProcessTappableProto + if request != nil { + if err := proto.Unmarshal(request, &tappableRequest); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse ProcessTappableProto %s", err) + } + } + + if tappable.Status != pogo.ProcessTappableOutProto_SUCCESS { + return fmt.Sprintf("Ignored ProcessTappableOutProto non-success status %s", tappable.Status) + } + var result string + if encounter := tappable.GetEncounter(); encounter != nil { + result = decoder.UpdatePokemonRecordWithTappableEncounter(ctx, dbDetails, &tappableRequest, encounter, username, timestampMs) + } + return result + " " + decoder.UpdateTappable(ctx, dbDetails, &tappableRequest, &tappable, timestampMs) +} + +func decodeGetEventRsvp(ctx context.Context, request []byte, data []byte) string { + var rsvp pogo.GetEventRsvpsOutProto + if err := proto.Unmarshal(data, &rsvp); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse GetEventRsvpsOutProto %s", err) + } + + var rsvpRequest pogo.GetEventRsvpsProto + if request != nil { + if err := proto.Unmarshal(request, &rsvpRequest); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse GetEventRsvpsProto %s", err) + } + } + + if rsvp.Status != pogo.GetEventRsvpsOutProto_SUCCESS { + return fmt.Sprintf("Ignored GetEventRsvpsOutProto non-success status %s", rsvp.Status) + } + + switch op := rsvpRequest.EventDetails.(type) { + case *pogo.GetEventRsvpsProto_Raid: + return decoder.UpdateGymRecordWithRsvpProto(ctx, dbDetails, op.Raid, &rsvp) + case *pogo.GetEventRsvpsProto_GmaxBattle: + return "Unsupported GmaxBattle Rsvp received" + } + + return "Failed to parse GetEventRsvpsProto - unknown event type" +} + +func decodeGetEventRsvpCount(ctx context.Context, data []byte) string { + var rsvp pogo.GetEventRsvpCountOutProto + if err := proto.Unmarshal(data, &rsvp); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse GetEventRsvpCountOutProto %s", err) + } + + if rsvp.Status != pogo.GetEventRsvpCountOutProto_SUCCESS { + return fmt.Sprintf("Ignored GetEventRsvpCountOutProto non-success status %s", rsvp.Status) + } + + var clearLocations []string + for _, rsvpDetails := range rsvp.RsvpDetails { + if rsvpDetails.MaybeCount == 0 && rsvpDetails.GoingCount == 0 { + clearLocations = append(clearLocations, rsvpDetails.LocationId) + decoder.ClearGymRsvp(ctx, dbDetails, rsvpDetails.LocationId) + } + } + + return "Cleared RSVP @ " + strings.Join(clearLocations, ", ") +} diff --git a/decoder/api_gym.go b/decoder/api_gym.go index 97bd60b9..8003cde3 100644 --- a/decoder/api_gym.go +++ b/decoder/api_gym.go @@ -8,8 +8,104 @@ import ( "golbat/db" "golbat/geo" + + "github.com/guregu/null/v6" ) +type ApiGymResult struct { + Id string `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + Name null.String `json:"name"` + Url null.String `json:"url"` + LastModifiedTimestamp null.Int `json:"last_modified_timestamp"` + RaidEndTimestamp null.Int `json:"raid_end_timestamp"` + RaidSpawnTimestamp null.Int `json:"raid_spawn_timestamp"` + RaidBattleTimestamp null.Int `json:"raid_battle_timestamp"` + Updated int64 `json:"updated"` + RaidPokemonId null.Int `json:"raid_pokemon_id"` + GuardingPokemonId null.Int `json:"guarding_pokemon_id"` + GuardingPokemonDisplay null.String `json:"guarding_pokemon_display"` + AvailableSlots null.Int `json:"available_slots"` + TeamId null.Int `json:"team_id"` + RaidLevel null.Int `json:"raid_level"` + Enabled null.Int `json:"enabled"` + ExRaidEligible null.Int `json:"ex_raid_eligible"` + InBattle null.Int `json:"in_battle"` + RaidPokemonMove1 null.Int `json:"raid_pokemon_move_1"` + RaidPokemonMove2 null.Int `json:"raid_pokemon_move_2"` + RaidPokemonForm null.Int `json:"raid_pokemon_form"` + RaidPokemonAlignment null.Int `json:"raid_pokemon_alignment"` + RaidPokemonCp null.Int `json:"raid_pokemon_cp"` + RaidIsExclusive null.Int `json:"raid_is_exclusive"` + CellId null.Int `json:"cell_id"` + Deleted bool `json:"deleted"` + TotalCp null.Int `json:"total_cp"` + FirstSeenTimestamp int64 `json:"first_seen_timestamp"` + RaidPokemonGender null.Int `json:"raid_pokemon_gender"` + SponsorId null.Int `json:"sponsor_id"` + PartnerId null.String `json:"partner_id"` + RaidPokemonCostume null.Int `json:"raid_pokemon_costume"` + RaidPokemonEvolution null.Int `json:"raid_pokemon_evolution"` + ArScanEligible null.Int `json:"ar_scan_eligible"` + PowerUpLevel null.Int `json:"power_up_level"` + PowerUpPoints null.Int `json:"power_up_points"` + PowerUpEndTimestamp null.Int `json:"power_up_end_timestamp"` + Description null.String `json:"description"` + Defenders null.String `json:"defenders"` + Rsvps null.String `json:"rsvps"` +} + +func buildGymResult(gym *Gym) ApiGymResult { + return ApiGymResult{ + Id: gym.Id, + Lat: gym.Lat, + Lon: gym.Lon, + Name: gym.Name, + Url: gym.Url, + LastModifiedTimestamp: gym.LastModifiedTimestamp, + RaidEndTimestamp: gym.RaidEndTimestamp, + RaidSpawnTimestamp: gym.RaidSpawnTimestamp, + RaidBattleTimestamp: gym.RaidBattleTimestamp, + Updated: gym.Updated, + RaidPokemonId: gym.RaidPokemonId, + GuardingPokemonId: gym.GuardingPokemonId, + GuardingPokemonDisplay: gym.GuardingPokemonDisplay, + AvailableSlots: gym.AvailableSlots, + TeamId: gym.TeamId, + RaidLevel: gym.RaidLevel, + Enabled: gym.Enabled, + ExRaidEligible: gym.ExRaidEligible, + InBattle: gym.InBattle, + RaidPokemonMove1: gym.RaidPokemonMove1, + RaidPokemonMove2: gym.RaidPokemonMove2, + RaidPokemonForm: gym.RaidPokemonForm, + RaidPokemonAlignment: gym.RaidPokemonAlignment, + RaidPokemonCp: gym.RaidPokemonCp, + RaidIsExclusive: gym.RaidIsExclusive, + CellId: gym.CellId, + Deleted: gym.Deleted, + TotalCp: gym.TotalCp, + FirstSeenTimestamp: gym.FirstSeenTimestamp, + RaidPokemonGender: gym.RaidPokemonGender, + SponsorId: gym.SponsorId, + PartnerId: gym.PartnerId, + RaidPokemonCostume: gym.RaidPokemonCostume, + RaidPokemonEvolution: gym.RaidPokemonEvolution, + ArScanEligible: gym.ArScanEligible, + PowerUpLevel: gym.PowerUpLevel, + PowerUpPoints: gym.PowerUpPoints, + PowerUpEndTimestamp: gym.PowerUpEndTimestamp, + Description: gym.Description, + Defenders: gym.Defenders, + Rsvps: gym.Rsvps, + } +} + +func BuildGymResult(gym *Gym) ApiGymResult { + return buildGymResult(gym) +} + type ApiGymSearch struct { Limit int `json:"limit"` Filters []ApiGymSearchFilter `json:"filters"` diff --git a/decoder/api_pokemon_common.go b/decoder/api_pokemon_common.go index 9eb74919..c9eeca16 100644 --- a/decoder/api_pokemon_common.go +++ b/decoder/api_pokemon_common.go @@ -10,8 +10,8 @@ import ( pb "golbat/grpc" "github.com/UnownHash/gohbem" + "github.com/guregu/null/v6" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) type ApiPokemonDnfId struct { diff --git a/decoder/api_pokestop.go b/decoder/api_pokestop.go new file mode 100644 index 00000000..618c81d4 --- /dev/null +++ b/decoder/api_pokestop.go @@ -0,0 +1,101 @@ +package decoder + +import "github.com/guregu/null/v6" + +type ApiPokestopResult struct { + Id string `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + Name null.String `json:"name"` + Url null.String `json:"url"` + LureExpireTimestamp null.Int `json:"lure_expire_timestamp"` + LastModifiedTimestamp null.Int `json:"last_modified_timestamp"` + Updated int64 `json:"updated"` + Enabled null.Bool `json:"enabled"` + QuestType null.Int `json:"quest_type"` + QuestTimestamp null.Int `json:"quest_timestamp"` + QuestTarget null.Int `json:"quest_target"` + QuestConditions null.String `json:"quest_conditions"` + QuestRewards null.String `json:"quest_rewards"` + QuestTemplate null.String `json:"quest_template"` + QuestTitle null.String `json:"quest_title"` + QuestExpiry null.Int `json:"quest_expiry"` + CellId null.Int `json:"cell_id"` + Deleted bool `json:"deleted"` + LureId int16 `json:"lure_id"` + FirstSeenTimestamp int16 `json:"first_seen_timestamp"` + SponsorId null.Int `json:"sponsor_id"` + PartnerId null.String `json:"partner_id"` + ArScanEligible null.Int `json:"ar_scan_eligible"` + PowerUpLevel null.Int `json:"power_up_level"` + PowerUpPoints null.Int `json:"power_up_points"` + PowerUpEndTimestamp null.Int `json:"power_up_end_timestamp"` + AlternativeQuestType null.Int `json:"alternative_quest_type"` + AlternativeQuestTimestamp null.Int `json:"alternative_quest_timestamp"` + AlternativeQuestTarget null.Int `json:"alternative_quest_target"` + AlternativeQuestConditions null.String `json:"alternative_quest_conditions"` + AlternativeQuestRewards null.String `json:"alternative_quest_rewards"` + AlternativeQuestTemplate null.String `json:"alternative_quest_template"` + AlternativeQuestTitle null.String `json:"alternative_quest_title"` + AlternativeQuestExpiry null.Int `json:"alternative_quest_expiry"` + Description null.String `json:"description"` + ShowcaseFocus null.String `json:"showcase_focus"` + ShowcasePokemon null.Int `json:"showcase_pokemon_id"` + ShowcasePokemonForm null.Int `json:"showcase_pokemon_form_id"` + ShowcasePokemonType null.Int `json:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `json:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `json:"showcase_expiry"` + ShowcaseRankings null.String `json:"showcase_rankings"` +} + +func buildPokestopResult(stop *Pokestop) ApiPokestopResult { + return ApiPokestopResult{ + Id: stop.Id, + Lat: stop.Lat, + Lon: stop.Lon, + Name: stop.Name, + Url: stop.Url, + LureExpireTimestamp: stop.LureExpireTimestamp, + LastModifiedTimestamp: stop.LastModifiedTimestamp, + Updated: stop.Updated, + Enabled: stop.Enabled, + QuestType: stop.QuestType, + QuestTimestamp: stop.QuestTimestamp, + QuestTarget: stop.QuestTarget, + QuestConditions: stop.QuestConditions, + QuestRewards: stop.QuestRewards, + QuestTemplate: stop.QuestTemplate, + QuestTitle: stop.QuestTitle, + QuestExpiry: stop.QuestExpiry, + CellId: stop.CellId, + Deleted: stop.Deleted, + LureId: stop.LureId, + FirstSeenTimestamp: stop.FirstSeenTimestamp, + SponsorId: stop.SponsorId, + PartnerId: stop.PartnerId, + ArScanEligible: stop.ArScanEligible, + PowerUpLevel: stop.PowerUpLevel, + PowerUpPoints: stop.PowerUpPoints, + PowerUpEndTimestamp: stop.PowerUpEndTimestamp, + AlternativeQuestType: stop.AlternativeQuestType, + AlternativeQuestTimestamp: stop.AlternativeQuestTimestamp, + AlternativeQuestTarget: stop.AlternativeQuestTarget, + AlternativeQuestConditions: stop.AlternativeQuestConditions, + AlternativeQuestRewards: stop.AlternativeQuestRewards, + AlternativeQuestTemplate: stop.AlternativeQuestTemplate, + AlternativeQuestTitle: stop.AlternativeQuestTitle, + AlternativeQuestExpiry: stop.AlternativeQuestExpiry, + Description: stop.Description, + ShowcaseFocus: stop.ShowcaseFocus, + ShowcasePokemon: stop.ShowcasePokemon, + ShowcasePokemonForm: stop.ShowcasePokemonForm, + ShowcasePokemonType: stop.ShowcasePokemonType, + ShowcaseRankingStandard: stop.ShowcaseRankingStandard, + ShowcaseExpiry: stop.ShowcaseExpiry, + ShowcaseRankings: stop.ShowcaseRankings, + } +} + +func BuildPokestopResult(stop *Pokestop) ApiPokestopResult { + return buildPokestopResult(stop) +} diff --git a/decoder/api_tappable.go b/decoder/api_tappable.go new file mode 100644 index 00000000..96874b65 --- /dev/null +++ b/decoder/api_tappable.go @@ -0,0 +1,39 @@ +package decoder + +import "github.com/guregu/null/v6" + +type ApiTappableResult struct { + Id uint64 `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + FortId null.String `json:"fort_id"` + SpawnId null.Int `json:"spawn_id"` + Type string `json:"type"` + Encounter null.Int `json:"pokemon_id"` + ItemId null.Int `json:"item_id"` + Count null.Int `json:"count"` + ExpireTimestamp null.Int `json:"expire_timestamp"` + ExpireTimestampVerified bool `json:"expire_timestamp_verified"` + Updated int64 `json:"updated"` +} + +func buildTappableResult(tappable *Tappable) ApiTappableResult { + return ApiTappableResult{ + Id: tappable.Id, + Lat: tappable.Lat, + Lon: tappable.Lon, + FortId: tappable.FortId, + SpawnId: tappable.SpawnId, + Type: tappable.Type, + Encounter: tappable.Encounter, + ItemId: tappable.ItemId, + Count: tappable.Count, + ExpireTimestamp: tappable.ExpireTimestamp, + ExpireTimestampVerified: tappable.ExpireTimestampVerified, + Updated: tappable.Updated, + } +} + +func BuildTappableResult(tappable *Tappable) ApiTappableResult { + return buildTappableResult(tappable) +} diff --git a/decoder/fort.go b/decoder/fort.go index 979bc2e1..4cc34417 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -99,11 +99,11 @@ func InitWebHookFortFromPokestop(stop *Pokestop) *FortWebhook { func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []string, fortType FortType, change FortChange) { if fortType == GYM { for _, id := range ids { - gym, unlock, err := getGymRecordReadOnly(ctx, dbDetails, id) - if err != nil { - continue - } - if gym == nil { + gym, unlock, err := GetGymRecordReadOnly(ctx, dbDetails, id) + if err != nil || gym == nil { + if unlock != nil { + unlock() + } continue } @@ -116,10 +116,10 @@ func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []strin if fortType == POKESTOP { for _, id := range ids { stop, unlock, err := getPokestopRecordReadOnly(ctx, dbDetails, id) - if err != nil { - continue - } - if stop == nil { + if err != nil || stop == nil { + if unlock != nil { + unlock() + } continue } diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 14222171..f914d73c 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -71,7 +71,7 @@ func LoadAllGyms(details db.DbDetails) { if err != nil { log.Fatalln(err) } - _, unlock, _ := getGymRecordReadOnly(context.Background(), details, place.Id) + _, unlock, _ := GetGymRecordReadOnly(context.Background(), details, place.Id) if unlock != nil { unlock() } diff --git a/decoder/gmo_decode.go b/decoder/gmo_decode.go new file mode 100644 index 00000000..95784faf --- /dev/null +++ b/decoder/gmo_decode.go @@ -0,0 +1,224 @@ +package decoder + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawFortData) { + // Logic is: + // 1. Filter out pokestops that are unchanged (last modified time) + // 2. Fetch current stops from database + // 3. Generate batch of inserts as needed (with on duplicate saveGymRecord) + + //var stopsToModify []string + + for _, fort := range p { + fortId := fort.Data.FortId + if fort.Data.FortType == pogo.FortType_CHECKPOINT && scanParameters.ProcessPokestops { + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fortId) + if err != nil { + log.Errorf("getOrCreatePokestopRecord: %s", err) + continue + } + + pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) + + // If this is a new pokestop, check if it was converted from a gym and copy shared fields + if pokestop.IsNewRecord() { + gym, gymUnlock, _ := GetGymRecordReadOnly(ctx, db, fortId) + if gym != nil { + pokestop.copySharedFieldsFrom(gym) + gymUnlock() + } + } + + savePokestopRecord(ctx, db, pokestop) + unlock() + + incidents := fort.Data.PokestopDisplays + if incidents == nil && fort.Data.PokestopDisplay != nil { + incidents = []*pogo.PokestopIncidentDisplayProto{fort.Data.PokestopDisplay} + } + + if incidents != nil { + for _, incidentProto := range incidents { + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, incidentProto.IncidentId, fortId) + if err != nil { + log.Errorf("getOrCreateIncidentRecord: %s", err) + continue + } + incident.updateFromPokestopIncidentDisplay(incidentProto) + saveIncidentRecord(ctx, db, incident) + unlock() + } + } + } + + if fort.Data.FortType == pogo.FortType_GYM && scanParameters.ProcessGyms { + gym, gymUnlock, err := getOrCreateGymRecord(ctx, db, fortId) + if err != nil { + log.Errorf("getOrCreateGymRecord: %s", err) + continue + } + + gym.updateGymFromFort(fort.Data, fort.Cell) + + // If this is a new gym, check if it was converted from a pokestop and copy shared fields + if gym.IsNewRecord() { + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, fortId) + if pokestop != nil { + gym.copySharedFieldsFrom(pokestop) + unlock() + } + } + + saveGymRecord(ctx, db, gym) + gymUnlock() + } + } +} + +func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawStationData) { + for _, stationProto := range p { + stationId := stationProto.Data.Id + station, unlock, err := getOrCreateStationRecord(ctx, db, stationId) + if err != nil { + log.Errorf("getOrCreateStationRecord: %s", err) + continue + } + station.updateFromStationProto(stationProto.Data, stationProto.Cell) + saveStationRecord(ctx, db, station) + unlock() + } +} + +func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, wildPokemonList []RawWildPokemonData, nearbyPokemonList []RawNearbyPokemonData, mapPokemonList []RawMapPokemonData, weather []*pogo.ClientWeatherProto, username string) { + weatherLookup := make(map[int64]pogo.GameplayWeatherProto_WeatherCondition) + for _, weatherProto := range weather { + weatherLookup[weatherProto.S2CellId] = weatherProto.GameplayWeather.GameplayCondition + } + + for _, wild := range wildPokemonList { + encounterId := wild.Data.EncounterId + + // spawnpointUpdateFromWild doesn't need Pokemon lock + spawnpointUpdateFromWild(ctx, db, wild.Data, wild.Timestamp) + + if scanParameters.ProcessWild { + // Use read-only getter - we're only checking if update is needed, then queuing + pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) + if err != nil { + log.Errorf("getPokemonRecordReadOnly: %s", err) + continue + } + + updateTime := wild.Timestamp / 1000 + shouldQueue := pokemon == nil || pokemon.wildSignificantUpdate(wild.Data, updateTime) + + if unlock != nil { + unlock() + } + + if shouldQueue { + // The sweeper will process it after timeout if no encounter arrives + pending := &PendingPokemon{ + EncounterId: encounterId, + WildPokemon: wild.Data, + CellId: int64(wild.Cell), + TimestampMs: wild.Timestamp, + UpdateTime: updateTime, + WeatherLookup: weatherLookup, + Username: username, + } + pokemonPendingQueue.AddPending(pending) + } + } + } + + if scanParameters.ProcessNearby { + for _, nearby := range nearbyPokemonList { + encounterId := nearby.Data.EncounterId + + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Printf("getOrCreatePokemonRecord: %s", err) + continue + } + + updateTime := nearby.Timestamp / 1000 + if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { + pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) + } + + unlock() + } + } + + for _, mapPokemon := range mapPokemonList { + encounterId := mapPokemon.Data.EncounterId + + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Printf("getOrCreatePokemonRecord: %s", err) + continue + } + + pokemon.updateFromMap(ctx, db, mapPokemon.Data, int64(mapPokemon.Cell), weatherLookup, mapPokemon.Timestamp, username) + storedDiskEncounter := diskEncounterCache.Get(encounterId) + if storedDiskEncounter != nil { + diskEncounter := storedDiskEncounter.Value() + diskEncounterCache.Delete(encounterId) + pokemon.updatePokemonFromDiskEncounterProto(ctx, db, diskEncounter, username) + //log.Infof("Processed stored disk encounter") + } + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, mapPokemon.Timestamp/1000) + + unlock() + } +} + +func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.ClientWeatherProto, timestampMs int64, account string) (updates []WeatherUpdate) { + hourKey := timestampMs / time.Hour.Milliseconds() + for _, weatherProto := range p { + weather, unlock, err := getOrCreateWeatherRecord(ctx, db, weatherProto.S2CellId) + if err != nil { + log.Printf("getOrCreateWeatherRecord: %s", err) + continue + } + + if weather.newRecord || timestampMs >= weather.UpdatedMs { + state := getWeatherConsensusState(weatherProto.S2CellId, hourKey) + if state != nil { + publish, publishProto := state.applyObservation(hourKey, account, weatherProto) + if publish { + if publishProto == nil { + publishProto = weatherProto + } + weather.UpdatedMs = timestampMs + weather.updateWeatherFromClientWeatherProto(publishProto) + saveWeatherRecord(ctx, db, weather) + if weather.oldValues.GameplayCondition != weather.GameplayCondition { + updates = append(updates, WeatherUpdate{ + S2CellId: publishProto.S2CellId, + NewWeather: int32(publishProto.GetGameplayWeather().GetGameplayCondition()), + }) + } + } + } + } + + unlock() + } + return updates +} + +func UpdateClientMapS2CellBatch(ctx context.Context, db db.DbDetails, cellIds []uint64) { + saveS2CellRecords(ctx, db, cellIds) +} diff --git a/decoder/gym.go b/decoder/gym.go index 5fc58626..c0004247 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -1,28 +1,9 @@ package decoder import ( - "cmp" - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "slices" - "strings" "sync" - "time" - "golbat/geo" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" - - "golbat/config" - "golbat/db" - "golbat/pogo" - "golbat/util" - "golbat/webhooks" + "github.com/guregu/null/v6" ) // Gym struct. @@ -580,814 +561,3 @@ func (gym *Gym) SetRsvps(v null.String) { } } } - -func loadGymFromDatabase(ctx context.Context, db db.DbDetails, fortId string, gym *Gym) error { - err := db.GeneralDb.GetContext(ctx, gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) - statsCollector.IncDbQuery("select gym", err) - return err -} - -// PeekGymRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func PeekGymRecord(fortId string) (*Gym, func(), error) { - if item := gymCache.Get(fortId); item != nil { - gym := item.Value() - gym.Lock() - return gym, func() { gym.Unlock() }, nil - } - return nil, nil, nil -} - -// getGymRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { - // Check cache first - if item := gymCache.Get(fortId); item != nil { - gym := item.Value() - gym.Lock() - return gym, func() { gym.Unlock() }, nil - } - - dbGym := Gym{} - err := loadGymFromDatabase(ctx, db, fortId, &dbGym) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbGym.ClearDirty() - - // Atomically cache the loaded Gym - if another goroutine raced us, - // we'll get their Gym and use that instead (ensuring same mutex) - existingGym, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { - // Only called if key doesn't exist - our Pokestop wins - if config.Config.TestFortInMemory { - fortRtreeUpdateGymOnGet(&dbGym) - } - return &dbGym - }) - - gym := existingGym.Value() - gym.Lock() - return gym, func() { gym.Unlock() }, nil -} - -// getGymRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Use when modifying the Gym. -// Caller MUST call returned unlock function if non-nil. -func getGymRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { - gym, unlock, err := getGymRecordReadOnly(ctx, db, fortId) - if err != nil || gym == nil { - return nil, nil, err - } - gym.snapshotOldValues() - return gym, unlock, nil -} - -// getOrCreateGymRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { - // Create new Gym atomically - function only called if key doesn't exist - gymItem, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { - return &Gym{Id: fortId, newRecord: true} - }) - - gym := gymItem.Value() - gym.Lock() - - if gym.newRecord { - // We should attempt to load from database - err := loadGymFromDatabase(ctx, db, fortId, gym) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - gym.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - gym.newRecord = false - gym.ClearDirty() - if config.Config.TestFortInMemory { - fortRtreeUpdateGymOnGet(gym) - } - } - } - - gym.snapshotOldValues() - return gym, func() { gym.Unlock() }, nil -} - -// GetGymRecord returns a copy of the Gym for external/API use. -// For internal use, prefer getGymRecordReadOnly or getGymRecordForUpdate. -func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, error) { - gym, unlock, err := getGymRecordReadOnly(ctx, db, fortId) - if err != nil { - return nil, err - } - if gym == nil { - return nil, nil - } - // Make a copy for safe external use - gymCopy := *gym - unlock() - return &gymCopy, nil -} - -func escapeLike(s string) string { - s = strings.ReplaceAll(s, `\`, `\\`) - s = strings.ReplaceAll(s, `%`, `\%`) - s = strings.ReplaceAll(s, `_`, `\_`) - return s -} - -func calculatePowerUpPoints(fortData *pogo.PokemonFortProto) (null.Int, null.Int) { - now := time.Now().Unix() - powerUpLevelExpirationMs := int64(fortData.PowerUpLevelExpirationMs) / 1000 - powerUpPoints := int64(fortData.PowerUpProgressPoints) - powerUpLevel := null.IntFrom(0) - powerUpEndTimestamp := null.NewInt(0, false) - if powerUpPoints < 50 { - powerUpLevel = null.IntFrom(0) - } else if powerUpPoints < 100 && powerUpLevelExpirationMs > now { - powerUpLevel = null.IntFrom(1) - powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) - } else if powerUpPoints < 150 && powerUpLevelExpirationMs > now { - powerUpLevel = null.IntFrom(2) - powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) - } else if powerUpLevelExpirationMs > now { - powerUpLevel = null.IntFrom(3) - powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) - } else { - powerUpLevel = null.IntFrom(0) - } - - return powerUpLevel, powerUpEndTimestamp -} - -func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64) *Gym { - type pokemonDisplay struct { - Form int `json:"form,omitempty"` - Costume int `json:"costume,omitempty"` - Gender int `json:"gender"` - Shiny bool `json:"shiny,omitempty"` - TempEvolution int `json:"temp_evolution,omitempty"` - TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` - Alignment int `json:"alignment,omitempty"` - Badge int `json:"badge,omitempty"` - Background *int64 `json:"background,omitempty"` - } - gym.SetId(fortData.FortId) - gym.SetLat(fortData.Latitude) - gym.SetLon(fortData.Longitude) - gym.SetEnabled(null.IntFrom(util.BoolToInt[int64](fortData.Enabled))) - gym.SetGuardingPokemonId(null.IntFrom(int64(fortData.GuardPokemonId))) - if fortData.GuardPokemonDisplay == nil { - gym.SetGuardingPokemonDisplay(null.NewString("", false)) - } else { - display, _ := json.Marshal(pokemonDisplay{ - Form: int(fortData.GuardPokemonDisplay.Form), - Costume: int(fortData.GuardPokemonDisplay.Costume), - Gender: int(fortData.GuardPokemonDisplay.Gender), - Shiny: fortData.GuardPokemonDisplay.Shiny, - TempEvolution: int(fortData.GuardPokemonDisplay.CurrentTempEvolution), - TempEvolutionFinishMs: fortData.GuardPokemonDisplay.TemporaryEvolutionFinishMs, - Alignment: int(fortData.GuardPokemonDisplay.Alignment), - Badge: int(fortData.GuardPokemonDisplay.PokemonBadge), - Background: util.ExtractBackgroundFromDisplay(fortData.GuardPokemonDisplay), - }) - gym.SetGuardingPokemonDisplay(null.StringFrom(string(display))) - } - gym.SetTeamId(null.IntFrom(int64(fortData.Team))) - if fortData.GymDisplay != nil { - gym.SetAvailableSlots(null.IntFrom(int64(fortData.GymDisplay.SlotsAvailable))) - } else { - gym.SetAvailableSlots(null.IntFrom(6)) // this may be an incorrect assumption - } - gym.SetLastModifiedTimestamp(null.IntFrom(fortData.LastModifiedMs / 1000)) - gym.SetExRaidEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsExRaidEligible))) - - if fortData.ImageUrl != "" { - gym.SetUrl(null.StringFrom(fortData.ImageUrl)) - } - gym.SetInBattle(null.IntFrom(util.BoolToInt[int64](fortData.IsInBattle))) - gym.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) - gym.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) - - powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) - gym.SetPowerUpLevel(powerUpLevel) - gym.SetPowerUpEndTimestamp(powerUpEndTimestamp) - - if fortData.PartnerId == "" { - gym.SetPartnerId(null.NewString("", false)) - } else { - gym.SetPartnerId(null.StringFrom(fortData.PartnerId)) - } - - if fortData.ImageUrl != "" { - gym.SetUrl(null.StringFrom(fortData.ImageUrl)) - } - if fortData.Team == 0 { // check!! - gym.SetTotalCp(null.IntFrom(0)) - } else { - if fortData.GymDisplay != nil { - totalCp := int64(fortData.GymDisplay.TotalGymCp) - if gym.TotalCp.Int64-totalCp > 100 || totalCp-gym.TotalCp.Int64 > 100 { - gym.SetTotalCp(null.IntFrom(totalCp)) - } - } else { - gym.SetTotalCp(null.IntFrom(0)) - } - } - - if fortData.RaidInfo != nil { - gym.SetRaidEndTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000)) - gym.SetRaidSpawnTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000)) - raidBattleTimestamp := int64(fortData.RaidInfo.RaidBattleMs) / 1000 - - if gym.RaidBattleTimestamp.ValueOrZero() != raidBattleTimestamp { - // We are reporting a new raid, clear rsvp data - gym.SetRsvps(null.NewString("", false)) - } - gym.SetRaidBattleTimestamp(null.IntFrom(raidBattleTimestamp)) - - gym.SetRaidLevel(null.IntFrom(int64(fortData.RaidInfo.RaidLevel))) - if fortData.RaidInfo.RaidPokemon != nil { - gym.SetRaidPokemonId(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonId))) - gym.SetRaidPokemonMove1(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move1))) - gym.SetRaidPokemonMove2(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move2))) - gym.SetRaidPokemonForm(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Form))) - gym.SetRaidPokemonAlignment(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Alignment))) - gym.SetRaidPokemonCp(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Cp))) - gym.SetRaidPokemonGender(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Gender))) - gym.SetRaidPokemonCostume(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Costume))) - gym.SetRaidPokemonEvolution(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.CurrentTempEvolution))) - } else { - gym.SetRaidPokemonId(null.IntFrom(0)) - gym.SetRaidPokemonMove1(null.IntFrom(0)) - gym.SetRaidPokemonMove2(null.IntFrom(0)) - gym.SetRaidPokemonForm(null.IntFrom(0)) - gym.SetRaidPokemonAlignment(null.IntFrom(0)) - gym.SetRaidPokemonCp(null.IntFrom(0)) - gym.SetRaidPokemonGender(null.IntFrom(0)) - gym.SetRaidPokemonCostume(null.IntFrom(0)) - gym.SetRaidPokemonEvolution(null.IntFrom(0)) - } - - gym.SetRaidIsExclusive(null.IntFrom(0)) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive)) - } - - gym.SetCellId(null.IntFrom(int64(cellId))) - - if gym.Deleted { - gym.SetDeleted(false) - log.Warnf("Cleared Gym with id '%s' is found again in GMO, therefore un-deleted", gym.Id) - // Restore in fort tracker if enabled - if fortTracker != nil { - fortTracker.RestoreFort(gym.Id, cellId, true, time.Now().Unix()) - } - } - - return gym -} - -func (gym *Gym) updateGymFromFortProto(fortData *pogo.FortDetailsOutProto) *Gym { - gym.SetId(fortData.Id) - gym.SetLat(fortData.Latitude) - gym.SetLon(fortData.Longitude) - if len(fortData.ImageUrl) > 0 { - gym.SetUrl(null.StringFrom(fortData.ImageUrl[0])) - } - gym.SetName(null.StringFrom(fortData.Name)) - - return gym -} - -func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) *Gym { - gym.SetId(gymData.GymStatusAndDefenders.PokemonFortProto.FortId) - gym.SetLat(gymData.GymStatusAndDefenders.PokemonFortProto.Latitude) - gym.SetLon(gymData.GymStatusAndDefenders.PokemonFortProto.Longitude) - - // This will have gym defenders in it... - if len(gymData.Url) > 0 { - gym.SetUrl(null.StringFrom(gymData.Url)) - } - gym.SetName(null.StringFrom(gymData.Name)) - - if gymData.Description == "" { - gym.SetDescription(null.NewString("", false)) - } else { - gym.SetDescription(null.StringFrom(gymData.Description)) - } - - type pokemonGymDefender struct { - PokemonId int `json:"pokemon_id,omitempty"` - Form int `json:"form,omitempty"` - Costume int `json:"costume,omitempty"` - Gender int `json:"gender"` - Shiny bool `json:"shiny,omitempty"` - TempEvolution int `json:"temp_evolution,omitempty"` - TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` - Alignment int `json:"alignment,omitempty"` - Badge int `json:"badge,omitempty"` - Background *int64 `json:"background,omitempty"` - DeployedMs int64 `json:"deployed_ms,omitempty"` - DeployedTime int64 `json:"deployed_time,omitempty"` - BattlesWon int32 `json:"battles_won"` - BattlesLost int32 `json:"battles_lost"` - TimesFed int32 `json:"times_fed"` - MotivationNow util.RoundedFloat4 `json:"motivation_now"` - CpNow int32 `json:"cp_now"` - CpWhenDeployed int32 `json:"cp_when_deployed"` - } - - var defenders []pokemonGymDefender - now := time.Now() - for _, protoDefender := range gymData.GymStatusAndDefenders.GymDefender { - motivatedPokemon := protoDefender.MotivatedPokemon - pokemonDisplay := motivatedPokemon.Pokemon.PokemonDisplay - deploymentTotals := protoDefender.DeploymentTotals - defender := pokemonGymDefender{ - DeployedMs: protoDefender.DeploymentTotals.DeploymentDurationMs, - DeployedTime: now. - Add(-1 * time.Millisecond * time.Duration(deploymentTotals.DeploymentDurationMs)). - Unix(), // This will only be approximately correct - BattlesLost: deploymentTotals.BattlesLost, - BattlesWon: deploymentTotals.BattlesWon, - TimesFed: deploymentTotals.TimesFed, - PokemonId: int(protoDefender.MotivatedPokemon.Pokemon.PokemonId), - Form: int(pokemonDisplay.Form), - Costume: int(pokemonDisplay.Costume), - Gender: int(pokemonDisplay.Gender), - TempEvolution: int(pokemonDisplay.CurrentTempEvolution), - TempEvolutionFinishMs: pokemonDisplay.TemporaryEvolutionFinishMs, - Alignment: int(pokemonDisplay.Alignment), - Badge: int(pokemonDisplay.PokemonBadge), - Background: util.ExtractBackgroundFromDisplay(pokemonDisplay), - Shiny: pokemonDisplay.Shiny, - MotivationNow: util.RoundedFloat4(motivatedPokemon.MotivationNow), - CpNow: motivatedPokemon.CpNow, - CpWhenDeployed: motivatedPokemon.CpWhenDeployed, - } - defenders = append(defenders, defender) - } - bDefenders, _ := json.Marshal(defenders) - gym.SetDefenders(null.StringFrom(string(bDefenders))) - // log.Debugf("Gym %s defenders %s ", gym.Id, string(bDefenders)) - - return gym -} - -func (gym *Gym) updateGymFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto, skipName bool) *Gym { - gym.SetId(fortData.Id) - gym.SetLat(fortData.Latitude) - gym.SetLon(fortData.Longitude) - - if len(fortData.Image) > 0 { - gym.SetUrl(null.StringFrom(fortData.Image[0].Url)) - } - if !skipName { - gym.SetName(null.StringFrom(fortData.Name)) - } - - if gym.Deleted { - log.Debugf("Cleared Gym with id '%s' is found again in GMF, therefore kept deleted", gym.Id) - } - - return gym -} - -func (gym *Gym) updateGymFromRsvpProto(fortData *pogo.GetEventRsvpsOutProto) *Gym { - type rsvpTimeslot struct { - Timeslot int64 `json:"timeslot"` - GoingCount int32 `json:"going_count"` - MaybeCount int32 `json:"maybe_count"` - } - - timeslots := make([]rsvpTimeslot, 0) - - for _, timeslot := range fortData.RsvpTimeslots { - if timeslot.GoingCount > 0 || timeslot.MaybeCount > 0 { - timeslots = append(timeslots, rsvpTimeslot{ - Timeslot: timeslot.TimeSlot, - GoingCount: timeslot.GoingCount, - MaybeCount: timeslot.MaybeCount, - }) - } - } - - if len(timeslots) == 0 { - gym.SetRsvps(null.NewString("", false)) - } else { - slices.SortFunc(timeslots, func(a, b rsvpTimeslot) int { - return cmp.Compare(a.Timeslot, b.Timeslot) - }) - - bRsvps, _ := json.Marshal(timeslots) - gym.SetRsvps(null.StringFrom(string(bRsvps))) - } - - return gym -} - -// hasChangesGym compares two Gym structs -// Float tolerance: Lat, Lon -func hasChangesGym(old *Gym, new *Gym) bool { - return old.Id != new.Id || - old.Name != new.Name || - old.Url != new.Url || - old.LastModifiedTimestamp != new.LastModifiedTimestamp || - old.RaidEndTimestamp != new.RaidEndTimestamp || - old.RaidSpawnTimestamp != new.RaidSpawnTimestamp || - old.RaidBattleTimestamp != new.RaidBattleTimestamp || - old.Updated != new.Updated || - old.RaidPokemonId != new.RaidPokemonId || - old.GuardingPokemonId != new.GuardingPokemonId || - old.AvailableSlots != new.AvailableSlots || - old.TeamId != new.TeamId || - old.RaidLevel != new.RaidLevel || - old.Enabled != new.Enabled || - old.ExRaidEligible != new.ExRaidEligible || - // old.InBattle != new.InBattle || - old.RaidPokemonMove1 != new.RaidPokemonMove1 || - old.RaidPokemonMove2 != new.RaidPokemonMove2 || - old.RaidPokemonForm != new.RaidPokemonForm || - old.RaidPokemonAlignment != new.RaidPokemonAlignment || - old.RaidPokemonCp != new.RaidPokemonCp || - old.RaidIsExclusive != new.RaidIsExclusive || - old.CellId != new.CellId || - old.Deleted != new.Deleted || - old.TotalCp != new.TotalCp || - old.FirstSeenTimestamp != new.FirstSeenTimestamp || - old.RaidPokemonGender != new.RaidPokemonGender || - old.SponsorId != new.SponsorId || - old.PartnerId != new.PartnerId || - old.RaidPokemonCostume != new.RaidPokemonCostume || - old.RaidPokemonEvolution != new.RaidPokemonEvolution || - old.ArScanEligible != new.ArScanEligible || - old.PowerUpLevel != new.PowerUpLevel || - old.PowerUpPoints != new.PowerUpPoints || - old.PowerUpEndTimestamp != new.PowerUpEndTimestamp || - old.Description != new.Description || - old.Rsvps != new.Rsvps || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) - -} - -// hasChangesInternalGym compares two Gym structs for changes that will be stored in memory -// Float tolerance: Lat, Lon -func hasInternalChangesGym(old *Gym, new *Gym) bool { - return old.InBattle != new.InBattle || - old.Defenders != new.Defenders -} - -type GymDetailsWebhook struct { - Id string `json:"id"` - Name string `json:"name"` - Url string `json:"url"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Team int64 `json:"team"` - GuardPokemonId int64 `json:"guard_pokemon_id"` - SlotsAvailable int64 `json:"slots_available"` - ExRaidEligible int64 `json:"ex_raid_eligible"` - InBattle bool `json:"in_battle"` - SponsorId int64 `json:"sponsor_id"` - PartnerId int64 `json:"partner_id"` - PowerUpPoints int64 `json:"power_up_points"` - PowerUpLevel int64 `json:"power_up_level"` - PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` - ArScanEligible int64 `json:"ar_scan_eligible"` - Defenders any `json:"defenders"` -} - -type RaidWebhook struct { - GymId string `json:"gym_id"` - GymName string `json:"gym_name"` - GymUrl string `json:"gym_url"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - TeamId int64 `json:"team_id"` - Spawn int64 `json:"spawn"` - Start int64 `json:"start"` - End int64 `json:"end"` - Level int64 `json:"level"` - PokemonId int64 `json:"pokemon_id"` - Cp int64 `json:"cp"` - Gender int64 `json:"gender"` - Form int64 `json:"form"` - Alignment int64 `json:"alignment"` - Costume int64 `json:"costume"` - Evolution int64 `json:"evolution"` - Move1 int64 `json:"move_1"` - Move2 int64 `json:"move_2"` - ExRaidEligible int64 `json:"ex_raid_eligible"` - IsExclusive int64 `json:"is_exclusive"` - SponsorId int64 `json:"sponsor_id"` - PartnerId string `json:"partner_id"` - PowerUpPoints int64 `json:"power_up_points"` - PowerUpLevel int64 `json:"power_up_level"` - PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` - ArScanEligible int64 `json:"ar_scan_eligible"` - Rsvps json.RawMessage `json:"rsvps"` -} - -func createGymFortWebhooks(gym *Gym) { - fort := InitWebHookFortFromGym(gym) - if gym.newRecord { - CreateFortWebHooks(nil, fort, NEW) - } else { - // Build old fort from saved old values - oldFort := &FortWebhook{ - Type: GYM.String(), - Id: gym.Id, - Name: gym.oldValues.Name.Ptr(), - ImageUrl: gym.oldValues.Url.Ptr(), - Description: gym.oldValues.Description.Ptr(), - Location: Location{Latitude: gym.oldValues.Lat, Longitude: gym.oldValues.Lon}, - } - CreateFortWebHooks(oldFort, fort, EDIT) - } -} - -func createGymWebhooks(gym *Gym, areas []geo.AreaName) { - if gym.newRecord || - (gym.oldValues.AvailableSlots != gym.AvailableSlots || gym.oldValues.TeamId != gym.TeamId || gym.oldValues.InBattle != gym.InBattle) { - gymDetails := GymDetailsWebhook{ - Id: gym.Id, - Name: gym.Name.ValueOrZero(), - Url: gym.Url.ValueOrZero(), - Latitude: gym.Lat, - Longitude: gym.Lon, - Team: gym.TeamId.ValueOrZero(), - GuardPokemonId: gym.GuardingPokemonId.ValueOrZero(), - SlotsAvailable: func() int64 { - if gym.AvailableSlots.Valid { - return gym.AvailableSlots.Int64 - } else { - return 6 - } - }(), - ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), - InBattle: func() bool { return gym.InBattle.ValueOrZero() != 0 }(), - Defenders: func() any { - if gym.Defenders.Valid { - return json.RawMessage(gym.Defenders.ValueOrZero()) - } else { - return nil - } - }(), - } - - webhooksSender.AddMessage(webhooks.GymDetails, gymDetails, areas) - } - - if gym.RaidSpawnTimestamp.ValueOrZero() > 0 && - (gym.newRecord || gym.oldValues.RaidLevel != gym.RaidLevel || - gym.oldValues.RaidPokemonId != gym.RaidPokemonId || - gym.oldValues.RaidSpawnTimestamp != gym.RaidSpawnTimestamp || gym.oldValues.Rsvps != gym.Rsvps) { - raidBattleTime := gym.RaidBattleTimestamp.ValueOrZero() - raidEndTime := gym.RaidEndTimestamp.ValueOrZero() - now := time.Now().Unix() - - if (raidBattleTime > now && gym.RaidLevel.ValueOrZero() > 0) || - (raidEndTime > now && gym.RaidPokemonId.ValueOrZero() > 0) { - gymName := "Unknown" - if gym.Name.Valid { - gymName = gym.Name.String - } - - var rsvps json.RawMessage - if gym.Rsvps.Valid { - rsvps = json.RawMessage(gym.Rsvps.ValueOrZero()) - } - - raidHook := RaidWebhook{ - GymId: gym.Id, - GymName: gymName, - GymUrl: gym.Url.ValueOrZero(), - Latitude: gym.Lat, - Longitude: gym.Lon, - TeamId: gym.TeamId.ValueOrZero(), - Spawn: gym.RaidSpawnTimestamp.ValueOrZero(), - Start: gym.RaidBattleTimestamp.ValueOrZero(), - End: gym.RaidEndTimestamp.ValueOrZero(), - Level: gym.RaidLevel.ValueOrZero(), - PokemonId: gym.RaidPokemonId.ValueOrZero(), - Cp: gym.RaidPokemonCp.ValueOrZero(), - Gender: gym.RaidPokemonGender.ValueOrZero(), - Form: gym.RaidPokemonForm.ValueOrZero(), - Alignment: gym.RaidPokemonAlignment.ValueOrZero(), - Costume: gym.RaidPokemonCostume.ValueOrZero(), - Evolution: gym.RaidPokemonEvolution.ValueOrZero(), - Move1: gym.RaidPokemonMove1.ValueOrZero(), - Move2: gym.RaidPokemonMove2.ValueOrZero(), - ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), - IsExclusive: gym.RaidIsExclusive.ValueOrZero(), - SponsorId: gym.SponsorId.ValueOrZero(), - PartnerId: gym.PartnerId.ValueOrZero(), - PowerUpPoints: gym.PowerUpPoints.ValueOrZero(), - PowerUpLevel: gym.PowerUpLevel.ValueOrZero(), - PowerUpEndTimestamp: gym.PowerUpEndTimestamp.ValueOrZero(), - ArScanEligible: gym.ArScanEligible.ValueOrZero(), - Rsvps: rsvps, - } - - webhooksSender.AddMessage(webhooks.Raid, raidHook, areas) - statsCollector.UpdateRaidCount(areas, gym.RaidLevel.ValueOrZero()) - } - } -} - -func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { - now := time.Now().Unix() - if !gym.IsNewRecord() && !gym.IsDirty() && !gym.IsInternalDirty() { - // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. - if gym.Updated > now-GetUpdateThreshold(900) { - // if a gym is unchanged and was seen recently, skip saving - return - } - } - gym.Updated = now - - if gym.IsDirty() { - if gym.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ - "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) - - statsCollector.IncDbQuery("insert gym", err) - if err != nil { - log.Errorf("insert gym: %s", err) - return - } - - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ - "lat = :lat, "+ - "lon = :lon, "+ - "name = :name, "+ - "url = :url, "+ - "last_modified_timestamp = :last_modified_timestamp, "+ - "raid_end_timestamp = :raid_end_timestamp, "+ - "raid_spawn_timestamp = :raid_spawn_timestamp, "+ - "raid_battle_timestamp = :raid_battle_timestamp, "+ - "updated = :updated, "+ - "raid_pokemon_id = :raid_pokemon_id, "+ - "guarding_pokemon_id = :guarding_pokemon_id, "+ - "guarding_pokemon_display = :guarding_pokemon_display, "+ - "available_slots = :available_slots, "+ - "team_id = :team_id, "+ - "raid_level = :raid_level, "+ - "enabled = :enabled, "+ - "ex_raid_eligible = :ex_raid_eligible, "+ - "in_battle = :in_battle, "+ - "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ - "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ - "raid_pokemon_form = :raid_pokemon_form, "+ - "raid_pokemon_alignment = :raid_pokemon_alignment, "+ - "raid_pokemon_cp = :raid_pokemon_cp, "+ - "raid_is_exclusive = :raid_is_exclusive, "+ - "cell_id = :cell_id, "+ - "deleted = :deleted, "+ - "total_cp = :total_cp, "+ - "raid_pokemon_gender = :raid_pokemon_gender, "+ - "sponsor_id = :sponsor_id, "+ - "partner_id = :partner_id, "+ - "raid_pokemon_costume = :raid_pokemon_costume, "+ - "raid_pokemon_evolution = :raid_pokemon_evolution, "+ - "ar_scan_eligible = :ar_scan_eligible, "+ - "power_up_level = :power_up_level, "+ - "power_up_points = :power_up_points, "+ - "power_up_end_timestamp = :power_up_end_timestamp,"+ - "description = :description,"+ - "defenders = :defenders,"+ - "rsvps = :rsvps "+ - "WHERE id = :id", gym, - ) - statsCollector.IncDbQuery("update gym", err) - if err != nil { - log.Errorf("Update gym %s", err) - } - _, _ = res, err - } - } - - //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) - areas := MatchStatsGeofence(gym.Lat, gym.Lon) - createGymWebhooks(gym, areas) - createGymFortWebhooks(gym) - updateRaidStats(gym, areas) - if dbDebugEnabled { - gym.changedFields = gym.changedFields[:0] - } - if gym.IsNewRecord() { - gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) - gym.newRecord = false - } - gym.ClearDirty() -} - -func updateGymGetMapFortCache(gym *Gym, skipName bool) { - storedGetMapFort := getMapFortsCache.Get(gym.Id) - if storedGetMapFort != nil { - getMapFort := storedGetMapFort.Value() - getMapFortsCache.Delete(gym.Id) - gym.updateGymFromGetMapFortsOutProto(getMapFort, skipName) - log.Debugf("Updated Gym using stored getMapFort: %s", gym.Id) - } -} - -func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { - gym, unlock, err := getOrCreateGymRecord(ctx, db, fort.Id) - if err != nil { - return err.Error() - } - defer unlock() - - gym.updateGymFromFortProto(fort) - - updateGymGetMapFortCache(gym, true) - saveGymRecord(ctx, db, gym) - - return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} - -func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymInfo *pogo.GymGetInfoOutProto) string { - gym, unlock, err := getOrCreateGymRecord(ctx, db, gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) - if err != nil { - return err.Error() - } - defer unlock() - - gym.updateGymFromGymInfoOutProto(gymInfo) - - updateGymGetMapFortCache(gym, true) - saveGymRecord(ctx, db, gym) - return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} - -func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { - gym, unlock, err := getGymRecordForUpdate(ctx, db, mapFort.Id) - if err != nil { - return false, err.Error() - } - - // we missed it in Pokestop & Gym. Lets save it to cache - if gym == nil { - return false, "" - } - defer unlock() - - gym.updateGymFromGetMapFortsOutProto(mapFort, false) - saveGymRecord(ctx, db, gym) - return true, fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} - -func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pogo.RaidDetails, resp *pogo.GetEventRsvpsOutProto) string { - gym, unlock, err := getGymRecordForUpdate(ctx, db, req.FortId) - if err != nil { - return err.Error() - } - - if gym == nil { - // Do not add RSVP details to unknown gyms - return fmt.Sprintf("%s Gym not present", req.FortId) - } - defer unlock() - - gym.updateGymFromRsvpProto(resp) - - saveGymRecord(ctx, db, gym) - - return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} - -func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { - gym, unlock, err := getGymRecordForUpdate(ctx, db, fortId) - if err != nil { - return err.Error() - } - - if gym == nil { - // Do not add RSVP details to unknown gyms - return fmt.Sprintf("%s Gym not present", fortId) - } - defer unlock() - - if gym.Rsvps.Valid { - gym.SetRsvps(null.NewString("", false)) - - saveGymRecord(ctx, db, gym) - } - - return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} diff --git a/decoder/gym_decode.go b/decoder/gym_decode.go new file mode 100644 index 00000000..3bec72aa --- /dev/null +++ b/decoder/gym_decode.go @@ -0,0 +1,311 @@ +package decoder + +import ( + "cmp" + "encoding/json" + "slices" + "strings" + "time" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" + "golbat/util" +) + +func escapeLike(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `%`, `\%`) + s = strings.ReplaceAll(s, `_`, `\_`) + return s +} + +func calculatePowerUpPoints(fortData *pogo.PokemonFortProto) (null.Int, null.Int) { + now := time.Now().Unix() + powerUpLevelExpirationMs := int64(fortData.PowerUpLevelExpirationMs) / 1000 + powerUpPoints := int64(fortData.PowerUpProgressPoints) + powerUpLevel := null.IntFrom(0) + powerUpEndTimestamp := null.NewInt(0, false) + if powerUpPoints < 50 { + powerUpLevel = null.IntFrom(0) + } else if powerUpPoints < 100 && powerUpLevelExpirationMs > now { + powerUpLevel = null.IntFrom(1) + powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) + } else if powerUpPoints < 150 && powerUpLevelExpirationMs > now { + powerUpLevel = null.IntFrom(2) + powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) + } else if powerUpLevelExpirationMs > now { + powerUpLevel = null.IntFrom(3) + powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) + } else { + powerUpLevel = null.IntFrom(0) + } + + return powerUpLevel, powerUpEndTimestamp +} + +func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64) *Gym { + type pokemonDisplay struct { + Form int `json:"form,omitempty"` + Costume int `json:"costume,omitempty"` + Gender int `json:"gender"` + Shiny bool `json:"shiny,omitempty"` + TempEvolution int `json:"temp_evolution,omitempty"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` + Alignment int `json:"alignment,omitempty"` + Badge int `json:"badge,omitempty"` + Background *int64 `json:"background,omitempty"` + } + gym.SetId(fortData.FortId) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) + gym.SetEnabled(null.IntFrom(util.BoolToInt[int64](fortData.Enabled))) + gym.SetGuardingPokemonId(null.IntFrom(int64(fortData.GuardPokemonId))) + if fortData.GuardPokemonDisplay == nil { + gym.SetGuardingPokemonDisplay(null.NewString("", false)) + } else { + display, _ := json.Marshal(pokemonDisplay{ + Form: int(fortData.GuardPokemonDisplay.Form), + Costume: int(fortData.GuardPokemonDisplay.Costume), + Gender: int(fortData.GuardPokemonDisplay.Gender), + Shiny: fortData.GuardPokemonDisplay.Shiny, + TempEvolution: int(fortData.GuardPokemonDisplay.CurrentTempEvolution), + TempEvolutionFinishMs: fortData.GuardPokemonDisplay.TemporaryEvolutionFinishMs, + Alignment: int(fortData.GuardPokemonDisplay.Alignment), + Badge: int(fortData.GuardPokemonDisplay.PokemonBadge), + Background: util.ExtractBackgroundFromDisplay(fortData.GuardPokemonDisplay), + }) + gym.SetGuardingPokemonDisplay(null.StringFrom(string(display))) + } + gym.SetTeamId(null.IntFrom(int64(fortData.Team))) + if fortData.GymDisplay != nil { + gym.SetAvailableSlots(null.IntFrom(int64(fortData.GymDisplay.SlotsAvailable))) + } else { + gym.SetAvailableSlots(null.IntFrom(6)) // this may be an incorrect assumption + } + gym.SetLastModifiedTimestamp(null.IntFrom(fortData.LastModifiedMs / 1000)) + gym.SetExRaidEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsExRaidEligible))) + + if fortData.ImageUrl != "" { + gym.SetUrl(null.StringFrom(fortData.ImageUrl)) + } + gym.SetInBattle(null.IntFrom(util.BoolToInt[int64](fortData.IsInBattle))) + gym.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) + gym.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) + + powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) + gym.SetPowerUpLevel(powerUpLevel) + gym.SetPowerUpEndTimestamp(powerUpEndTimestamp) + + if fortData.PartnerId == "" { + gym.SetPartnerId(null.NewString("", false)) + } else { + gym.SetPartnerId(null.StringFrom(fortData.PartnerId)) + } + + if fortData.ImageUrl != "" { + gym.SetUrl(null.StringFrom(fortData.ImageUrl)) + } + if fortData.Team == 0 { // check!! + gym.SetTotalCp(null.IntFrom(0)) + } else { + if fortData.GymDisplay != nil { + totalCp := int64(fortData.GymDisplay.TotalGymCp) + if gym.TotalCp.Int64-totalCp > 100 || totalCp-gym.TotalCp.Int64 > 100 { + gym.SetTotalCp(null.IntFrom(totalCp)) + } + } else { + gym.SetTotalCp(null.IntFrom(0)) + } + } + + if fortData.RaidInfo != nil { + gym.SetRaidEndTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000)) + gym.SetRaidSpawnTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000)) + raidBattleTimestamp := int64(fortData.RaidInfo.RaidBattleMs) / 1000 + + if gym.RaidBattleTimestamp.ValueOrZero() != raidBattleTimestamp { + // We are reporting a new raid, clear rsvp data + gym.SetRsvps(null.NewString("", false)) + } + gym.SetRaidBattleTimestamp(null.IntFrom(raidBattleTimestamp)) + + gym.SetRaidLevel(null.IntFrom(int64(fortData.RaidInfo.RaidLevel))) + if fortData.RaidInfo.RaidPokemon != nil { + gym.SetRaidPokemonId(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonId))) + gym.SetRaidPokemonMove1(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move1))) + gym.SetRaidPokemonMove2(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move2))) + gym.SetRaidPokemonForm(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Form))) + gym.SetRaidPokemonAlignment(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Alignment))) + gym.SetRaidPokemonCp(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Cp))) + gym.SetRaidPokemonGender(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Gender))) + gym.SetRaidPokemonCostume(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Costume))) + gym.SetRaidPokemonEvolution(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.CurrentTempEvolution))) + } else { + gym.SetRaidPokemonId(null.IntFrom(0)) + gym.SetRaidPokemonMove1(null.IntFrom(0)) + gym.SetRaidPokemonMove2(null.IntFrom(0)) + gym.SetRaidPokemonForm(null.IntFrom(0)) + gym.SetRaidPokemonAlignment(null.IntFrom(0)) + gym.SetRaidPokemonCp(null.IntFrom(0)) + gym.SetRaidPokemonGender(null.IntFrom(0)) + gym.SetRaidPokemonCostume(null.IntFrom(0)) + gym.SetRaidPokemonEvolution(null.IntFrom(0)) + } + + gym.SetRaidIsExclusive(null.IntFrom(0)) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive)) + } + + gym.SetCellId(null.IntFrom(int64(cellId))) + + if gym.Deleted { + gym.SetDeleted(false) + log.Warnf("Cleared Gym with id '%s' is found again in GMO, therefore un-deleted", gym.Id) + // Restore in fort tracker if enabled + if fortTracker != nil { + fortTracker.RestoreFort(gym.Id, cellId, true, time.Now().Unix()) + } + } + + return gym +} + +func (gym *Gym) updateGymFromFortProto(fortData *pogo.FortDetailsOutProto) *Gym { + gym.SetId(fortData.Id) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) + if len(fortData.ImageUrl) > 0 { + gym.SetUrl(null.StringFrom(fortData.ImageUrl[0])) + } + gym.SetName(null.StringFrom(fortData.Name)) + + return gym +} + +func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) *Gym { + gym.SetId(gymData.GymStatusAndDefenders.PokemonFortProto.FortId) + gym.SetLat(gymData.GymStatusAndDefenders.PokemonFortProto.Latitude) + gym.SetLon(gymData.GymStatusAndDefenders.PokemonFortProto.Longitude) + + // This will have gym defenders in it... + if len(gymData.Url) > 0 { + gym.SetUrl(null.StringFrom(gymData.Url)) + } + gym.SetName(null.StringFrom(gymData.Name)) + + if gymData.Description == "" { + gym.SetDescription(null.NewString("", false)) + } else { + gym.SetDescription(null.StringFrom(gymData.Description)) + } + + type pokemonGymDefender struct { + PokemonId int `json:"pokemon_id,omitempty"` + Form int `json:"form,omitempty"` + Costume int `json:"costume,omitempty"` + Gender int `json:"gender"` + Shiny bool `json:"shiny,omitempty"` + TempEvolution int `json:"temp_evolution,omitempty"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` + Alignment int `json:"alignment,omitempty"` + Badge int `json:"badge,omitempty"` + Background *int64 `json:"background,omitempty"` + DeployedMs int64 `json:"deployed_ms,omitempty"` + DeployedTime int64 `json:"deployed_time,omitempty"` + BattlesWon int32 `json:"battles_won"` + BattlesLost int32 `json:"battles_lost"` + TimesFed int32 `json:"times_fed"` + MotivationNow util.RoundedFloat4 `json:"motivation_now"` + CpNow int32 `json:"cp_now"` + CpWhenDeployed int32 `json:"cp_when_deployed"` + } + + var defenders []pokemonGymDefender + now := time.Now() + for _, protoDefender := range gymData.GymStatusAndDefenders.GymDefender { + motivatedPokemon := protoDefender.MotivatedPokemon + pokemonDisplay := motivatedPokemon.Pokemon.PokemonDisplay + deploymentTotals := protoDefender.DeploymentTotals + defender := pokemonGymDefender{ + DeployedMs: protoDefender.DeploymentTotals.DeploymentDurationMs, + DeployedTime: now. + Add(-1 * time.Millisecond * time.Duration(deploymentTotals.DeploymentDurationMs)). + Unix(), // This will only be approximately correct + BattlesLost: deploymentTotals.BattlesLost, + BattlesWon: deploymentTotals.BattlesWon, + TimesFed: deploymentTotals.TimesFed, + PokemonId: int(protoDefender.MotivatedPokemon.Pokemon.PokemonId), + Form: int(pokemonDisplay.Form), + Costume: int(pokemonDisplay.Costume), + Gender: int(pokemonDisplay.Gender), + TempEvolution: int(pokemonDisplay.CurrentTempEvolution), + TempEvolutionFinishMs: pokemonDisplay.TemporaryEvolutionFinishMs, + Alignment: int(pokemonDisplay.Alignment), + Badge: int(pokemonDisplay.PokemonBadge), + Background: util.ExtractBackgroundFromDisplay(pokemonDisplay), + Shiny: pokemonDisplay.Shiny, + MotivationNow: util.RoundedFloat4(motivatedPokemon.MotivationNow), + CpNow: motivatedPokemon.CpNow, + CpWhenDeployed: motivatedPokemon.CpWhenDeployed, + } + defenders = append(defenders, defender) + } + bDefenders, _ := json.Marshal(defenders) + gym.SetDefenders(null.StringFrom(string(bDefenders))) + // log.Debugf("Gym %s defenders %s ", gym.Id, string(bDefenders)) + + return gym +} + +func (gym *Gym) updateGymFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto, skipName bool) *Gym { + gym.SetId(fortData.Id) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) + + if len(fortData.Image) > 0 { + gym.SetUrl(null.StringFrom(fortData.Image[0].Url)) + } + if !skipName { + gym.SetName(null.StringFrom(fortData.Name)) + } + + if gym.Deleted { + log.Debugf("Cleared Gym with id '%s' is found again in GMF, therefore kept deleted", gym.Id) + } + + return gym +} + +func (gym *Gym) updateGymFromRsvpProto(fortData *pogo.GetEventRsvpsOutProto) *Gym { + type rsvpTimeslot struct { + Timeslot int64 `json:"timeslot"` + GoingCount int32 `json:"going_count"` + MaybeCount int32 `json:"maybe_count"` + } + + timeslots := make([]rsvpTimeslot, 0) + + for _, timeslot := range fortData.RsvpTimeslots { + if timeslot.GoingCount > 0 || timeslot.MaybeCount > 0 { + timeslots = append(timeslots, rsvpTimeslot{ + Timeslot: timeslot.TimeSlot, + GoingCount: timeslot.GoingCount, + MaybeCount: timeslot.MaybeCount, + }) + } + } + + if len(timeslots) == 0 { + gym.SetRsvps(null.NewString("", false)) + } else { + slices.SortFunc(timeslots, func(a, b rsvpTimeslot) int { + return cmp.Compare(a.Timeslot, b.Timeslot) + }) + + bRsvps, _ := json.Marshal(timeslots) + gym.SetRsvps(null.StringFrom(string(bRsvps))) + } + + return gym +} diff --git a/decoder/gym_process.go b/decoder/gym_process.go new file mode 100644 index 00000000..1c34104c --- /dev/null +++ b/decoder/gym_process.go @@ -0,0 +1,97 @@ +package decoder + +import ( + "context" + "fmt" + + "github.com/guregu/null/v6" + + "golbat/db" + "golbat/pogo" +) + +func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { + gym, unlock, err := getOrCreateGymRecord(ctx, db, fort.Id) + if err != nil { + return err.Error() + } + defer unlock() + + gym.updateGymFromFortProto(fort) + + updateGymGetMapFortCache(gym, true) + saveGymRecord(ctx, db, gym) + + return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} + +func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymInfo *pogo.GymGetInfoOutProto) string { + gym, unlock, err := getOrCreateGymRecord(ctx, db, gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) + if err != nil { + return err.Error() + } + defer unlock() + + gym.updateGymFromGymInfoOutProto(gymInfo) + + updateGymGetMapFortCache(gym, true) + saveGymRecord(ctx, db, gym) + return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} + +func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { + gym, unlock, err := getGymRecordForUpdate(ctx, db, mapFort.Id) + if err != nil { + return false, err.Error() + } + + // we missed it in Pokestop & Gym. Lets save it to cache + if gym == nil { + return false, "" + } + defer unlock() + + gym.updateGymFromGetMapFortsOutProto(mapFort, false) + saveGymRecord(ctx, db, gym) + return true, fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} + +func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pogo.RaidDetails, resp *pogo.GetEventRsvpsOutProto) string { + gym, unlock, err := getGymRecordForUpdate(ctx, db, req.FortId) + if err != nil { + return err.Error() + } + + if gym == nil { + // Do not add RSVP details to unknown gyms + return fmt.Sprintf("%s Gym not present", req.FortId) + } + defer unlock() + + gym.updateGymFromRsvpProto(resp) + + saveGymRecord(ctx, db, gym) + + return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} + +func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { + gym, unlock, err := getGymRecordForUpdate(ctx, db, fortId) + if err != nil { + return err.Error() + } + + if gym == nil { + // Do not add RSVP details to unknown gyms + return fmt.Sprintf("%s Gym not present", fortId) + } + defer unlock() + + if gym.Rsvps.Valid { + gym.SetRsvps(null.NewString("", false)) + + saveGymRecord(ctx, db, gym) + } + + return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} diff --git a/decoder/gym_state.go b/decoder/gym_state.go new file mode 100644 index 00000000..ce235085 --- /dev/null +++ b/decoder/gym_state.go @@ -0,0 +1,430 @@ +package decoder + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "time" + + "golbat/config" + "golbat/db" + "golbat/geo" + "golbat/webhooks" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" +) + +func loadGymFromDatabase(ctx context.Context, db db.DbDetails, fortId string, gym *Gym) error { + err := db.GeneralDb.GetContext(ctx, gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) + statsCollector.IncDbQuery("select gym", err) + return err +} + +// PeekGymRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekGymRecord(fortId string) (*Gym, func(), error) { + if item := gymCache.Get(fortId); item != nil { + gym := item.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil + } + return nil, nil, nil +} + +// GetGymRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func GetGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + // Check cache first + if item := gymCache.Get(fortId); item != nil { + gym := item.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil + } + + dbGym := Gym{} + err := loadGymFromDatabase(ctx, db, fortId, &dbGym) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbGym.ClearDirty() + + // Atomically cache the loaded Gym - if another goroutine raced us, + // we'll get their Gym and use that instead (ensuring same mutex) + existingGym, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { + // Only called if key doesn't exist - our Pokestop wins + if config.Config.TestFortInMemory { + fortRtreeUpdateGymOnGet(&dbGym) + } + return &dbGym + }) + + gym := existingGym.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil +} + +// getGymRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Gym. +// Caller MUST call returned unlock function if non-nil. +func getGymRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + gym, unlock, err := GetGymRecordReadOnly(ctx, db, fortId) + if err != nil || gym == nil { + return nil, nil, err + } + gym.snapshotOldValues() + return gym, unlock, nil +} + +// getOrCreateGymRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + // Create new Gym atomically - function only called if key doesn't exist + gymItem, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { + return &Gym{Id: fortId, newRecord: true} + }) + + gym := gymItem.Value() + gym.Lock() + + if gym.newRecord { + // We should attempt to load from database + err := loadGymFromDatabase(ctx, db, fortId, gym) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + gym.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + gym.newRecord = false + gym.ClearDirty() + if config.Config.TestFortInMemory { + fortRtreeUpdateGymOnGet(gym) + } + } + } + + gym.snapshotOldValues() + return gym, func() { gym.Unlock() }, nil +} + +// hasChangesGym compares two Gym structs +// Float tolerance: Lat, Lon +func hasChangesGym(old *Gym, new *Gym) bool { + return old.Id != new.Id || + old.Name != new.Name || + old.Url != new.Url || + old.LastModifiedTimestamp != new.LastModifiedTimestamp || + old.RaidEndTimestamp != new.RaidEndTimestamp || + old.RaidSpawnTimestamp != new.RaidSpawnTimestamp || + old.RaidBattleTimestamp != new.RaidBattleTimestamp || + old.Updated != new.Updated || + old.RaidPokemonId != new.RaidPokemonId || + old.GuardingPokemonId != new.GuardingPokemonId || + old.AvailableSlots != new.AvailableSlots || + old.TeamId != new.TeamId || + old.RaidLevel != new.RaidLevel || + old.Enabled != new.Enabled || + old.ExRaidEligible != new.ExRaidEligible || + // old.InBattle != new.InBattle || + old.RaidPokemonMove1 != new.RaidPokemonMove1 || + old.RaidPokemonMove2 != new.RaidPokemonMove2 || + old.RaidPokemonForm != new.RaidPokemonForm || + old.RaidPokemonAlignment != new.RaidPokemonAlignment || + old.RaidPokemonCp != new.RaidPokemonCp || + old.RaidIsExclusive != new.RaidIsExclusive || + old.CellId != new.CellId || + old.Deleted != new.Deleted || + old.TotalCp != new.TotalCp || + old.FirstSeenTimestamp != new.FirstSeenTimestamp || + old.RaidPokemonGender != new.RaidPokemonGender || + old.SponsorId != new.SponsorId || + old.PartnerId != new.PartnerId || + old.RaidPokemonCostume != new.RaidPokemonCostume || + old.RaidPokemonEvolution != new.RaidPokemonEvolution || + old.ArScanEligible != new.ArScanEligible || + old.PowerUpLevel != new.PowerUpLevel || + old.PowerUpPoints != new.PowerUpPoints || + old.PowerUpEndTimestamp != new.PowerUpEndTimestamp || + old.Description != new.Description || + old.Rsvps != new.Rsvps || + !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || + !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) + +} + +// hasChangesInternalGym compares two Gym structs for changes that will be stored in memory +// Float tolerance: Lat, Lon +func hasInternalChangesGym(old *Gym, new *Gym) bool { + return old.InBattle != new.InBattle || + old.Defenders != new.Defenders +} + +type GymDetailsWebhook struct { + Id string `json:"id"` + Name string `json:"name"` + Url string `json:"url"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Team int64 `json:"team"` + GuardPokemonId int64 `json:"guard_pokemon_id"` + SlotsAvailable int64 `json:"slots_available"` + ExRaidEligible int64 `json:"ex_raid_eligible"` + InBattle bool `json:"in_battle"` + SponsorId int64 `json:"sponsor_id"` + PartnerId int64 `json:"partner_id"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + ArScanEligible int64 `json:"ar_scan_eligible"` + Defenders any `json:"defenders"` +} + +type RaidWebhook struct { + GymId string `json:"gym_id"` + GymName string `json:"gym_name"` + GymUrl string `json:"gym_url"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + TeamId int64 `json:"team_id"` + Spawn int64 `json:"spawn"` + Start int64 `json:"start"` + End int64 `json:"end"` + Level int64 `json:"level"` + PokemonId int64 `json:"pokemon_id"` + Cp int64 `json:"cp"` + Gender int64 `json:"gender"` + Form int64 `json:"form"` + Alignment int64 `json:"alignment"` + Costume int64 `json:"costume"` + Evolution int64 `json:"evolution"` + Move1 int64 `json:"move_1"` + Move2 int64 `json:"move_2"` + ExRaidEligible int64 `json:"ex_raid_eligible"` + IsExclusive int64 `json:"is_exclusive"` + SponsorId int64 `json:"sponsor_id"` + PartnerId string `json:"partner_id"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + ArScanEligible int64 `json:"ar_scan_eligible"` + Rsvps json.RawMessage `json:"rsvps"` +} + +func createGymFortWebhooks(gym *Gym) { + fort := InitWebHookFortFromGym(gym) + if gym.newRecord { + CreateFortWebHooks(nil, fort, NEW) + } else { + // Build old fort from saved old values + oldFort := &FortWebhook{ + Type: GYM.String(), + Id: gym.Id, + Name: gym.oldValues.Name.Ptr(), + ImageUrl: gym.oldValues.Url.Ptr(), + Description: gym.oldValues.Description.Ptr(), + Location: Location{Latitude: gym.oldValues.Lat, Longitude: gym.oldValues.Lon}, + } + CreateFortWebHooks(oldFort, fort, EDIT) + } +} + +func createGymWebhooks(gym *Gym, areas []geo.AreaName) { + if gym.newRecord || + (gym.oldValues.AvailableSlots != gym.AvailableSlots || gym.oldValues.TeamId != gym.TeamId || gym.oldValues.InBattle != gym.InBattle) { + gymDetails := GymDetailsWebhook{ + Id: gym.Id, + Name: gym.Name.ValueOrZero(), + Url: gym.Url.ValueOrZero(), + Latitude: gym.Lat, + Longitude: gym.Lon, + Team: gym.TeamId.ValueOrZero(), + GuardPokemonId: gym.GuardingPokemonId.ValueOrZero(), + SlotsAvailable: func() int64 { + if gym.AvailableSlots.Valid { + return gym.AvailableSlots.Int64 + } else { + return 6 + } + }(), + ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), + InBattle: func() bool { return gym.InBattle.ValueOrZero() != 0 }(), + Defenders: func() any { + if gym.Defenders.Valid { + return json.RawMessage(gym.Defenders.ValueOrZero()) + } else { + return nil + } + }(), + } + + webhooksSender.AddMessage(webhooks.GymDetails, gymDetails, areas) + } + + if gym.RaidSpawnTimestamp.ValueOrZero() > 0 && + (gym.newRecord || gym.oldValues.RaidLevel != gym.RaidLevel || + gym.oldValues.RaidPokemonId != gym.RaidPokemonId || + gym.oldValues.RaidSpawnTimestamp != gym.RaidSpawnTimestamp || gym.oldValues.Rsvps != gym.Rsvps) { + raidBattleTime := gym.RaidBattleTimestamp.ValueOrZero() + raidEndTime := gym.RaidEndTimestamp.ValueOrZero() + now := time.Now().Unix() + + if (raidBattleTime > now && gym.RaidLevel.ValueOrZero() > 0) || + (raidEndTime > now && gym.RaidPokemonId.ValueOrZero() > 0) { + gymName := "Unknown" + if gym.Name.Valid { + gymName = gym.Name.String + } + + var rsvps json.RawMessage + if gym.Rsvps.Valid { + rsvps = json.RawMessage(gym.Rsvps.ValueOrZero()) + } + + raidHook := RaidWebhook{ + GymId: gym.Id, + GymName: gymName, + GymUrl: gym.Url.ValueOrZero(), + Latitude: gym.Lat, + Longitude: gym.Lon, + TeamId: gym.TeamId.ValueOrZero(), + Spawn: gym.RaidSpawnTimestamp.ValueOrZero(), + Start: gym.RaidBattleTimestamp.ValueOrZero(), + End: gym.RaidEndTimestamp.ValueOrZero(), + Level: gym.RaidLevel.ValueOrZero(), + PokemonId: gym.RaidPokemonId.ValueOrZero(), + Cp: gym.RaidPokemonCp.ValueOrZero(), + Gender: gym.RaidPokemonGender.ValueOrZero(), + Form: gym.RaidPokemonForm.ValueOrZero(), + Alignment: gym.RaidPokemonAlignment.ValueOrZero(), + Costume: gym.RaidPokemonCostume.ValueOrZero(), + Evolution: gym.RaidPokemonEvolution.ValueOrZero(), + Move1: gym.RaidPokemonMove1.ValueOrZero(), + Move2: gym.RaidPokemonMove2.ValueOrZero(), + ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), + IsExclusive: gym.RaidIsExclusive.ValueOrZero(), + SponsorId: gym.SponsorId.ValueOrZero(), + PartnerId: gym.PartnerId.ValueOrZero(), + PowerUpPoints: gym.PowerUpPoints.ValueOrZero(), + PowerUpLevel: gym.PowerUpLevel.ValueOrZero(), + PowerUpEndTimestamp: gym.PowerUpEndTimestamp.ValueOrZero(), + ArScanEligible: gym.ArScanEligible.ValueOrZero(), + Rsvps: rsvps, + } + + webhooksSender.AddMessage(webhooks.Raid, raidHook, areas) + statsCollector.UpdateRaidCount(areas, gym.RaidLevel.ValueOrZero()) + } + } +} + +func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { + now := time.Now().Unix() + if !gym.IsNewRecord() && !gym.IsDirty() && !gym.IsInternalDirty() { + // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. + if gym.Updated > now-GetUpdateThreshold(900) { + // if a gym is unchanged and was seen recently, skip saving + return + } + } + gym.Updated = now + + if gym.IsDirty() { + if gym.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ + "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) + + statsCollector.IncDbQuery("insert gym", err) + if err != nil { + log.Errorf("insert gym: %s", err) + return + } + + _, _ = res, err + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ + "lat = :lat, "+ + "lon = :lon, "+ + "name = :name, "+ + "url = :url, "+ + "last_modified_timestamp = :last_modified_timestamp, "+ + "raid_end_timestamp = :raid_end_timestamp, "+ + "raid_spawn_timestamp = :raid_spawn_timestamp, "+ + "raid_battle_timestamp = :raid_battle_timestamp, "+ + "updated = :updated, "+ + "raid_pokemon_id = :raid_pokemon_id, "+ + "guarding_pokemon_id = :guarding_pokemon_id, "+ + "guarding_pokemon_display = :guarding_pokemon_display, "+ + "available_slots = :available_slots, "+ + "team_id = :team_id, "+ + "raid_level = :raid_level, "+ + "enabled = :enabled, "+ + "ex_raid_eligible = :ex_raid_eligible, "+ + "in_battle = :in_battle, "+ + "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ + "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ + "raid_pokemon_form = :raid_pokemon_form, "+ + "raid_pokemon_alignment = :raid_pokemon_alignment, "+ + "raid_pokemon_cp = :raid_pokemon_cp, "+ + "raid_is_exclusive = :raid_is_exclusive, "+ + "cell_id = :cell_id, "+ + "deleted = :deleted, "+ + "total_cp = :total_cp, "+ + "raid_pokemon_gender = :raid_pokemon_gender, "+ + "sponsor_id = :sponsor_id, "+ + "partner_id = :partner_id, "+ + "raid_pokemon_costume = :raid_pokemon_costume, "+ + "raid_pokemon_evolution = :raid_pokemon_evolution, "+ + "ar_scan_eligible = :ar_scan_eligible, "+ + "power_up_level = :power_up_level, "+ + "power_up_points = :power_up_points, "+ + "power_up_end_timestamp = :power_up_end_timestamp,"+ + "description = :description,"+ + "defenders = :defenders,"+ + "rsvps = :rsvps "+ + "WHERE id = :id", gym, + ) + statsCollector.IncDbQuery("update gym", err) + if err != nil { + log.Errorf("Update gym %s", err) + } + _, _ = res, err + } + } + + //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + areas := MatchStatsGeofence(gym.Lat, gym.Lon) + createGymWebhooks(gym, areas) + createGymFortWebhooks(gym) + updateRaidStats(gym, areas) + if dbDebugEnabled { + gym.changedFields = gym.changedFields[:0] + } + if gym.IsNewRecord() { + gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + gym.newRecord = false + } + gym.ClearDirty() +} + +func updateGymGetMapFortCache(gym *Gym, skipName bool) { + storedGetMapFort := getMapFortsCache.Get(gym.Id) + if storedGetMapFort != nil { + getMapFort := storedGetMapFort.Value() + getMapFortsCache.Delete(gym.Id) + gym.updateGymFromGetMapFortsOutProto(getMapFort, skipName) + log.Debugf("Updated Gym using stored getMapFort: %s", gym.Id) + } +} diff --git a/decoder/incident.go b/decoder/incident.go index 38b5a992..7149a978 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -1,19 +1,9 @@ package decoder import ( - "context" - "database/sql" - "errors" "sync" - "time" - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - null "gopkg.in/guregu/null.v4" - - "golbat/db" - "golbat/pogo" - "golbat/webhooks" + null "github.com/guregu/null/v6" ) // Incident struct. @@ -223,269 +213,3 @@ func (incident *Incident) SetSlot3Form(v null.Int) { incident.dirty = true } } - -func loadIncidentFromDatabase(ctx context.Context, db db.DbDetails, incidentId string, incident *Incident) error { - err := db.GeneralDb.GetContext(ctx, incident, - "SELECT id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form "+ - "FROM incident WHERE incident.id = ?", incidentId) - statsCollector.IncDbQuery("select incident", err) - return err -} - -// peekIncidentRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func peekIncidentRecord(incidentId string) (*Incident, func(), error) { - if item := incidentCache.Get(incidentId); item != nil { - incident := item.Value() - incident.Lock() - return incident, func() { incident.Unlock() }, nil - } - return nil, nil, nil -} - -// getIncidentRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getIncidentRecordReadOnly(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { - // Check cache first - if item := incidentCache.Get(incidentId); item != nil { - incident := item.Value() - incident.Lock() - return incident, func() { incident.Unlock() }, nil - } - - dbIncident := Incident{} - err := loadIncidentFromDatabase(ctx, db, incidentId, &dbIncident) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbIncident.ClearDirty() - - // Atomically cache the loaded Incident - if another goroutine raced us, - // we'll get their Incident and use that instead (ensuring same mutex) - existingIncident, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { - return &dbIncident - }) - - incident := existingIncident.Value() - incident.Lock() - return incident, func() { incident.Unlock() }, nil -} - -// getIncidentRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Caller MUST call returned unlock function if non-nil. -func getIncidentRecordForUpdate(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { - incident, unlock, err := getIncidentRecordReadOnly(ctx, db, incidentId) - if err != nil || incident == nil { - return nil, nil, err - } - incident.snapshotOldValues() - return incident, unlock, nil -} - -// getOrCreateIncidentRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreateIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string, pokestopId string) (*Incident, func(), error) { - // Create new Incident atomically - function only called if key doesn't exist - incidentItem, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { - return &Incident{Id: incidentId, PokestopId: pokestopId, newRecord: true} - }) - - incident := incidentItem.Value() - incident.Lock() - - if incident.newRecord { - // We should attempt to load from database - err := loadIncidentFromDatabase(ctx, db, incidentId, incident) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - incident.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - incident.newRecord = false - incident.ClearDirty() - } - } - - incident.snapshotOldValues() - return incident, func() { incident.Unlock() }, nil -} - -func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident) { - // Skip save if not dirty and not new - if !incident.IsDirty() && !incident.IsNewRecord() { - return - } - - incident.Updated = time.Now().Unix() - - if incident.IsNewRecord() { - res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ - "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) - - if err != nil { - log.Errorf("insert incident: %s", err) - return - } - statsCollector.IncDbQuery("insert incident", err) - _, _ = res, err - } else { - res, err := db.GeneralDb.NamedExec("UPDATE incident SET "+ - "start = :start, "+ - "expiration = :expiration, "+ - "display_type = :display_type, "+ - "style = :style, "+ - "`character` = :character, "+ - "updated = :updated, "+ - "confirmed = :confirmed, "+ - "slot_1_pokemon_id = :slot_1_pokemon_id, "+ - "slot_1_form = :slot_1_form, "+ - "slot_2_pokemon_id = :slot_2_pokemon_id, "+ - "slot_2_form = :slot_2_form, "+ - "slot_3_pokemon_id = :slot_3_pokemon_id, "+ - "slot_3_form = :slot_3_form "+ - "WHERE id = :id", incident, - ) - statsCollector.IncDbQuery("update incident", err) - if err != nil { - log.Errorf("Update incident %s", err) - } - _, _ = res, err - } - - createIncidentWebhooks(ctx, db, incident) - - var stopLat, stopLon float64 - stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) - if stop != nil { - stopLat, stopLon = stop.Lat, stop.Lon - unlock() - } - - areas := MatchStatsGeofence(stopLat, stopLon) - updateIncidentStats(incident, areas) - - incident.ClearDirty() - if incident.IsNewRecord() { - incident.newRecord = false - incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) - } -} - -func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { - old := &incident.oldValues - isNew := incident.IsNewRecord() - - if isNew || (old.ExpirationTime != incident.ExpirationTime || old.Character != incident.Character || old.Confirmed != incident.Confirmed || old.Slot1PokemonId != incident.Slot1PokemonId) { - var pokestopName, stopUrl string - var stopLat, stopLon float64 - var stopEnabled bool - stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) - if stop != nil { - pokestopName = stop.Name.ValueOrZero() - stopLat, stopLon = stop.Lat, stop.Lon - stopUrl = stop.Url.ValueOrZero() - stopEnabled = stop.Enabled.ValueOrZero() - unlock() - } - if pokestopName == "" { - pokestopName = "Unknown" - } - - var lineup []webhookLineup - if incident.Slot1PokemonId.Valid { - lineup = []webhookLineup{ - { - Slot: 1, - PokemonId: incident.Slot1PokemonId, - Form: incident.Slot1Form, - }, - { - Slot: 2, - PokemonId: incident.Slot2PokemonId, - Form: incident.Slot2Form, - }, - { - Slot: 3, - PokemonId: incident.Slot3PokemonId, - Form: incident.Slot3Form, - }, - } - } - - incidentHook := IncidentWebhook{ - Id: incident.Id, - PokestopId: incident.PokestopId, - Latitude: stopLat, - Longitude: stopLon, - PokestopName: pokestopName, - Url: stopUrl, - Enabled: stopEnabled, - Start: incident.StartTime, - IncidentExpireTimestamp: incident.ExpirationTime, - Expiration: incident.ExpirationTime, - DisplayType: incident.DisplayType, - Style: incident.Style, - GruntType: incident.Character, - Character: incident.Character, - Updated: incident.Updated, - Confirmed: incident.Confirmed, - Lineup: lineup, - } - - areas := MatchStatsGeofence(stop.Lat, stop.Lon) - webhooksSender.AddMessage(webhooks.Invasion, incidentHook, areas) - statsCollector.UpdateIncidentCount(areas) - } -} - -func (incident *Incident) updateFromPokestopIncidentDisplay(pokestopDisplay *pogo.PokestopIncidentDisplayProto) { - incident.SetId(pokestopDisplay.IncidentId) - incident.SetStartTime(int64(pokestopDisplay.IncidentStartMs / 1000)) - incident.SetExpirationTime(int64(pokestopDisplay.IncidentExpirationMs / 1000)) - incident.SetDisplayType(int16(pokestopDisplay.IncidentDisplayType)) - if (incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE)) && incident.Confirmed { - log.Debugf("Incident has already been confirmed as a decoy: %s", incident.Id) - return - } - characterDisplay := pokestopDisplay.GetCharacterDisplay() - if characterDisplay != nil { - // team := pokestopDisplay.Open - incident.SetStyle(int16(characterDisplay.Style)) - incident.SetCharacter(int16(characterDisplay.Character)) - } else { - incident.SetStyle(0) - incident.SetCharacter(0) - } -} - -func (incident *Incident) updateFromOpenInvasionCombatSessionOut(protoRes *pogo.OpenInvasionCombatSessionOutProto) { - incident.SetSlot1PokemonId(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokedexId.Number()), true)) - incident.SetSlot1Form(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokemonDisplay.Form.Number()), true)) - for i, pokemon := range protoRes.Combat.Opponent.ReservePokemon { - if i == 0 { - incident.SetSlot2PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) - incident.SetSlot2Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) - } else if i == 1 { - incident.SetSlot3PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) - incident.SetSlot3Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) - } - } - incident.SetConfirmed(true) -} - -func (incident *Incident) updateFromStartIncidentOut(proto *pogo.StartIncidentOutProto) { - incident.SetCharacter(int16(proto.GetIncident().GetStep()[0].GetPokestopDialogue().GetDialogueLine()[0].GetCharacter())) - if incident.Character == int16(pogo.EnumWrapper_CHARACTER_GIOVANNI) || - incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || - incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE) { - incident.SetConfirmed(true) - } - incident.SetStartTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentStartMs() / 1000)) - incident.SetExpirationTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentExpirationMs() / 1000)) -} diff --git a/decoder/incident_decode.go b/decoder/incident_decode.go new file mode 100644 index 00000000..046105fc --- /dev/null +++ b/decoder/incident_decode.go @@ -0,0 +1,54 @@ +package decoder + +import ( + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" +) + +func (incident *Incident) updateFromPokestopIncidentDisplay(pokestopDisplay *pogo.PokestopIncidentDisplayProto) { + incident.SetId(pokestopDisplay.IncidentId) + incident.SetStartTime(int64(pokestopDisplay.IncidentStartMs / 1000)) + incident.SetExpirationTime(int64(pokestopDisplay.IncidentExpirationMs / 1000)) + incident.SetDisplayType(int16(pokestopDisplay.IncidentDisplayType)) + if (incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE)) && incident.Confirmed { + log.Debugf("Incident has already been confirmed as a decoy: %s", incident.Id) + return + } + characterDisplay := pokestopDisplay.GetCharacterDisplay() + if characterDisplay != nil { + // team := pokestopDisplay.Open + incident.SetStyle(int16(characterDisplay.Style)) + incident.SetCharacter(int16(characterDisplay.Character)) + } else { + incident.SetStyle(0) + incident.SetCharacter(0) + } +} + +func (incident *Incident) updateFromOpenInvasionCombatSessionOut(protoRes *pogo.OpenInvasionCombatSessionOutProto) { + incident.SetSlot1PokemonId(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokedexId.Number()), true)) + incident.SetSlot1Form(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokemonDisplay.Form.Number()), true)) + for i, pokemon := range protoRes.Combat.Opponent.ReservePokemon { + if i == 0 { + incident.SetSlot2PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) + incident.SetSlot2Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) + } else if i == 1 { + incident.SetSlot3PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) + incident.SetSlot3Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) + } + } + incident.SetConfirmed(true) +} + +func (incident *Incident) updateFromStartIncidentOut(proto *pogo.StartIncidentOutProto) { + incident.SetCharacter(int16(proto.GetIncident().GetStep()[0].GetPokestopDialogue().GetDialogueLine()[0].GetCharacter())) + if incident.Character == int16(pogo.EnumWrapper_CHARACTER_GIOVANNI) || + incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || + incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE) { + incident.SetConfirmed(true) + } + incident.SetStartTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentStartMs() / 1000)) + incident.SetExpirationTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentExpirationMs() / 1000)) +} diff --git a/decoder/incident_process.go b/decoder/incident_process.go new file mode 100644 index 00000000..4550c7c9 --- /dev/null +++ b/decoder/incident_process.go @@ -0,0 +1,43 @@ +package decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func UpdateIncidentLineup(ctx context.Context, db db.DbDetails, protoReq *pogo.OpenInvasionCombatSessionProto, protoRes *pogo.OpenInvasionCombatSessionOutProto) string { + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, protoReq.IncidentLookup.IncidentId, protoReq.IncidentLookup.FortId) + if err != nil { + return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) + } + defer unlock() + + if incident.newRecord { + log.Debugf("Updating lineup before it was saved: %s", protoReq.IncidentLookup.IncidentId) + } + incident.updateFromOpenInvasionCombatSessionOut(protoRes) + + saveIncidentRecord(ctx, db, incident) + return "" +} + +func ConfirmIncident(ctx context.Context, db db.DbDetails, proto *pogo.StartIncidentOutProto) string { + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, proto.Incident.IncidentId, proto.Incident.FortId) + if err != nil { + return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) + } + defer unlock() + + if incident.newRecord { + log.Debugf("Confirming incident before it was saved: %s", proto.Incident.IncidentId) + } + incident.updateFromStartIncidentOut(proto) + + saveIncidentRecord(ctx, db, incident) + return "" +} diff --git a/decoder/incident_state.go b/decoder/incident_state.go new file mode 100644 index 00000000..d3b8d80f --- /dev/null +++ b/decoder/incident_state.go @@ -0,0 +1,234 @@ +package decoder + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/webhooks" +) + +func loadIncidentFromDatabase(ctx context.Context, db db.DbDetails, incidentId string, incident *Incident) error { + err := db.GeneralDb.GetContext(ctx, incident, + "SELECT id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form "+ + "FROM incident WHERE incident.id = ?", incidentId) + statsCollector.IncDbQuery("select incident", err) + return err +} + +// peekIncidentRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekIncidentRecord(incidentId string) (*Incident, func(), error) { + if item := incidentCache.Get(incidentId); item != nil { + incident := item.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil + } + return nil, nil, nil +} + +// getIncidentRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getIncidentRecordReadOnly(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { + // Check cache first + if item := incidentCache.Get(incidentId); item != nil { + incident := item.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil + } + + dbIncident := Incident{} + err := loadIncidentFromDatabase(ctx, db, incidentId, &dbIncident) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbIncident.ClearDirty() + + // Atomically cache the loaded Incident - if another goroutine raced us, + // we'll get their Incident and use that instead (ensuring same mutex) + existingIncident, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + return &dbIncident + }) + + incident := existingIncident.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil +} + +// getIncidentRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getIncidentRecordForUpdate(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { + incident, unlock, err := getIncidentRecordReadOnly(ctx, db, incidentId) + if err != nil || incident == nil { + return nil, nil, err + } + incident.snapshotOldValues() + return incident, unlock, nil +} + +// getOrCreateIncidentRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string, pokestopId string) (*Incident, func(), error) { + // Create new Incident atomically - function only called if key doesn't exist + incidentItem, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + return &Incident{Id: incidentId, PokestopId: pokestopId, newRecord: true} + }) + + incident := incidentItem.Value() + incident.Lock() + + if incident.newRecord { + // We should attempt to load from database + err := loadIncidentFromDatabase(ctx, db, incidentId, incident) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + incident.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + incident.newRecord = false + incident.ClearDirty() + } + } + + incident.snapshotOldValues() + return incident, func() { incident.Unlock() }, nil +} + +func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident) { + // Skip save if not dirty and not new + if !incident.IsDirty() && !incident.IsNewRecord() { + return + } + + incident.Updated = time.Now().Unix() + + if incident.IsNewRecord() { + res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ + "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) + + if err != nil { + log.Errorf("insert incident: %s", err) + return + } + statsCollector.IncDbQuery("insert incident", err) + _, _ = res, err + } else { + res, err := db.GeneralDb.NamedExec("UPDATE incident SET "+ + "start = :start, "+ + "expiration = :expiration, "+ + "display_type = :display_type, "+ + "style = :style, "+ + "`character` = :character, "+ + "updated = :updated, "+ + "confirmed = :confirmed, "+ + "slot_1_pokemon_id = :slot_1_pokemon_id, "+ + "slot_1_form = :slot_1_form, "+ + "slot_2_pokemon_id = :slot_2_pokemon_id, "+ + "slot_2_form = :slot_2_form, "+ + "slot_3_pokemon_id = :slot_3_pokemon_id, "+ + "slot_3_form = :slot_3_form "+ + "WHERE id = :id", incident, + ) + statsCollector.IncDbQuery("update incident", err) + if err != nil { + log.Errorf("Update incident %s", err) + } + _, _ = res, err + } + + createIncidentWebhooks(ctx, db, incident) + + var stopLat, stopLon float64 + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + stopLat, stopLon = stop.Lat, stop.Lon + unlock() + } + + areas := MatchStatsGeofence(stopLat, stopLon) + updateIncidentStats(incident, areas) + + incident.ClearDirty() + if incident.IsNewRecord() { + incident.newRecord = false + incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) + } +} + +func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { + old := &incident.oldValues + isNew := incident.IsNewRecord() + + if isNew || (old.ExpirationTime != incident.ExpirationTime || old.Character != incident.Character || old.Confirmed != incident.Confirmed || old.Slot1PokemonId != incident.Slot1PokemonId) { + var pokestopName, stopUrl string + var stopLat, stopLon float64 + var stopEnabled bool + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + pokestopName = stop.Name.ValueOrZero() + stopLat, stopLon = stop.Lat, stop.Lon + stopUrl = stop.Url.ValueOrZero() + stopEnabled = stop.Enabled.ValueOrZero() + unlock() + } + if pokestopName == "" { + pokestopName = "Unknown" + } + + var lineup []webhookLineup + if incident.Slot1PokemonId.Valid { + lineup = []webhookLineup{ + { + Slot: 1, + PokemonId: incident.Slot1PokemonId, + Form: incident.Slot1Form, + }, + { + Slot: 2, + PokemonId: incident.Slot2PokemonId, + Form: incident.Slot2Form, + }, + { + Slot: 3, + PokemonId: incident.Slot3PokemonId, + Form: incident.Slot3Form, + }, + } + } + + incidentHook := IncidentWebhook{ + Id: incident.Id, + PokestopId: incident.PokestopId, + Latitude: stopLat, + Longitude: stopLon, + PokestopName: pokestopName, + Url: stopUrl, + Enabled: stopEnabled, + Start: incident.StartTime, + IncidentExpireTimestamp: incident.ExpirationTime, + Expiration: incident.ExpirationTime, + DisplayType: incident.DisplayType, + Style: incident.Style, + GruntType: incident.Character, + Character: incident.Character, + Updated: incident.Updated, + Confirmed: incident.Confirmed, + Lineup: lineup, + } + + areas := MatchStatsGeofence(stopLat, stopLon) + webhooksSender.AddMessage(webhooks.Invasion, incidentHook, areas) + statsCollector.UpdateIncidentCount(areas) + } +} diff --git a/decoder/main.go b/decoder/main.go index 21467b7c..51dc4e56 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -1,19 +1,16 @@ package decoder import ( - "context" - "fmt" "math" "runtime" "time" "github.com/UnownHash/gohbem" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" "golbat/config" - "golbat/db" "golbat/geo" "golbat/pogo" "golbat/stats_collector" @@ -249,251 +246,6 @@ func nullFloatAlmostEqual(a, b null.Float, tolerance float64) bool { } } -func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawFortData) { - // Logic is: - // 1. Filter out pokestops that are unchanged (last modified time) - // 2. Fetch current stops from database - // 3. Generate batch of inserts as needed (with on duplicate saveGymRecord) - - //var stopsToModify []string - - for _, fort := range p { - fortId := fort.Data.FortId - if fort.Data.FortType == pogo.FortType_CHECKPOINT && scanParameters.ProcessPokestops { - pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fortId) - if err != nil { - log.Errorf("getOrCreatePokestopRecord: %s", err) - continue - } - - pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) - - // If this is a new pokestop, check if it was converted from a gym and copy shared fields - if pokestop.IsNewRecord() { - gym, gymUnlock, _ := getGymRecordReadOnly(ctx, db, fortId) - if gym != nil { - pokestop.copySharedFieldsFrom(gym) - gymUnlock() - } - } - - savePokestopRecord(ctx, db, pokestop) - unlock() - - incidents := fort.Data.PokestopDisplays - if incidents == nil && fort.Data.PokestopDisplay != nil { - incidents = []*pogo.PokestopIncidentDisplayProto{fort.Data.PokestopDisplay} - } - - if incidents != nil { - for _, incidentProto := range incidents { - incident, unlock, err := getOrCreateIncidentRecord(ctx, db, incidentProto.IncidentId, fortId) - if err != nil { - log.Errorf("getOrCreateIncidentRecord: %s", err) - continue - } - incident.updateFromPokestopIncidentDisplay(incidentProto) - saveIncidentRecord(ctx, db, incident) - unlock() - } - } - } - - if fort.Data.FortType == pogo.FortType_GYM && scanParameters.ProcessGyms { - gym, gymUnlock, err := getOrCreateGymRecord(ctx, db, fortId) - if err != nil { - log.Errorf("getOrCreateGymRecord: %s", err) - continue - } - - gym.updateGymFromFort(fort.Data, fort.Cell) - - // If this is a new gym, check if it was converted from a pokestop and copy shared fields - if gym.IsNewRecord() { - pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, fortId) - if pokestop != nil { - gym.copySharedFieldsFrom(pokestop) - unlock() - } - } - - saveGymRecord(ctx, db, gym) - gymUnlock() - } - } -} - -func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawStationData) { - for _, stationProto := range p { - stationId := stationProto.Data.Id - station, unlock, err := getOrCreateStationRecord(ctx, db, stationId) - if err != nil { - log.Errorf("getOrCreateStationRecord: %s", err) - continue - } - station.updateFromStationProto(stationProto.Data, stationProto.Cell) - saveStationRecord(ctx, db, station) - unlock() - } -} - -func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, wildPokemonList []RawWildPokemonData, nearbyPokemonList []RawNearbyPokemonData, mapPokemonList []RawMapPokemonData, weather []*pogo.ClientWeatherProto, username string) { - weatherLookup := make(map[int64]pogo.GameplayWeatherProto_WeatherCondition) - for _, weatherProto := range weather { - weatherLookup[weatherProto.S2CellId] = weatherProto.GameplayWeather.GameplayCondition - } - - for _, wild := range wildPokemonList { - encounterId := wild.Data.EncounterId - - // spawnpointUpdateFromWild doesn't need Pokemon lock - spawnpointUpdateFromWild(ctx, db, wild.Data, wild.Timestamp) - - if scanParameters.ProcessWild { - // Use read-only getter - we're only checking if update is needed, then queuing - pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) - if err != nil { - log.Errorf("getPokemonRecordReadOnly: %s", err) - continue - } - - updateTime := wild.Timestamp / 1000 - shouldQueue := pokemon == nil || pokemon.wildSignificantUpdate(wild.Data, updateTime) - - if unlock != nil { - unlock() - } - - if shouldQueue { - // The sweeper will process it after timeout if no encounter arrives - pending := &PendingPokemon{ - EncounterId: encounterId, - WildPokemon: wild.Data, - CellId: int64(wild.Cell), - TimestampMs: wild.Timestamp, - UpdateTime: updateTime, - WeatherLookup: weatherLookup, - Username: username, - } - pokemonPendingQueue.AddPending(pending) - } - } - } - - if scanParameters.ProcessNearby { - for _, nearby := range nearbyPokemonList { - encounterId := nearby.Data.EncounterId - - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Printf("getOrCreatePokemonRecord: %s", err) - continue - } - - updateTime := nearby.Timestamp / 1000 - if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { - pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) - } - - unlock() - } - } - - for _, mapPokemon := range mapPokemonList { - encounterId := mapPokemon.Data.EncounterId - - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Printf("getOrCreatePokemonRecord: %s", err) - continue - } - - pokemon.updateFromMap(ctx, db, mapPokemon.Data, int64(mapPokemon.Cell), weatherLookup, mapPokemon.Timestamp, username) - storedDiskEncounter := diskEncounterCache.Get(encounterId) - if storedDiskEncounter != nil { - diskEncounter := storedDiskEncounter.Value() - diskEncounterCache.Delete(encounterId) - pokemon.updatePokemonFromDiskEncounterProto(ctx, db, diskEncounter, username) - //log.Infof("Processed stored disk encounter") - } - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, mapPokemon.Timestamp/1000) - - unlock() - } -} - -func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.ClientWeatherProto, timestampMs int64, account string) (updates []WeatherUpdate) { - hourKey := timestampMs / time.Hour.Milliseconds() - for _, weatherProto := range p { - weather, unlock, err := getOrCreateWeatherRecord(ctx, db, weatherProto.S2CellId) - if err != nil { - log.Printf("getOrCreateWeatherRecord: %s", err) - continue - } - - if weather.newRecord || timestampMs >= weather.UpdatedMs { - state := getWeatherConsensusState(weatherProto.S2CellId, hourKey) - if state != nil { - publish, publishProto := state.applyObservation(hourKey, account, weatherProto) - if publish { - if publishProto == nil { - publishProto = weatherProto - } - weather.UpdatedMs = timestampMs - weather.updateWeatherFromClientWeatherProto(publishProto) - saveWeatherRecord(ctx, db, weather) - if weather.oldValues.GameplayCondition != weather.GameplayCondition { - updates = append(updates, WeatherUpdate{ - S2CellId: publishProto.S2CellId, - NewWeather: int32(publishProto.GetGameplayWeather().GetGameplayCondition()), - }) - } - } - } - } - - unlock() - } - return updates -} - -func UpdateClientMapS2CellBatch(ctx context.Context, db db.DbDetails, cellIds []uint64) { - saveS2CellRecords(ctx, db, cellIds) -} - -func UpdateIncidentLineup(ctx context.Context, db db.DbDetails, protoReq *pogo.OpenInvasionCombatSessionProto, protoRes *pogo.OpenInvasionCombatSessionOutProto) string { - incident, unlock, err := getOrCreateIncidentRecord(ctx, db, protoReq.IncidentLookup.IncidentId, protoReq.IncidentLookup.FortId) - if err != nil { - return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) - } - defer unlock() - - if incident.newRecord { - log.Debugf("Updating lineup before it was saved: %s", protoReq.IncidentLookup.IncidentId) - } - incident.updateFromOpenInvasionCombatSessionOut(protoRes) - - saveIncidentRecord(ctx, db, incident) - return "" -} - -func ConfirmIncident(ctx context.Context, db db.DbDetails, proto *pogo.StartIncidentOutProto) string { - incident, unlock, err := getOrCreateIncidentRecord(ctx, db, proto.Incident.IncidentId, proto.Incident.FortId) - if err != nil { - return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) - } - defer unlock() - - if incident.newRecord { - log.Debugf("Confirming incident before it was saved: %s", proto.Incident.IncidentId) - } - incident.updateFromStartIncidentOut(proto) - - saveIncidentRecord(ctx, db, incident) - return "" -} - func SetWebhooksSender(whSender webhooksSenderInterface) { webhooksSender = whSender } diff --git a/decoder/player.go b/decoder/player.go index 0478ae75..01dea73f 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -9,9 +9,9 @@ import ( "golbat/db" "golbat/pogo" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) // Player struct. Name is the primary key. diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 47096a4d..cc69bdb6 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -1,29 +1,11 @@ package decoder import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" "sync" - "time" - "golbat/config" - "golbat/db" - "golbat/geo" "golbat/grpc" - "golbat/pogo" - "golbat/webhooks" - "github.com/UnownHash/gohbem" - "github.com/golang/geo/s2" - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "google.golang.org/protobuf/proto" - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" ) // Pokemon struct. @@ -480,1495 +462,3 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { } } } - -// peekPokemonRecordReadOnly acquires lock, does NOT take snapshot. -// Use for read-only checks which will not cause a backing database lookup -// Caller must use returned unlock function -func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { - if item := pokemonCache.Get(encounterId); item != nil { - pokemon := item.Value() - pokemon.Lock() - return pokemon, func() { pokemon.Unlock() }, nil - } - - return nil, nil, nil -} - -func loadPokemonFromDatabase(ctx context.Context, db db.DbDetails, encounterId uint64, pokemon *Pokemon) error { - err := db.PokemonDb.GetContext(ctx, pokemon, - "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ - "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ - "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ - "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ - "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) - statsCollector.IncDbQuery("select pokemon", err) - - return err -} - -// getPokemonRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks, but will cause a backing database lookup -// Caller MUST call returned unlock function. -func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { - // If we are in-memory only, this is identical to peek - if config.Config.PokemonMemoryOnly { - return peekPokemonRecordReadOnly(encounterId) - } - - // Check cache first - if item := pokemonCache.Get(encounterId); item != nil { - pokemon := item.Value() - pokemon.Lock() - return pokemon, func() { pokemon.Unlock() }, nil - } - - dbPokemon := Pokemon{} - err := loadPokemonFromDatabase(ctx, db, encounterId, &dbPokemon) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbPokemon.ClearDirty() - - // Atomically cache the loaded Pokemon - if another goroutine raced us, - // we'll get their Pokemon and use that instead (ensuring same mutex) - existingPokemon, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - // Only called if key doesn't exist - our Pokemon wins - pokemonRtreeUpdatePokemonOnGet(&dbPokemon) - return &dbPokemon - }) - - pokemon := existingPokemon.Value() - pokemon.Lock() - return pokemon, func() { pokemon.Unlock() }, nil -} - -// getPokemonRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Use when modifying the Pokemon. -// Caller MUST call returned unlock function. -func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { - pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) - if err != nil || pokemon == nil { - return nil, nil, err - } - pokemon.snapshotOldValues() - return pokemon, unlock, nil -} - -// getOrCreatePokemonRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { - // Create new Pokemon atomically - function only called if key doesn't exist - pokemonItem, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - return &Pokemon{Id: encounterId, newRecord: true} - }) - - pokemon := pokemonItem.Value() - pokemon.Lock() - - if config.Config.PokemonMemoryOnly { - pokemon.snapshotOldValues() - return pokemon, func() { pokemon.Unlock() }, nil - } - - if pokemon.newRecord { - // We should attempt to load from database - err := loadPokemonFromDatabase(ctx, db, encounterId, pokemon) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - pokemon.Unlock() - return nil, nil, err - } - } else { - // We loaded - pokemon.newRecord = false - pokemon.ClearDirty() - pokemonRtreeUpdatePokemonOnGet(pokemon) - } - } - - pokemon.snapshotOldValues() - return pokemon, func() { pokemon.Unlock() }, nil -} - -// hasChangesPokemon compares two Pokemon structs -// Ignored: Username, Iv, Pvp -// Float tolerance: Lat, Lon -// Null Float tolerance: Weight, Height, Capture1, Capture2, Capture3 -func hasChangesPokemon(old *Pokemon, new *Pokemon) bool { - return old.Id != new.Id || - old.PokestopId != new.PokestopId || - old.SpawnId != new.SpawnId || - old.Size != new.Size || - old.ExpireTimestamp != new.ExpireTimestamp || - old.Updated != new.Updated || - old.PokemonId != new.PokemonId || - old.Move1 != new.Move1 || - old.Move2 != new.Move2 || - old.Gender != new.Gender || - old.Cp != new.Cp || - old.AtkIv != new.AtkIv || - old.DefIv != new.DefIv || - old.StaIv != new.StaIv || - old.Form != new.Form || - old.Level != new.Level || - old.IsStrong != new.IsStrong || - old.Weather != new.Weather || - old.Costume != new.Costume || - old.FirstSeenTimestamp != new.FirstSeenTimestamp || - old.Changed != new.Changed || - old.CellId != new.CellId || - old.ExpireTimestampVerified != new.ExpireTimestampVerified || - old.DisplayPokemonId != new.DisplayPokemonId || - old.IsDitto != new.IsDitto || - old.SeenType != new.SeenType || - old.Shiny != new.Shiny || - old.IsEvent != new.IsEvent || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) || - !nullFloatAlmostEqual(old.Weight, new.Weight, floatTolerance) || - !nullFloatAlmostEqual(old.Height, new.Height, floatTolerance) || - !nullFloatAlmostEqual(old.Capture1, new.Capture1, floatTolerance) || - !nullFloatAlmostEqual(old.Capture2, new.Capture2, floatTolerance) || - !nullFloatAlmostEqual(old.Capture3, new.Capture3, floatTolerance) -} - -func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Pokemon, isEncounter, writeDB, webhook bool, now int64) { - if !pokemon.newRecord && !pokemon.IsDirty() { - return - } - - // uncomment to debug excessive writes - //if !pokemon.isNewRecord() && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { - // log.Errorf("Why are we updating this? %s", cmp.Diff(oldPokemon, pokemon, cmp.Options{ - // ignoreNearFloats, ignoreNearNullFloats, - // cmpopts.IgnoreFields(Pokemon{}, "Username", "Iv", "Pvp"), - // })) - //} - - if pokemon.FirstSeenTimestamp == 0 { - pokemon.FirstSeenTimestamp = now - } - - pokemon.Updated = null.IntFrom(now) - if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId || pokemon.oldValues.Cp != pokemon.Cp { - pokemon.Changed = now - } - - changePvpField := false - var pvpResults map[string][]gohbem.PokemonEntry - if ohbem != nil { - // Calculating PVP data - check for changes in pokemon properties that affect PVP rankings - // For new records, always calculate; for existing, check if relevant fields changed - shouldCalculatePvp := pokemon.AtkIv.Valid && (pokemon.isNewRecord() || pokemon.IsDirty()) - if shouldCalculatePvp { - pvp, err := ohbem.QueryPvPRank(int(pokemon.PokemonId), - int(pokemon.Form.ValueOrZero()), - int(pokemon.Costume.ValueOrZero()), - int(pokemon.Gender.ValueOrZero()), - int(pokemon.AtkIv.ValueOrZero()), - int(pokemon.DefIv.ValueOrZero()), - int(pokemon.StaIv.ValueOrZero()), - float64(pokemon.Level.ValueOrZero())) - - if err == nil { - pvpBytes, _ := json.Marshal(pvp) - pokemon.Pvp = null.StringFrom(string(pvpBytes)) - changePvpField = true - pvpResults = pvp - } - } - if !pokemon.AtkIv.Valid && pokemon.isNewRecord() { - pokemon.Pvp = null.NewString("", false) - changePvpField = true - } - } - - var oldSeenType string - if !pokemon.oldValues.SeenType.Valid { - oldSeenType = "n/a" - } else { - oldSeenType = pokemon.oldValues.SeenType.ValueOrZero() - } - - log.Debugf("Updating pokemon [%d] from %s->%s - newRecord: %t", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero(), pokemon.isNewRecord()) - //log.Println(cmp.Diff(oldPokemon, pokemon)) - - if writeDB && !config.Config.PokemonMemoryOnly { - if isEncounter && config.Config.PokemonInternalToDb { - unboosted, boosted, strong := pokemon.locateAllScans() - if unboosted != nil && boosted != nil { - unboosted.RemoveDittoAuxInfo() - boosted.RemoveDittoAuxInfo() - } - if strong != nil { - strong.RemoveDittoAuxInfo() - } - marshaled, err := proto.Marshal(&pokemon.internal) - if err == nil { - pokemon.GolbatInternal = marshaled - } else { - log.Errorf("[POKEMON] Failed to marshal internal data for %d, data may be lost: %s", pokemon.Id, err) - } - } - if pokemon.isNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) - } - pvpField, pvpValue := "", "" - if changePvpField { - pvpField, pvpValue = "pvp, ", ":pvp, " - } - res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("INSERT INTO pokemon (id, pokemon_id, lat, lon,"+ - "spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, move_1, move_2,"+ - "gender, form, cp, level, strong, weather, costume, weight, height, size,"+ - "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id,"+ - "expire_timestamp_verified, shiny, username, %s is_event, seen_type) "+ - "VALUES (\"%d\", :pokemon_id, :lat, :lon, :spawn_id, :expire_timestamp, :atk_iv, :def_iv, :sta_iv,"+ - ":golbat_internal, :iv, :move_1, :move_2, :gender, :form, :cp, :level, :strong, :weather, :costume,"+ - ":weight, :height, :size, :display_pokemon_id, :is_ditto, :pokestop_id, :updated,"+ - ":first_seen_timestamp, :changed, :cell_id, :expire_timestamp_verified, :shiny, :username, %s :is_event,"+ - ":seen_type)", pvpField, pokemon.Id, pvpValue), pokemon) - - statsCollector.IncDbQuery("insert pokemon", err) - if err != nil { - log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) - log.Errorf("Full structure: %+v", pokemon) - pokemonCache.Delete(pokemon.Id) - // Force reload of pokemon from database - return - } - - rows, rowsErr := res.RowsAffected() - log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) - } - pvpUpdate := "" - if changePvpField { - pvpUpdate = "pvp = :pvp, " - } - res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("UPDATE pokemon SET "+ - "pokestop_id = :pokestop_id, "+ - "spawn_id = :spawn_id, "+ - "lat = :lat, "+ - "lon = :lon, "+ - "weight = :weight, "+ - "height = :height, "+ - "size = :size, "+ - "expire_timestamp = :expire_timestamp, "+ - "updated = :updated, "+ - "pokemon_id = :pokemon_id, "+ - "move_1 = :move_1, "+ - "move_2 = :move_2, "+ - "gender = :gender, "+ - "cp = :cp, "+ - "atk_iv = :atk_iv, "+ - "def_iv = :def_iv, "+ - "sta_iv = :sta_iv, "+ - "golbat_internal = :golbat_internal,"+ - "iv = :iv,"+ - "form = :form, "+ - "level = :level, "+ - "strong = :strong, "+ - "weather = :weather, "+ - "costume = :costume, "+ - "first_seen_timestamp = :first_seen_timestamp, "+ - "changed = :changed, "+ - "cell_id = :cell_id, "+ - "expire_timestamp_verified = :expire_timestamp_verified, "+ - "display_pokemon_id = :display_pokemon_id, "+ - "is_ditto = :is_ditto, "+ - "seen_type = :seen_type, "+ - "shiny = :shiny, "+ - "username = :username, "+ - "%s"+ - "is_event = :is_event "+ - "WHERE id = \"%d\"", pvpUpdate, pokemon.Id), pokemon, - ) - statsCollector.IncDbQuery("update pokemon", err) - if err != nil { - log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) - log.Errorf("Full structure: %+v", pokemon) - pokemonCache.Delete(pokemon.Id) - // Force reload of pokemon from database - - return - } - rows, rowsErr := res.RowsAffected() - log.Debugf("Updating pokemon [%d] after update res = %d %v", pokemon.Id, rows, rowsErr) - } - } - - // Update pokemon rtree - if pokemon.isNewRecord() { - addPokemonToTree(pokemon) - } else if pokemon.Lat != pokemon.oldValues.Lat || pokemon.Lon != pokemon.oldValues.Lon { - // Position changed - update R-tree by removing from old position and adding to new - removePokemonFromTree(pokemon.Id, pokemon.oldValues.Lat, pokemon.oldValues.Lon) - addPokemonToTree(pokemon) - } - - updatePokemonLookup(pokemon, changePvpField, pvpResults) - - areas := MatchStatsGeofence(pokemon.Lat, pokemon.Lon) - if webhook { - createPokemonWebhooks(ctx, db, pokemon, areas) - } - updatePokemonStats(pokemon, areas, now) - - if dbDebugEnabled { - pokemon.changedFields = pokemon.changedFields[:0] - } - pokemon.newRecord = false // After saving, it's no longer a new record - pokemon.ClearDirty() - - pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache - - if db.UsePokemonCache { - pokemonCache.Set(pokemon.Id, pokemon, pokemon.remainingDuration(now)) - } -} - -type PokemonWebhook struct { - SpawnpointId string `json:"spawnpoint_id"` - PokestopId string `json:"pokestop_id"` - PokestopName *string `json:"pokestop_name"` - EncounterId string `json:"encounter_id"` - PokemonId int16 `json:"pokemon_id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - DisappearTime int64 `json:"disappear_time"` - DisappearTimeVerified bool `json:"disappear_time_verified"` - FirstSeen int64 `json:"first_seen"` - LastModifiedTime null.Int `json:"last_modified_time"` - Gender null.Int `json:"gender"` - Cp null.Int `json:"cp"` - Form null.Int `json:"form"` - Costume null.Int `json:"costume"` - IndividualAttack null.Int `json:"individual_attack"` - IndividualDefense null.Int `json:"individual_defense"` - IndividualStamina null.Int `json:"individual_stamina"` - PokemonLevel null.Int `json:"pokemon_level"` - Move1 null.Int `json:"move_1"` - Move2 null.Int `json:"move_2"` - Weight null.Float `json:"weight"` - Size null.Int `json:"size"` - Height null.Float `json:"height"` - Weather null.Int `json:"weather"` - Capture1 float64 `json:"capture_1"` - Capture2 float64 `json:"capture_2"` - Capture3 float64 `json:"capture_3"` - Shiny null.Bool `json:"shiny"` - Username null.String `json:"username"` - DisplayPokemonId null.Int `json:"display_pokemon_id"` - IsEvent int8 `json:"is_event"` - SeenType null.String `json:"seen_type"` - Pvp json.RawMessage `json:"pvp"` -} - -func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemon, areas []geo.AreaName) { - if pokemon.isNewRecord() || - pokemon.oldValues.PokemonId != pokemon.PokemonId || - pokemon.oldValues.Weather != pokemon.Weather || - pokemon.oldValues.Cp != pokemon.Cp { - - spawnpointId := "None" - if pokemon.SpawnId.Valid { - spawnpointId = strconv.FormatInt(pokemon.SpawnId.ValueOrZero(), 16) - } - - pokestopId := "None" - if pokemon.PokestopId.Valid { - pokestopId = pokemon.PokestopId.ValueOrZero() - } - - var pokestopName *string - if pokemon.PokestopId.Valid { - pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokemon.PokestopId.String) - name := "Unknown" - if pokestop != nil { - name = pokestop.Name.ValueOrZero() - unlock() - } - pokestopName = &name - } - - var pvp json.RawMessage - if pokemon.Pvp.Valid { - pvp = json.RawMessage(pokemon.Pvp.ValueOrZero()) - } - - pokemonHook := PokemonWebhook{ - SpawnpointId: spawnpointId, - PokestopId: pokestopId, - PokestopName: pokestopName, - EncounterId: strconv.FormatUint(pokemon.Id, 10), - PokemonId: pokemon.PokemonId, - Latitude: pokemon.Lat, - Longitude: pokemon.Lon, - DisappearTime: pokemon.ExpireTimestamp.ValueOrZero(), - DisappearTimeVerified: pokemon.ExpireTimestampVerified, - FirstSeen: pokemon.FirstSeenTimestamp, - LastModifiedTime: pokemon.Updated, - Gender: pokemon.Gender, - Cp: pokemon.Cp, - Form: pokemon.Form, - Costume: pokemon.Costume, - IndividualAttack: pokemon.AtkIv, - IndividualDefense: pokemon.DefIv, - IndividualStamina: pokemon.StaIv, - PokemonLevel: pokemon.Level, - Move1: pokemon.Move1, - Move2: pokemon.Move2, - Weight: pokemon.Weight, - Size: pokemon.Size, - Height: pokemon.Height, - Weather: pokemon.Weather, - Capture1: pokemon.Capture1.ValueOrZero(), - Capture2: pokemon.Capture2.ValueOrZero(), - Capture3: pokemon.Capture3.ValueOrZero(), - Shiny: pokemon.Shiny, - Username: pokemon.Username, - DisplayPokemonId: pokemon.DisplayPokemonId, - IsEvent: pokemon.IsEvent, - SeenType: pokemon.SeenType, - Pvp: pvp, - } - - if pokemon.AtkIv.Valid && pokemon.DefIv.Valid && pokemon.StaIv.Valid { - webhooksSender.AddMessage(webhooks.PokemonIV, pokemonHook, areas) - } else { - webhooksSender.AddMessage(webhooks.PokemonNoIV, pokemonHook, areas) - } - } -} - -func (pokemon *Pokemon) populateInternal() { - if len(pokemon.GolbatInternal) == 0 || len(pokemon.internal.ScanHistory) != 0 { - return - } - err := proto.Unmarshal(pokemon.GolbatInternal, &pokemon.internal) - if err != nil { - log.Warnf("Failed to parse internal data for %d: %s", pokemon.Id, err) - pokemon.internal.Reset() - } -} - -func (pokemon *Pokemon) locateScan(isStrong bool, isBoosted bool) (*grpc.PokemonScan, bool) { - pokemon.populateInternal() - var bestMatching *grpc.PokemonScan - for _, entry := range pokemon.internal.ScanHistory { - if entry.Strong != isStrong { - continue - } - if entry.Weather != int32(pogo.GameplayWeatherProto_NONE) == isBoosted { - return entry, true - } else { - bestMatching = entry - } - } - return bestMatching, false -} - -func (pokemon *Pokemon) locateAllScans() (unboosted, boosted, strong *grpc.PokemonScan) { - pokemon.populateInternal() - for _, entry := range pokemon.internal.ScanHistory { - if entry.Strong { - strong = entry - } else if entry.Weather != int32(pogo.GameplayWeatherProto_NONE) { - boosted = entry - } else { - unboosted = entry - } - } - return -} - -func (pokemon *Pokemon) isNewRecord() bool { - return pokemon.newRecord -} - -func (pokemon *Pokemon) remainingDuration(now int64) time.Duration { - remaining := ttlcache.DefaultTTL - if pokemon.ExpireTimestampVerified { - timeLeft := 60 + pokemon.ExpireTimestamp.ValueOrZero() - now - if timeLeft > 1 { - remaining = time.Duration(timeLeft) * time.Second - } - } - return remaining -} - -func (pokemon *Pokemon) addWildPokemon(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, timestampMs int64, trustworthyTimestamp bool) { - if wildPokemon.EncounterId != pokemon.Id { - panic("Unmatched EncounterId") - } - pokemon.SetLat(wildPokemon.Latitude) - pokemon.SetLon(wildPokemon.Longitude) - - spawnId, err := strconv.ParseInt(wildPokemon.SpawnPointId, 16, 64) - if err != nil { - panic(err) - } - pokemon.SetSpawnId(null.IntFrom(spawnId)) - - pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, trustworthyTimestamp) - pokemon.setPokemonDisplay(int16(wildPokemon.Pokemon.PokemonId), wildPokemon.Pokemon.PokemonDisplay) -} - -// wildSignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and -// should be written. -func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto, time int64) bool { - pokemonDisplay := wildPokemon.Pokemon.PokemonDisplay - // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired - - return pokemon.SeenType.ValueOrZero() == SeenType_Cell || - pokemon.SeenType.ValueOrZero() == SeenType_NearbyStop || - pokemon.PokemonId != int16(wildPokemon.Pokemon.PokemonId) || - pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || - pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || - pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || - pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) || - (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) -} - -// wildSignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and -// should be written. -func (pokemon *Pokemon) nearbySignificantUpdate(wildPokemon *pogo.NearbyPokemonProto, time int64) bool { - pokemonDisplay := wildPokemon.PokemonDisplay - // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired - - pokemonChanged := pokemon.PokemonId != int16(pokemonDisplay.DisplayId) || - pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || - pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || - pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || - pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) - - if pokemonChanged { - return true - } - - hasExpired := (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) - - if hasExpired { - return true - } - - if pokemon.SeenType.ValueOrZero() == SeenType_Cell { - return true - } - - // if it's at a nearby stop, or encounter and no other details have changed update is not worthwhile - return false -} - -func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - pokemon.SetIsEvent(0) - switch pokemon.SeenType.ValueOrZero() { - case "", SeenType_Cell, SeenType_NearbyStop: - pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) - } - pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs, true) - pokemon.recomputeCpIfNeeded(ctx, db, weather) - pokemon.SetUsername(null.StringFrom(username)) - pokemon.SetCellId(null.IntFrom(cellId)) -} - -func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapPokemon *pogo.MapPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - - if !pokemon.isNewRecord() { - // Do not ever overwrite lure details based on seeing it again in the GMO - return - } - - pokemon.SetIsEvent(0) - - pokemon.Id = mapPokemon.EncounterId - - spawnpointId := mapPokemon.SpawnpointId - - pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, spawnpointId) - if pokestop == nil { - // Unrecognised pokestop - return - } - pokemon.SetPokestopId(null.StringFrom(pokestop.Id)) - pokemon.SetLat(pokestop.Lat) - pokemon.SetLon(pokestop.Lon) - pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) - unlock() - - if mapPokemon.PokemonDisplay != nil { - pokemon.setPokemonDisplay(int16(mapPokemon.PokedexTypeId), mapPokemon.PokemonDisplay) - pokemon.recomputeCpIfNeeded(ctx, db, weather) - // The mapPokemon and nearbyPokemon GMOs don't contain actual shininess. - // shiny = mapPokemon.pokemonDisplay.shiny - } else { - log.Warnf("[POKEMON] MapPokemonProto missing PokemonDisplay for %d", pokemon.Id) - } - if !pokemon.Username.Valid { - pokemon.SetUsername(null.StringFrom(username)) - } - - if mapPokemon.ExpirationTimeMs > 0 && !pokemon.ExpireTimestampVerified { - pokemon.SetExpireTimestamp(null.IntFrom(mapPokemon.ExpirationTimeMs / 1000)) - pokemon.SetExpireTimestampVerified(true) - // if we have cached an encounter for this pokemon, update the TTL. - encounterCache.UpdateTTL(pokemon.Id, pokemon.remainingDuration(timestampMs/1000)) - } else { - pokemon.SetExpireTimestampVerified(false) - } - - pokemon.SetCellId(null.IntFrom(cellId)) -} - -func (pokemon *Pokemon) calculateIv(a int64, d int64, s int64) { - if pokemon.AtkIv.ValueOrZero() != a || pokemon.DefIv.ValueOrZero() != d || pokemon.StaIv.ValueOrZero() != s || - !pokemon.AtkIv.Valid || !pokemon.DefIv.Valid || !pokemon.StaIv.Valid { - pokemon.AtkIv = null.IntFrom(a) - pokemon.DefIv = null.IntFrom(d) - pokemon.StaIv = null.IntFrom(s) - pokemon.Iv = null.FloatFrom(float64(a+d+s) / .45) - pokemon.dirty = true - } -} - -func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, nearbyPokemon *pogo.NearbyPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - pokemon.SetIsEvent(0) - pokestopId := nearbyPokemon.FortId - pokemon.setPokemonDisplay(int16(nearbyPokemon.PokedexNumber), nearbyPokemon.PokemonDisplay) - pokemon.recomputeCpIfNeeded(ctx, db, weather) - pokemon.SetUsername(null.StringFrom(username)) - - var lat, lon float64 - overrideLatLon := pokemon.isNewRecord() - useCellLatLon := true - if pokestopId != "" { - switch pokemon.SeenType.ValueOrZero() { - case "", SeenType_Cell: - overrideLatLon = true // a better estimate is available - case SeenType_NearbyStop: - default: - return - } - pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokestopId) - if pokestop == nil { - // Unrecognised pokestop, rollback changes - overrideLatLon = pokemon.isNewRecord() - } else { - pokemon.SetSeenType(null.StringFrom(SeenType_NearbyStop)) - pokemon.SetPokestopId(null.StringFrom(pokestopId)) - lat, lon = pokestop.Lat, pokestop.Lon - useCellLatLon = false - unlock() - } - } - if useCellLatLon { - // Cell Pokemon - if !overrideLatLon && pokemon.SeenType.ValueOrZero() != SeenType_Cell { - // do not downgrade to nearby cell - return - } - - s2cell := s2.CellFromCellID(s2.CellID(cellId)) - lat = s2cell.CapBound().RectBound().Center().Lat.Degrees() - lon = s2cell.CapBound().RectBound().Center().Lng.Degrees() - - pokemon.SetSeenType(null.StringFrom(SeenType_Cell)) - } - if overrideLatLon { - pokemon.SetLat(lat) - pokemon.SetLon(lon) - } else { - midpoint := s2.LatLngFromPoint(s2.Point{s2.PointFromLatLng(s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon)). - Add(s2.PointFromLatLng(s2.LatLngFromDegrees(lat, lon)).Vector)}) - pokemon.SetLat(midpoint.Lat.Degrees()) - pokemon.SetLon(midpoint.Lng.Degrees()) - } - pokemon.SetCellId(null.IntFrom(cellId)) - pokemon.setUnknownTimestamp(timestampMs / 1000) -} - -const SeenType_Cell string = "nearby_cell" // Pokemon was seen in a cell (without accurate location) -const SeenType_NearbyStop string = "nearby_stop" // Pokemon was seen at a nearby Pokestop, location set to lon, lat of pokestop -const SeenType_Wild string = "wild" // Pokemon was seen in the wild, accurate location but with no IV details -const SeenType_Encounter string = "encounter" // Pokemon has been encountered giving exact details of current IV -const SeenType_LureWild string = "lure_wild" // Pokemon was seen at a lure -const SeenType_LureEncounter string = "lure_encounter" // Pokemon has been encountered at a lure -const SeenType_TappableEncounter string = "tappable_encounter" // Pokemon has been encountered from tappable -const SeenType_TappableLureEncounter string = "tappable_lure_encounter" // Pokemon has been encountered from a lured tappable - -// setExpireTimestampFromSpawnpoint sets the current Pokemon object ExpireTimeStamp, and ExpireTimeStampVerified from the Spawnpoint -// information held. -// db - the database connection to be used -// timestampMs - the timestamp to be used for calculations -// trustworthyTimestamp - whether this timestamp is fully trustworthy (ie comes from GMO server time) -func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db db.DbDetails, timestampMs int64, trustworthyTimestamp bool) { - if !trustworthyTimestamp && pokemon.ExpireTimestampVerified { - // If our time is not trustworthy, and we have already set a time from some other source (eg a GMO) - // don't modify it - - return - } - - spawnId := pokemon.SpawnId.ValueOrZero() - if spawnId == 0 { - return - } - - pokemon.ExpireTimestampVerified = false - spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) - if spawnPoint != nil && spawnPoint.DespawnSec.Valid { - despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) - unlock() - - date := time.Unix(timestampMs/1000, 0) - secondOfHour := date.Second() + date.Minute()*60 - - despawnOffset := despawnSecond - secondOfHour - if despawnOffset < 0 { - despawnOffset += 3600 - } - pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) - pokemon.SetExpireTimestampVerified(true) - } else { - if unlock != nil { - unlock() - } - pokemon.setUnknownTimestamp(timestampMs / 1000) - } -} - -func (pokemon *Pokemon) setUnknownTimestamp(now int64) { - if !pokemon.ExpireTimestamp.Valid { - pokemon.SetExpireTimestamp(null.IntFrom(now + 20*60)) // should be configurable, add on 20min - } else { - if pokemon.ExpireTimestamp.Int64 < now { - pokemon.SetExpireTimestamp(null.IntFrom(now + 10*60)) // should be configurable, add on 10min - } - } -} - -func checkScans(old *grpc.PokemonScan, new *grpc.PokemonScan) error { - if old == nil || old.CompressedIv() == new.CompressedIv() { - return nil - } - return errors.New(fmt.Sprintf("Unexpected IV mismatch %s != %s", old, new)) -} - -func (pokemon *Pokemon) setDittoAttributes(mode string, isDitto bool, old, new *grpc.PokemonScan) { - if isDitto { - log.Debugf("[POKEMON] %d: %s Ditto found %s -> %s", pokemon.Id, mode, old, new) - pokemon.SetIsDitto(true) - pokemon.SetDisplayPokemonId(null.IntFrom(int64(pokemon.PokemonId))) - pokemon.SetPokemonId(int16(pogo.HoloPokemonId_DITTO)) - } else { - log.Debugf("[POKEMON] %d: %s not Ditto found %s -> %s", pokemon.Id, mode, old, new) - } -} -func (pokemon *Pokemon) resetDittoAttributes(mode string, old, aux, new *grpc.PokemonScan) (*grpc.PokemonScan, error) { - log.Debugf("[POKEMON] %d: %s Ditto was reset %s (%s) -> %s", pokemon.Id, mode, old, aux, new) - pokemon.SetIsDitto(false) - pokemon.SetDisplayPokemonId(null.NewInt(0, false)) - pokemon.SetPokemonId(int16(pokemon.DisplayPokemonId.Int64)) - return new, checkScans(old, new) -} - -// As far as I'm concerned, wild Ditto only depends on species but not costume/gender/form -var dittoDisguises sync.Map - -func confirmDitto(scan *grpc.PokemonScan) { - now := time.Now() - lastSeen, exists := dittoDisguises.Swap(scan.Pokemon, now) - if exists { - log.Debugf("[DITTO] Disguise %s reseen after %s", scan, now.Sub(lastSeen.(time.Time))) - } else { - var sb strings.Builder - sb.WriteString("[DITTO] New disguise ") - sb.WriteString(scan.String()) - sb.WriteString(" found. Current disguises ") - dittoDisguises.Range(func(disguise, lastSeen interface{}) bool { - sb.WriteString(strconv.FormatInt(int64(disguise.(int32)), 10)) - sb.WriteString(" (") - sb.WriteString(now.Sub(lastSeen.(time.Time)).String()) - sb.WriteString(") ") - return true - }) - log.Info(sb.String()) - } -} - -// detectDitto returns the IV/level set that should be used for persisting to db/seen if caught. -// error is set if something unexpected happened and the scan history should be cleared. -func (pokemon *Pokemon) detectDitto(scan *grpc.PokemonScan) (*grpc.PokemonScan, error) { - unboostedScan, boostedScan, strongScan := pokemon.locateAllScans() - if scan.Strong { - if strongScan != nil { - expectedLevel := strongScan.Level - isBoosted := scan.Weather != int32(pogo.GameplayWeatherProto_NONE) - if strongScan.Weather != int32(pogo.GameplayWeatherProto_NONE) != isBoosted { - if isBoosted { - expectedLevel += 5 - } else { - expectedLevel -= 5 - } - } - if scan.Level != expectedLevel || scan.CompressedIv() != strongScan.CompressedIv() { - return scan, errors.New(fmt.Sprintf("Unexpected strong Pokemon (Ditto?), %s -> %s", - strongScan, scan)) - } - } - return scan, nil - } - - // Here comes the Ditto logic. Embrace yourself :) - // Ditto weather can be split into 4 categories: - // - 00: No weather boost - // - 0P: No weather boost but Ditto is actually boosted by partly cloudy causing seen IV to be boosted [atypical] - // - B0: Weather boosts disguise but not Ditto causing seen IV to be unboosted [atypical] - // - PP: Weather being partly cloudy boosts both disguise and Ditto - // - // We will also use 0N/BN/PN to denote a normal non-Ditto spawn with corresponding weather boosts. - // Disguise IV depends on Ditto weather boost instead, and caught Ditto is boosted only in PP state. - if pokemon.IsDitto { - var unboostedLevel int32 - if boostedScan != nil { - unboostedLevel = boostedScan.Level - 5 - } else if unboostedScan != nil { - unboostedLevel = unboostedScan.Level - } else { - pokemon.resetDittoAttributes("?", nil, nil, scan) - return scan, errors.New("Missing past scans. Ditto will be reset") - } - // If IsDitto = true, then the IV sets in history are ALWAYS confirmed - scan.Confirmed = true - switch scan.Weather { - case int32(pogo.GameplayWeatherProto_NONE): - if scan.CellWeather == int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { - switch scan.Level { - case unboostedLevel: - return pokemon.resetDittoAttributes("0N", unboostedScan, boostedScan, scan) - case unboostedLevel + 5: - // For a confirmed Ditto, we persist IV in inactive only in 0P state - // when disguise is boosted, it has same IV as Ditto - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return unboostedScan, checkScans(boostedScan, scan) - } - return scan, errors.New(fmt.Sprintf("Unexpected 0P Ditto level change, %s/%s -> %s", - unboostedScan, boostedScan, scan)) - } - return scan, checkScans(unboostedScan, scan) - case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): - return scan, checkScans(boostedScan, scan) - } - switch scan.Level { - case unboostedLevel: - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, checkScans(unboostedScan, scan) - case unboostedLevel + 5: - return pokemon.resetDittoAttributes("BN", boostedScan, unboostedScan, scan) - } - return scan, errors.New(fmt.Sprintf("Unexpected B0 Ditto level change, %s/%s -> %s", - unboostedScan, boostedScan, scan)) - } - - isBoosted := scan.Weather != int32(pogo.GameplayWeatherProto_NONE) - var matchingScan *grpc.PokemonScan - if unboostedScan != nil || boostedScan != nil { - if unboostedScan != nil && boostedScan != nil { // if we have both IVs then they must be correct - if unboostedScan.Level == scan.Level { - if isBoosted { - pokemon.setDittoAttributes(">B0", true, unboostedScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, nil - } - return scan, checkScans(unboostedScan, scan) - } else if boostedScan.Level == scan.Level { - if isBoosted { - return scan, checkScans(boostedScan, scan) - } - pokemon.setDittoAttributes(">0P", true, boostedScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return unboostedScan, nil - } - return scan, errors.New(fmt.Sprintf("Unexpected third level found %s, %s vs %s", - unboostedScan, boostedScan, scan)) - } - - levelAdjustment := int32(0) - if isBoosted { - if boostedScan != nil { - matchingScan = boostedScan - } else { - matchingScan = unboostedScan - levelAdjustment = 5 - } - } else { - if unboostedScan != nil { - matchingScan = unboostedScan - } else { - matchingScan = boostedScan - levelAdjustment = -5 - } - } - // There are 10 total possible transitions among these states, i.e. all 12 of them except for 0P <-> PP. - // A Ditto in 00/PP state is undetectable. We try to detect them in the remaining possibilities. - // Now we try to detect all 10 possible conditions where we could identify Ditto with certainty - switch scan.Level - (matchingScan.Level + levelAdjustment) { - case 0: - // the Pokémon has been encountered before, but we find an unexpected level when reencountering it => Ditto - // note that at this point the level should have been already readjusted according to the new weather boost - case 5: - switch scan.Weather { - case int32(pogo.GameplayWeatherProto_NONE): - switch matchingScan.Weather { - case int32(pogo.GameplayWeatherProto_NONE): - pokemon.setDittoAttributes("00/0N>0P", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return unboostedScan, nil - case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): - if err := checkScans(matchingScan, scan); err != nil { - return scan, err - } - pokemon.setDittoAttributes("PN>0P", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - return unboostedScan, nil - } - if err := checkScans(matchingScan, scan); err != nil { - return scan, err - } - if scan.CellWeather != int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { - if scan.MustHaveRerolled(matchingScan) { - pokemon.setDittoAttributes("B0>00/[0N]", false, matchingScan, scan) - } else { - // set Ditto as it is most likely B0>00 if species did not reroll - pokemon.setDittoAttributes("B0>[00]/0N", true, matchingScan, scan) - } - scan.Confirmed = true - } else if matchingScan.Confirmed || scan.MustBeBoosted() { - pokemon.setDittoAttributes("BN>0P", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - return unboostedScan, nil - // scan.MustBeUnboosted() need not be checked since matchingScan would not have been in B0 - } else { - // in case of BN>0P, we set Ditto to be a hidden 0P state, hoping we rediscover later - // setting 0P Ditto would also mean that we have a Ditto with unconfirmed IV which is a bad idea - if _, possible := dittoDisguises.Load(scan.Pokemon); possible { - if _, possible := dittoDisguises.Load(matchingScan.Pokemon); !possible { - // this guess is most likely to be correct except when Ditto pool just rerolled - pokemon.setDittoAttributes("BN>[0P] or B0>0N", true, matchingScan, scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return unboostedScan, nil - } - } - pokemon.setDittoAttributes("BN>0P or B0>[0N]", false, matchingScan, scan) - } - matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) - case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): - // we can never be sure if this is a Ditto or rerolling into non-Ditto - if scan.MustHaveRerolled(matchingScan) { - pokemon.setDittoAttributes("B0>PP/[PN]", false, matchingScan, scan) - } else { - pokemon.setDittoAttributes("B0>[PP]/PN", true, matchingScan, scan) - } - matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) - default: - pokemon.setDittoAttributes("B0>BN", false, matchingScan, scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) - } - return scan, nil - case -5: - switch scan.Weather { - case int32(pogo.GameplayWeatherProto_NONE): - // we can never be sure if this is a Ditto or rerolling into non-Ditto - if scan.MustHaveRerolled(matchingScan) { - pokemon.setDittoAttributes("0P>00/[0N]", false, matchingScan, scan) - } else { - pokemon.setDittoAttributes("0P>[00]/0N", true, matchingScan, scan) - } - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return scan, nil - case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): - pokemon.setDittoAttributes("0P>PN", false, matchingScan, scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - return scan, checkScans(matchingScan, scan) - } - if matchingScan.Weather != int32(pogo.GameplayWeatherProto_NONE) { - pokemon.setDittoAttributes("BN/PP/PN>B0", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, nil - } - if err := checkScans(matchingScan, scan); err != nil { - return scan, err - } - if scan.MustBeBoosted() { - pokemon.setDittoAttributes("0P>BN", false, matchingScan, scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - } else if matchingScan.Confirmed || // this covers scan.MustBeUnboosted() - matchingScan.CellWeather != int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { - pokemon.setDittoAttributes("00/0N>B0", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - scan.Confirmed = true - } else { - // same rationale as BN>0P or B0>[0N] - if _, possible := dittoDisguises.Load(scan.Pokemon); possible { - if _, possible := dittoDisguises.Load(matchingScan.Pokemon); !possible { - // this guess is most likely to be correct except when Ditto pool just rerolled - pokemon.setDittoAttributes("0N>[B0] or 0P>BN", true, matchingScan, scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, nil - } - } - pokemon.setDittoAttributes("0N>B0 or 0P>[BN]", false, matchingScan, scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - } - return scan, nil - case 10: - pokemon.setDittoAttributes("B0>0P", true, matchingScan, scan) - confirmDitto(scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return matchingScan, nil // unboostedScan is a wrong guess in this case - case -10: - pokemon.setDittoAttributes("0P>B0", true, matchingScan, scan) - confirmDitto(scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, nil - default: - return scan, errors.New(fmt.Sprintf("Unexpected level %s -> %s", matchingScan, scan)) - } - } - if isBoosted { - if scan.MustBeUnboosted() { - pokemon.setDittoAttributes("B0", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - scan.Confirmed = true - return scan, checkScans(unboostedScan, scan) - } - scan.Confirmed = scan.MustBeBoosted() - return scan, checkScans(boostedScan, scan) - } else if scan.MustBeBoosted() { - pokemon.setDittoAttributes("0P", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - return unboostedScan, checkScans(boostedScan, scan) - } - scan.Confirmed = scan.MustBeUnboosted() - return scan, checkScans(unboostedScan, scan) -} - -func (pokemon *Pokemon) clearIv(cp bool) { - if pokemon.AtkIv.Valid || pokemon.DefIv.Valid || pokemon.StaIv.Valid || pokemon.Iv.Valid { - pokemon.dirty = true - } - pokemon.AtkIv = null.NewInt(0, false) - pokemon.DefIv = null.NewInt(0, false) - pokemon.StaIv = null.NewInt(0, false) - pokemon.Iv = null.NewFloat(0, false) - if cp { - switch pokemon.SeenType.ValueOrZero() { - case SeenType_LureEncounter: - pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) - case SeenType_Encounter: - pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) - } - pokemon.SetCp(null.NewInt(0, false)) - pokemon.SetPvp(null.NewString("", false)) - } -} - -// caller should setPokemonDisplay prior to calling this -func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails, proto *pogo.PokemonProto, username string) { - pokemon.SetUsername(null.StringFrom(username)) - pokemon.SetShiny(null.BoolFrom(proto.PokemonDisplay.Shiny)) - pokemon.SetCp(null.IntFrom(int64(proto.Cp))) - pokemon.SetMove1(null.IntFrom(int64(proto.Move1))) - pokemon.SetMove2(null.IntFrom(int64(proto.Move2))) - pokemon.SetHeight(null.FloatFrom(float64(proto.HeightM))) - pokemon.SetSize(null.IntFrom(int64(proto.Size))) - pokemon.SetWeight(null.FloatFrom(float64(proto.WeightKg))) - - scan := grpc.PokemonScan{ - Weather: int32(pokemon.Weather.Int64), - Strong: pokemon.IsStrong.Bool, - Attack: proto.IndividualAttack, - Defense: proto.IndividualDefense, - Stamina: proto.IndividualStamina, - CellWeather: int32(pokemon.Weather.Int64), - Pokemon: int32(proto.PokemonId), - Costume: int32(proto.PokemonDisplay.Costume), - Gender: int32(proto.PokemonDisplay.Gender), - Form: int32(proto.PokemonDisplay.Form), - } - if scan.CellWeather == int32(pogo.GameplayWeatherProto_NONE) { - weather, unlock, err := peekWeatherRecord(weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon)) - if weather == nil || !weather.GameplayCondition.Valid { - log.Warnf("Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) - } else { - scan.CellWeather = int32(weather.GameplayCondition.Int64) - } - if unlock != nil { - unlock() - } - } - if proto.CpMultiplier < 0.734 { - scan.Level = int32((58.215688455154954*proto.CpMultiplier-2.7012478057856497)*proto.CpMultiplier + 1.3220677708486794) - } else if proto.CpMultiplier < .795 { - scan.Level = int32(171.34093607855277*proto.CpMultiplier - 94.95626666368578) - } else { - scan.Level = int32(199.99995231630976*proto.CpMultiplier - 117.55996066890287) - } - - caughtIv, err := pokemon.detectDitto(&scan) - if err != nil { - caughtIv = &scan - log.Errorf("[POKEMON] Unexpected %d: %s", pokemon.Id, err) - } - if caughtIv == nil { // this can only happen for a 0P Ditto - pokemon.SetLevel(null.IntFrom(int64(scan.Level - 5))) - pokemon.clearIv(false) - } else { - pokemon.SetLevel(null.IntFrom(int64(caughtIv.Level))) - pokemon.calculateIv(int64(caughtIv.Attack), int64(caughtIv.Defense), int64(caughtIv.Stamina)) - } - if err == nil { - newScans := make([]*grpc.PokemonScan, len(pokemon.internal.ScanHistory)+1) - entriesCount := 0 - for _, oldEntry := range pokemon.internal.ScanHistory { - if oldEntry.Strong != scan.Strong || !oldEntry.Strong && - oldEntry.Weather == int32(pogo.GameplayWeatherProto_NONE) != - (scan.Weather == int32(pogo.GameplayWeatherProto_NONE)) { - newScans[entriesCount] = oldEntry - entriesCount++ - } - } - newScans[entriesCount] = &scan - pokemon.internal.ScanHistory = newScans[:entriesCount+1] - } else { - // undo possible changes - scan.Confirmed = false - scan.Weather = int32(pokemon.Weather.Int64) - pokemon.internal.ScanHistory = make([]*grpc.PokemonScan, 1) - pokemon.internal.ScanHistory[0] = &scan - } -} - -func (pokemon *Pokemon) updatePokemonFromEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.EncounterOutProto, username string, timestampMs int64) { - pokemon.SetIsEvent(0) - pokemon.addWildPokemon(ctx, db, encounterData.Pokemon, timestampMs, false) - // tappable encounter can also be available in seen as normal encounter once tapped - if pokemon.isSeenFromTappable() { - pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) - } - pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon.Pokemon, username) - - if pokemon.CellId.Valid == false { - centerCoord := s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon) - cellID := s2.CellIDFromLatLng(centerCoord).Parent(15) - pokemon.SetCellId(null.IntFrom(int64(cellID))) - } -} - -func (pokemon *Pokemon) isSeenFromTappable() bool { - return pokemon.SeenType.ValueOrZero() != SeenType_TappableEncounter && pokemon.SeenType.ValueOrZero() != SeenType_TappableLureEncounter -} - -func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.DiskEncounterOutProto, username string) { - pokemon.SetIsEvent(0) - pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) - pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) - pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) -} - -func (pokemon *Pokemon) updatePokemonFromTappableEncounterProto(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounterData *pogo.TappableEncounterProto, username string, timestampMs int64) { - pokemon.SetIsEvent(0) - pokemon.SetLat(request.LocationHintLat) - pokemon.SetLon(request.LocationHintLng) - - if spawnpointId := request.GetLocation().GetSpawnpointId(); spawnpointId != "" { - pokemon.SetSeenType(null.StringFrom(SeenType_TappableEncounter)) - - spawnId, err := strconv.ParseInt(spawnpointId, 16, 64) - if err != nil { - panic(err) - } - - pokemon.SetSpawnId(null.IntFrom(spawnId)) - pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, false) - } else if fortId := request.GetLocation().GetFortId(); fortId != "" { - pokemon.SetSeenType(null.StringFrom(SeenType_TappableLureEncounter)) - - pokemon.SetPokestopId(null.StringFrom(fortId)) - // we don't know any despawn times from lured/fort tappables - pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) - pokemon.SetExpireTimestampVerified(false) - } - if !pokemon.Username.Valid { - pokemon.SetUsername(null.StringFrom(username)) - } - pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) - pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) -} - -func (pokemon *Pokemon) setPokemonDisplay(pokemonId int16, display *pogo.PokemonDisplayProto) { - if !pokemon.isNewRecord() { - // If we would like to support detect A/B spawn in the future, fill in more code here from Chuck - var oldId int16 - if pokemon.IsDitto { - oldId = int16(pokemon.DisplayPokemonId.ValueOrZero()) - } else { - oldId = pokemon.PokemonId - } - if oldId != pokemonId || pokemon.Form != null.IntFrom(int64(display.Form)) || - pokemon.Costume != null.IntFrom(int64(display.Costume)) || - pokemon.Gender != null.IntFrom(int64(display.Gender)) || - pokemon.IsStrong.ValueOrZero() != display.IsStrongPokemon { - log.Debugf("Pokemon %d changed from (%d,%d,%d,%d,%t) to (%d,%d,%d,%d,%t)", pokemon.Id, oldId, - pokemon.Form.ValueOrZero(), pokemon.Costume.ValueOrZero(), pokemon.Gender.ValueOrZero(), - pokemon.IsStrong.ValueOrZero(), - pokemonId, display.Form, display.Costume, display.Gender, display.IsStrongPokemon) - pokemon.SetWeight(null.NewFloat(0, false)) - pokemon.SetHeight(null.NewFloat(0, false)) - pokemon.SetSize(null.NewInt(0, false)) - pokemon.SetMove1(null.NewInt(0, false)) - pokemon.SetMove2(null.NewInt(0, false)) - pokemon.SetCp(null.NewInt(0, false)) - pokemon.SetShiny(null.NewBool(false, false)) - pokemon.SetIsDitto(false) - pokemon.SetDisplayPokemonId(null.NewInt(0, false)) - pokemon.SetPvp(null.NewString("", false)) - } - } - if pokemon.isNewRecord() || !pokemon.IsDitto { - pokemon.SetPokemonId(pokemonId) - } - pokemon.SetGender(null.IntFrom(int64(display.Gender))) - pokemon.SetForm(null.IntFrom(int64(display.Form))) - pokemon.SetCostume(null.IntFrom(int64(display.Costume))) - if !pokemon.isNewRecord() { - pokemon.repopulateIv(int64(display.WeatherBoostedCondition), display.IsStrongPokemon) - } - pokemon.SetWeather(null.IntFrom(int64(display.WeatherBoostedCondition))) - pokemon.SetIsStrong(null.BoolFrom(display.IsStrongPokemon)) -} - -func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { - var isBoosted bool - if !pokemon.IsDitto { - isBoosted = weather != int64(pogo.GameplayWeatherProto_NONE) - if isStrong == pokemon.IsStrong.ValueOrZero() && - pokemon.Weather.ValueOrZero() != int64(pogo.GameplayWeatherProto_NONE) == isBoosted { - return - } - } else if isStrong { - log.Errorf("Strong Ditto??? I can't handle this fml %d", pokemon.Id) - pokemon.clearIv(true) - return - } else { - isBoosted = weather == int64(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - // both Ditto and disguise are boosted and Ditto was not boosted: none -> boosted - // or both Ditto and disguise were boosted and Ditto is not boosted: boosted -> none - if pokemon.Weather.ValueOrZero() == int64(pogo.GameplayWeatherProto_PARTLY_CLOUDY) == isBoosted { - return - } - } - matchingScan, isBoostedMatches := pokemon.locateScan(isStrong, isBoosted) - var oldAtk, oldDef, oldSta int64 - if matchingScan == nil { - pokemon.SetLevel(null.NewInt(0, false)) - pokemon.clearIv(true) - } else { - oldLevel := pokemon.Level.ValueOrZero() - if pokemon.AtkIv.Valid { - oldAtk = pokemon.AtkIv.Int64 - oldDef = pokemon.DefIv.Int64 - oldSta = pokemon.StaIv.Int64 - } else { - oldAtk = -1 - oldDef = -1 - oldSta = -1 - } - newLevel := int64(matchingScan.Level) - if isBoostedMatches || isStrong { // strong Pokemon IV is unaffected by weather - pokemon.calculateIv(int64(matchingScan.Attack), int64(matchingScan.Defense), int64(matchingScan.Stamina)) - switch pokemon.SeenType.ValueOrZero() { - case SeenType_LureWild: - pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) - case SeenType_Wild: - pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) - } - } else { - pokemon.clearIv(true) - } - if !isBoostedMatches { - if isBoosted { - newLevel += 5 - } else { - newLevel -= 5 - } - } - pokemon.SetLevel(null.IntFrom(newLevel)) - if newLevel != oldLevel || pokemon.AtkIv.Valid && - (pokemon.AtkIv.Int64 != oldAtk || pokemon.DefIv.Int64 != oldDef || pokemon.StaIv.Int64 != oldSta) { - pokemon.SetCp(null.NewInt(0, false)) - pokemon.SetPvp(null.NewString("", false)) - } - } -} - -func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition) { - if pokemon.Cp.Valid || ohbem == nil { - return - } - var displayPokemon int - shouldOverrideIv := false - var overrideIv *grpc.PokemonScan - if pokemon.IsDitto { - displayPokemon = int(pokemon.DisplayPokemonId.Int64) - if pokemon.Weather.Int64 == int64(pogo.GameplayWeatherProto_NONE) { - cellId := weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon) - cellWeather, found := weather[cellId] - if !found { - record, unlock, err := getWeatherRecordReadOnly(ctx, db, cellId) - if err != nil || record == nil || !record.GameplayCondition.Valid { - log.Warnf("[POKEMON] Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) - } else { - log.Warnf("[POKEMON] Weather not found locally for %d at %d", pokemon.Id, cellId) - cellWeather = pogo.GameplayWeatherProto_WeatherCondition(record.GameplayCondition.Int64) - found = true - } - if unlock != nil { - unlock() - } - } - if found && cellWeather == pogo.GameplayWeatherProto_PARTLY_CLOUDY { - shouldOverrideIv = true - scan, isBoostedMatches := pokemon.locateScan(false, false) - if scan != nil && isBoostedMatches { - overrideIv = scan - } - } - } - } else { - displayPokemon = int(pokemon.PokemonId) - } - var cp int - var err error - if shouldOverrideIv { - if overrideIv == nil { - return - } - // You should see boosted IV for 0P Ditto - cp, err = ohbem.CalculateCp(displayPokemon, int(pokemon.Form.ValueOrZero()), 0, - int(overrideIv.Attack), int(overrideIv.Defense), int(overrideIv.Stamina), float64(overrideIv.Level)) - } else { - if !pokemon.AtkIv.Valid || !pokemon.Level.Valid { - return - } - cp, err = ohbem.CalculateCp(displayPokemon, int(pokemon.Form.ValueOrZero()), 0, - int(pokemon.AtkIv.Int64), int(pokemon.DefIv.Int64), int(pokemon.StaIv.Int64), - float64(pokemon.Level.Int64)) - } - if err == nil { - pokemon.SetCp(null.IntFrom(int64(cp))) - } else { - log.Warnf("Pokemon %d %d CP unset due to error %s", pokemon.Id, displayPokemon, err) - } -} - -func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, encounter *pogo.EncounterOutProto, username string, timestamp int64) string { - if encounter.Pokemon == nil { - return "No encounter" - } - - encounterId := encounter.Pokemon.EncounterId - - // Remove from pending queue - encounter arrived so no need for delayed wild update - if pokemonPendingQueue != nil { - pokemonPendingQueue.Remove(encounterId) - } - - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Errorf("Error pokemon [%d]: %s", encounterId, err) - return fmt.Sprintf("Error finding pokemon %s", err) - } - defer unlock() - - pokemon.updatePokemonFromEncounterProto(ctx, db, encounter, username, timestamp) - savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, timestamp/1000) - // updateEncounterStats() should only be called for encounters, and called - // even if we have the pokemon record already. - updateEncounterStats(pokemon) - - return fmt.Sprintf("%d %d Pokemon %d CP%d", encounter.Pokemon.EncounterId, encounterId, pokemon.PokemonId, encounter.Pokemon.Pokemon.Cp) -} - -func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDetails, encounter *pogo.DiskEncounterOutProto, username string) string { - if encounter.Pokemon == nil { - return "No encounter" - } - - encounterId := uint64(encounter.Pokemon.PokemonDisplay.DisplayId) - - pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) - if err != nil { - log.Errorf("Error pokemon [%d]: %s", encounterId, err) - return fmt.Sprintf("Error finding pokemon %s", err) - } - - if pokemon == nil || pokemon.isNewRecord() { - // No pokemon found - unlock not set when pokemon is nil - if unlock != nil { - unlock() - } - diskEncounterCache.Set(encounterId, encounter, ttlcache.DefaultTTL) - return fmt.Sprintf("%d Disk encounter without previous GMO - Pokemon stored for later", encounterId) - } - defer unlock() - - pokemon.updatePokemonFromDiskEncounterProto(ctx, db, encounter, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) - // updateEncounterStats() should only be called for encounters, and called - // even if we have the pokemon record already. - updateEncounterStats(pokemon) - - return fmt.Sprintf("%d Disk Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp) -} - -func UpdatePokemonRecordWithTappableEncounter(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounter *pogo.TappableEncounterProto, username string, timestampMs int64) string { - encounterId := request.GetEncounterId() - - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Errorf("Error pokemon [%d]: %s", encounterId, err) - return fmt.Sprintf("Error finding pokemon %s", err) - } - defer unlock() - - pokemon.updatePokemonFromTappableEncounterProto(ctx, db, request, encounter, username, timestampMs) - savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) - // updateEncounterStats() should only be called for encounters, and called - // even if we have the pokemon record already. - updateEncounterStats(pokemon) - - return fmt.Sprintf("%d Tappable Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp) -} diff --git a/decoder/pokemonRtree.go b/decoder/pokemonRtree.go index 48aadc01..16dff824 100644 --- a/decoder/pokemonRtree.go +++ b/decoder/pokemonRtree.go @@ -8,11 +8,11 @@ import ( "golbat/config" "github.com/UnownHash/gohbem" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" "github.com/puzpuzpuz/xsync/v3" log "github.com/sirupsen/logrus" "github.com/tidwall/rtree" - "gopkg.in/guregu/null.v4" ) type PokemonLookupCacheItem struct { diff --git a/decoder/pokemon_decode.go b/decoder/pokemon_decode.go new file mode 100644 index 00000000..e29de264 --- /dev/null +++ b/decoder/pokemon_decode.go @@ -0,0 +1,968 @@ +package decoder + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "golbat/db" + "golbat/grpc" + "golbat/pogo" + + "github.com/golang/geo/s2" + "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" +) + +func (pokemon *Pokemon) populateInternal() { + if len(pokemon.GolbatInternal) == 0 || len(pokemon.internal.ScanHistory) != 0 { + return + } + err := proto.Unmarshal(pokemon.GolbatInternal, &pokemon.internal) + if err != nil { + log.Warnf("Failed to parse internal data for %d: %s", pokemon.Id, err) + pokemon.internal.Reset() + } +} + +func (pokemon *Pokemon) locateScan(isStrong bool, isBoosted bool) (*grpc.PokemonScan, bool) { + pokemon.populateInternal() + var bestMatching *grpc.PokemonScan + for _, entry := range pokemon.internal.ScanHistory { + if entry.Strong != isStrong { + continue + } + if entry.Weather != int32(pogo.GameplayWeatherProto_NONE) == isBoosted { + return entry, true + } else { + bestMatching = entry + } + } + return bestMatching, false +} + +func (pokemon *Pokemon) locateAllScans() (unboosted, boosted, strong *grpc.PokemonScan) { + pokemon.populateInternal() + for _, entry := range pokemon.internal.ScanHistory { + if entry.Strong { + strong = entry + } else if entry.Weather != int32(pogo.GameplayWeatherProto_NONE) { + boosted = entry + } else { + unboosted = entry + } + } + return +} + +func (pokemon *Pokemon) isNewRecord() bool { + return pokemon.newRecord +} + +func (pokemon *Pokemon) remainingDuration(now int64) time.Duration { + remaining := ttlcache.DefaultTTL + if pokemon.ExpireTimestampVerified { + timeLeft := 60 + pokemon.ExpireTimestamp.ValueOrZero() - now + if timeLeft > 1 { + remaining = time.Duration(timeLeft) * time.Second + } + } + return remaining +} + +func (pokemon *Pokemon) addWildPokemon(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, timestampMs int64, trustworthyTimestamp bool) { + if wildPokemon.EncounterId != pokemon.Id { + panic("Unmatched EncounterId") + } + pokemon.SetLat(wildPokemon.Latitude) + pokemon.SetLon(wildPokemon.Longitude) + + spawnId, err := strconv.ParseInt(wildPokemon.SpawnPointId, 16, 64) + if err != nil { + panic(err) + } + pokemon.SetSpawnId(null.IntFrom(spawnId)) + + pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, trustworthyTimestamp) + pokemon.setPokemonDisplay(int16(wildPokemon.Pokemon.PokemonId), wildPokemon.Pokemon.PokemonDisplay) +} + +// wildSignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and +// should be written. +func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto, time int64) bool { + pokemonDisplay := wildPokemon.Pokemon.PokemonDisplay + // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired + + return pokemon.SeenType.ValueOrZero() == SeenType_Cell || + pokemon.SeenType.ValueOrZero() == SeenType_NearbyStop || + pokemon.PokemonId != int16(wildPokemon.Pokemon.PokemonId) || + pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || + pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || + pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || + pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) || + (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) +} + +// nearbySignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and +// should be written. +func (pokemon *Pokemon) nearbySignificantUpdate(wildPokemon *pogo.NearbyPokemonProto, time int64) bool { + pokemonDisplay := wildPokemon.PokemonDisplay + // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired + + pokemonChanged := pokemon.PokemonId != int16(pokemonDisplay.DisplayId) || + pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || + pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || + pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || + pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) + + if pokemonChanged { + return true + } + + hasExpired := (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) + + if hasExpired { + return true + } + + if pokemon.SeenType.ValueOrZero() == SeenType_Cell { + return true + } + + // if it's at a nearby stop, or encounter and no other details have changed update is not worthwhile + return false +} + +func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { + pokemon.SetIsEvent(0) + switch pokemon.SeenType.ValueOrZero() { + case "", SeenType_Cell, SeenType_NearbyStop: + pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) + } + pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs, true) + pokemon.recomputeCpIfNeeded(ctx, db, weather) + pokemon.SetUsername(null.StringFrom(username)) + pokemon.SetCellId(null.IntFrom(cellId)) +} + +func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapPokemon *pogo.MapPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { + + if !pokemon.isNewRecord() { + // Do not ever overwrite lure details based on seeing it again in the GMO + return + } + + pokemon.SetIsEvent(0) + + pokemon.Id = mapPokemon.EncounterId + + spawnpointId := mapPokemon.SpawnpointId + + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, spawnpointId) + if pokestop == nil { + // Unrecognised pokestop + return + } + pokemon.SetPokestopId(null.StringFrom(pokestop.Id)) + pokemon.SetLat(pokestop.Lat) + pokemon.SetLon(pokestop.Lon) + pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) + unlock() + + if mapPokemon.PokemonDisplay != nil { + pokemon.setPokemonDisplay(int16(mapPokemon.PokedexTypeId), mapPokemon.PokemonDisplay) + pokemon.recomputeCpIfNeeded(ctx, db, weather) + // The mapPokemon and nearbyPokemon GMOs don't contain actual shininess. + // shiny = mapPokemon.pokemonDisplay.shiny + } else { + log.Warnf("[POKEMON] MapPokemonProto missing PokemonDisplay for %d", pokemon.Id) + } + if !pokemon.Username.Valid { + pokemon.SetUsername(null.StringFrom(username)) + } + + if mapPokemon.ExpirationTimeMs > 0 && !pokemon.ExpireTimestampVerified { + pokemon.SetExpireTimestamp(null.IntFrom(mapPokemon.ExpirationTimeMs / 1000)) + pokemon.SetExpireTimestampVerified(true) + // if we have cached an encounter for this pokemon, update the TTL. + encounterCache.UpdateTTL(pokemon.Id, pokemon.remainingDuration(timestampMs/1000)) + } else { + pokemon.SetExpireTimestampVerified(false) + } + + pokemon.SetCellId(null.IntFrom(cellId)) +} + +func (pokemon *Pokemon) calculateIv(a int64, d int64, s int64) { + if pokemon.AtkIv.ValueOrZero() != a || pokemon.DefIv.ValueOrZero() != d || pokemon.StaIv.ValueOrZero() != s || + !pokemon.AtkIv.Valid || !pokemon.DefIv.Valid || !pokemon.StaIv.Valid { + pokemon.AtkIv = null.IntFrom(a) + pokemon.DefIv = null.IntFrom(d) + pokemon.StaIv = null.IntFrom(s) + pokemon.Iv = null.FloatFrom(float64(a+d+s) / .45) + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, nearbyPokemon *pogo.NearbyPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { + pokemon.SetIsEvent(0) + pokestopId := nearbyPokemon.FortId + pokemon.setPokemonDisplay(int16(nearbyPokemon.PokedexNumber), nearbyPokemon.PokemonDisplay) + pokemon.recomputeCpIfNeeded(ctx, db, weather) + pokemon.SetUsername(null.StringFrom(username)) + + var lat, lon float64 + overrideLatLon := pokemon.isNewRecord() + useCellLatLon := true + if pokestopId != "" { + switch pokemon.SeenType.ValueOrZero() { + case "", SeenType_Cell: + overrideLatLon = true // a better estimate is available + case SeenType_NearbyStop: + default: + return + } + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokestopId) + if pokestop == nil { + // Unrecognised pokestop, rollback changes + overrideLatLon = pokemon.isNewRecord() + } else { + pokemon.SetSeenType(null.StringFrom(SeenType_NearbyStop)) + pokemon.SetPokestopId(null.StringFrom(pokestopId)) + lat, lon = pokestop.Lat, pokestop.Lon + useCellLatLon = false + unlock() + } + } + if useCellLatLon { + // Cell Pokemon + if !overrideLatLon && pokemon.SeenType.ValueOrZero() != SeenType_Cell { + // do not downgrade to nearby cell + return + } + + s2cell := s2.CellFromCellID(s2.CellID(cellId)) + lat = s2cell.CapBound().RectBound().Center().Lat.Degrees() + lon = s2cell.CapBound().RectBound().Center().Lng.Degrees() + + pokemon.SetSeenType(null.StringFrom(SeenType_Cell)) + } + if overrideLatLon { + pokemon.SetLat(lat) + pokemon.SetLon(lon) + } else { + midpoint := s2.LatLngFromPoint(s2.Point{s2.PointFromLatLng(s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon)). + Add(s2.PointFromLatLng(s2.LatLngFromDegrees(lat, lon)).Vector)}) + pokemon.SetLat(midpoint.Lat.Degrees()) + pokemon.SetLon(midpoint.Lng.Degrees()) + } + pokemon.SetCellId(null.IntFrom(cellId)) + pokemon.setUnknownTimestamp(timestampMs / 1000) +} + +const SeenType_Cell string = "nearby_cell" // Pokemon was seen in a cell (without accurate location) +const SeenType_NearbyStop string = "nearby_stop" // Pokemon was seen at a nearby Pokestop, location set to lon, lat of pokestop +const SeenType_Wild string = "wild" // Pokemon was seen in the wild, accurate location but with no IV details +const SeenType_Encounter string = "encounter" // Pokemon has been encountered giving exact details of current IV +const SeenType_LureWild string = "lure_wild" // Pokemon was seen at a lure +const SeenType_LureEncounter string = "lure_encounter" // Pokemon has been encountered at a lure +const SeenType_TappableEncounter string = "tappable_encounter" // Pokemon has been encountered from tappable +const SeenType_TappableLureEncounter string = "tappable_lure_encounter" // Pokemon has been encountered from a lured tappable + +// setExpireTimestampFromSpawnpoint sets the current Pokemon object ExpireTimeStamp, and ExpireTimeStampVerified from the Spawnpoint +// information held. +// db - the database connection to be used +// timestampMs - the timestamp to be used for calculations +// trustworthyTimestamp - whether this timestamp is fully trustworthy (ie comes from GMO server time) +func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db db.DbDetails, timestampMs int64, trustworthyTimestamp bool) { + if !trustworthyTimestamp && pokemon.ExpireTimestampVerified { + // If our time is not trustworthy, and we have already set a time from some other source (eg a GMO) + // don't modify it + + return + } + + spawnId := pokemon.SpawnId.ValueOrZero() + if spawnId == 0 { + return + } + + pokemon.ExpireTimestampVerified = false + spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) + if spawnPoint != nil && spawnPoint.DespawnSec.Valid { + despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) + unlock() + + date := time.Unix(timestampMs/1000, 0) + secondOfHour := date.Second() + date.Minute()*60 + + despawnOffset := despawnSecond - secondOfHour + if despawnOffset < 0 { + despawnOffset += 3600 + } + pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) + pokemon.SetExpireTimestampVerified(true) + } else { + if unlock != nil { + unlock() + } + pokemon.setUnknownTimestamp(timestampMs / 1000) + } +} + +func (pokemon *Pokemon) setUnknownTimestamp(now int64) { + if !pokemon.ExpireTimestamp.Valid { + pokemon.SetExpireTimestamp(null.IntFrom(now + 20*60)) // should be configurable, add on 20min + } else { + if pokemon.ExpireTimestamp.Int64 < now { + pokemon.SetExpireTimestamp(null.IntFrom(now + 10*60)) // should be configurable, add on 10min + } + } +} + +func checkScans(old *grpc.PokemonScan, new *grpc.PokemonScan) error { + if old == nil || old.CompressedIv() == new.CompressedIv() { + return nil + } + return errors.New(fmt.Sprintf("Unexpected IV mismatch %s != %s", old, new)) +} + +func (pokemon *Pokemon) setDittoAttributes(mode string, isDitto bool, old, new *grpc.PokemonScan) { + if isDitto { + log.Debugf("[POKEMON] %d: %s Ditto found %s -> %s", pokemon.Id, mode, old, new) + pokemon.SetIsDitto(true) + pokemon.SetDisplayPokemonId(null.IntFrom(int64(pokemon.PokemonId))) + pokemon.SetPokemonId(int16(pogo.HoloPokemonId_DITTO)) + } else { + log.Debugf("[POKEMON] %d: %s not Ditto found %s -> %s", pokemon.Id, mode, old, new) + } +} +func (pokemon *Pokemon) resetDittoAttributes(mode string, old, aux, new *grpc.PokemonScan) (*grpc.PokemonScan, error) { + log.Debugf("[POKEMON] %d: %s Ditto was reset %s (%s) -> %s", pokemon.Id, mode, old, aux, new) + pokemon.SetIsDitto(false) + pokemon.SetDisplayPokemonId(null.NewInt(0, false)) + pokemon.SetPokemonId(int16(pokemon.DisplayPokemonId.Int64)) + return new, checkScans(old, new) +} + +// As far as I'm concerned, wild Ditto only depends on species but not costume/gender/form +var dittoDisguises sync.Map + +func confirmDitto(scan *grpc.PokemonScan) { + now := time.Now() + lastSeen, exists := dittoDisguises.Swap(scan.Pokemon, now) + if exists { + log.Debugf("[DITTO] Disguise %s reseen after %s", scan, now.Sub(lastSeen.(time.Time))) + } else { + var sb strings.Builder + sb.WriteString("[DITTO] New disguise ") + sb.WriteString(scan.String()) + sb.WriteString(" found. Current disguises ") + dittoDisguises.Range(func(disguise, lastSeen interface{}) bool { + sb.WriteString(strconv.FormatInt(int64(disguise.(int32)), 10)) + sb.WriteString(" (") + sb.WriteString(now.Sub(lastSeen.(time.Time)).String()) + sb.WriteString(") ") + return true + }) + log.Info(sb.String()) + } +} + +// detectDitto returns the IV/level set that should be used for persisting to db/seen if caught. +// error is set if something unexpected happened and the scan history should be cleared. +func (pokemon *Pokemon) detectDitto(scan *grpc.PokemonScan) (*grpc.PokemonScan, error) { + unboostedScan, boostedScan, strongScan := pokemon.locateAllScans() + if scan.Strong { + if strongScan != nil { + expectedLevel := strongScan.Level + isBoosted := scan.Weather != int32(pogo.GameplayWeatherProto_NONE) + if strongScan.Weather != int32(pogo.GameplayWeatherProto_NONE) != isBoosted { + if isBoosted { + expectedLevel += 5 + } else { + expectedLevel -= 5 + } + } + if scan.Level != expectedLevel || scan.CompressedIv() != strongScan.CompressedIv() { + return scan, errors.New(fmt.Sprintf("Unexpected strong Pokemon (Ditto?), %s -> %s", + strongScan, scan)) + } + } + return scan, nil + } + + // Here comes the Ditto logic. Embrace yourself :) + // Ditto weather can be split into 4 categories: + // - 00: No weather boost + // - 0P: No weather boost but Ditto is actually boosted by partly cloudy causing seen IV to be boosted [atypical] + // - B0: Weather boosts disguise but not Ditto causing seen IV to be unboosted [atypical] + // - PP: Weather being partly cloudy boosts both disguise and Ditto + // + // We will also use 0N/BN/PN to denote a normal non-Ditto spawn with corresponding weather boosts. + // Disguise IV depends on Ditto weather boost instead, and caught Ditto is boosted only in PP state. + if pokemon.IsDitto { + var unboostedLevel int32 + if boostedScan != nil { + unboostedLevel = boostedScan.Level - 5 + } else if unboostedScan != nil { + unboostedLevel = unboostedScan.Level + } else { + pokemon.resetDittoAttributes("?", nil, nil, scan) + return scan, errors.New("Missing past scans. Ditto will be reset") + } + // If IsDitto = true, then the IV sets in history are ALWAYS confirmed + scan.Confirmed = true + switch scan.Weather { + case int32(pogo.GameplayWeatherProto_NONE): + if scan.CellWeather == int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { + switch scan.Level { + case unboostedLevel: + return pokemon.resetDittoAttributes("0N", unboostedScan, boostedScan, scan) + case unboostedLevel + 5: + // For a confirmed Ditto, we persist IV in inactive only in 0P state + // when disguise is boosted, it has same IV as Ditto + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return unboostedScan, checkScans(boostedScan, scan) + } + return scan, errors.New(fmt.Sprintf("Unexpected 0P Ditto level change, %s/%s -> %s", + unboostedScan, boostedScan, scan)) + } + return scan, checkScans(unboostedScan, scan) + case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): + return scan, checkScans(boostedScan, scan) + } + switch scan.Level { + case unboostedLevel: + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, checkScans(unboostedScan, scan) + case unboostedLevel + 5: + return pokemon.resetDittoAttributes("BN", boostedScan, unboostedScan, scan) + } + return scan, errors.New(fmt.Sprintf("Unexpected B0 Ditto level change, %s/%s -> %s", + unboostedScan, boostedScan, scan)) + } + + isBoosted := scan.Weather != int32(pogo.GameplayWeatherProto_NONE) + var matchingScan *grpc.PokemonScan + if unboostedScan != nil || boostedScan != nil { + if unboostedScan != nil && boostedScan != nil { // if we have both IVs then they must be correct + if unboostedScan.Level == scan.Level { + if isBoosted { + pokemon.setDittoAttributes(">B0", true, unboostedScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, nil + } + return scan, checkScans(unboostedScan, scan) + } else if boostedScan.Level == scan.Level { + if isBoosted { + return scan, checkScans(boostedScan, scan) + } + pokemon.setDittoAttributes(">0P", true, boostedScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return unboostedScan, nil + } + return scan, errors.New(fmt.Sprintf("Unexpected third level found %s, %s vs %s", + unboostedScan, boostedScan, scan)) + } + + levelAdjustment := int32(0) + if isBoosted { + if boostedScan != nil { + matchingScan = boostedScan + } else { + matchingScan = unboostedScan + levelAdjustment = 5 + } + } else { + if unboostedScan != nil { + matchingScan = unboostedScan + } else { + matchingScan = boostedScan + levelAdjustment = -5 + } + } + // There are 10 total possible transitions among these states, i.e. all 12 of them except for 0P <-> PP. + // A Ditto in 00/PP state is undetectable. We try to detect them in the remaining possibilities. + // Now we try to detect all 10 possible conditions where we could identify Ditto with certainty + switch scan.Level - (matchingScan.Level + levelAdjustment) { + case 0: + // the Pokémon has been encountered before, but we find an unexpected level when reencountering it => Ditto + // note that at this point the level should have been already readjusted according to the new weather boost + case 5: + switch scan.Weather { + case int32(pogo.GameplayWeatherProto_NONE): + switch matchingScan.Weather { + case int32(pogo.GameplayWeatherProto_NONE): + pokemon.setDittoAttributes("00/0N>0P", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return unboostedScan, nil + case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): + if err := checkScans(matchingScan, scan); err != nil { + return scan, err + } + pokemon.setDittoAttributes("PN>0P", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + return unboostedScan, nil + } + if err := checkScans(matchingScan, scan); err != nil { + return scan, err + } + if scan.CellWeather != int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { + if scan.MustHaveRerolled(matchingScan) { + pokemon.setDittoAttributes("B0>00/[0N]", false, matchingScan, scan) + } else { + // set Ditto as it is most likely B0>00 if species did not reroll + pokemon.setDittoAttributes("B0>[00]/0N", true, matchingScan, scan) + } + scan.Confirmed = true + } else if matchingScan.Confirmed || scan.MustBeBoosted() { + pokemon.setDittoAttributes("BN>0P", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + return unboostedScan, nil + // scan.MustBeUnboosted() need not be checked since matchingScan would not have been in B0 + } else { + // in case of BN>0P, we set Ditto to be a hidden 0P state, hoping we rediscover later + // setting 0P Ditto would also mean that we have a Ditto with unconfirmed IV which is a bad idea + if _, possible := dittoDisguises.Load(scan.Pokemon); possible { + if _, possible := dittoDisguises.Load(matchingScan.Pokemon); !possible { + // this guess is most likely to be correct except when Ditto pool just rerolled + pokemon.setDittoAttributes("BN>[0P] or B0>0N", true, matchingScan, scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return unboostedScan, nil + } + } + pokemon.setDittoAttributes("BN>0P or B0>[0N]", false, matchingScan, scan) + } + matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) + case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): + // we can never be sure if this is a Ditto or rerolling into non-Ditto + if scan.MustHaveRerolled(matchingScan) { + pokemon.setDittoAttributes("B0>PP/[PN]", false, matchingScan, scan) + } else { + pokemon.setDittoAttributes("B0>[PP]/PN", true, matchingScan, scan) + } + matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) + default: + pokemon.setDittoAttributes("B0>BN", false, matchingScan, scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) + } + return scan, nil + case -5: + switch scan.Weather { + case int32(pogo.GameplayWeatherProto_NONE): + // we can never be sure if this is a Ditto or rerolling into non-Ditto + if scan.MustHaveRerolled(matchingScan) { + pokemon.setDittoAttributes("0P>00/[0N]", false, matchingScan, scan) + } else { + pokemon.setDittoAttributes("0P>[00]/0N", true, matchingScan, scan) + } + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return scan, nil + case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): + pokemon.setDittoAttributes("0P>PN", false, matchingScan, scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + return scan, checkScans(matchingScan, scan) + } + if matchingScan.Weather != int32(pogo.GameplayWeatherProto_NONE) { + pokemon.setDittoAttributes("BN/PP/PN>B0", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, nil + } + if err := checkScans(matchingScan, scan); err != nil { + return scan, err + } + if scan.MustBeBoosted() { + pokemon.setDittoAttributes("0P>BN", false, matchingScan, scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + } else if matchingScan.Confirmed || // this covers scan.MustBeUnboosted() + matchingScan.CellWeather != int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { + pokemon.setDittoAttributes("00/0N>B0", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + scan.Confirmed = true + } else { + // same rationale as BN>0P or B0>[0N] + if _, possible := dittoDisguises.Load(scan.Pokemon); possible { + if _, possible := dittoDisguises.Load(matchingScan.Pokemon); !possible { + // this guess is most likely to be correct except when Ditto pool just rerolled + pokemon.setDittoAttributes("0N>[B0] or 0P>BN", true, matchingScan, scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, nil + } + } + pokemon.setDittoAttributes("0N>B0 or 0P>[BN]", false, matchingScan, scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + } + return scan, nil + case 10: + pokemon.setDittoAttributes("B0>0P", true, matchingScan, scan) + confirmDitto(scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return matchingScan, nil // unboostedScan is a wrong guess in this case + case -10: + pokemon.setDittoAttributes("0P>B0", true, matchingScan, scan) + confirmDitto(scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, nil + default: + return scan, errors.New(fmt.Sprintf("Unexpected level %s -> %s", matchingScan, scan)) + } + } + if isBoosted { + if scan.MustBeUnboosted() { + pokemon.setDittoAttributes("B0", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + scan.Confirmed = true + return scan, checkScans(unboostedScan, scan) + } + scan.Confirmed = scan.MustBeBoosted() + return scan, checkScans(boostedScan, scan) + } else if scan.MustBeBoosted() { + pokemon.setDittoAttributes("0P", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + return unboostedScan, checkScans(boostedScan, scan) + } + scan.Confirmed = scan.MustBeUnboosted() + return scan, checkScans(unboostedScan, scan) +} + +func (pokemon *Pokemon) clearIv(cp bool) { + if pokemon.AtkIv.Valid || pokemon.DefIv.Valid || pokemon.StaIv.Valid || pokemon.Iv.Valid { + pokemon.dirty = true + } + pokemon.AtkIv = null.NewInt(0, false) + pokemon.DefIv = null.NewInt(0, false) + pokemon.StaIv = null.NewInt(0, false) + pokemon.Iv = null.NewFloat(0, false) + if cp { + switch pokemon.SeenType.ValueOrZero() { + case SeenType_LureEncounter: + pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) + case SeenType_Encounter: + pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) + } + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) + } +} + +// caller should setPokemonDisplay prior to calling this +func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails, proto *pogo.PokemonProto, username string) { + pokemon.SetUsername(null.StringFrom(username)) + pokemon.SetShiny(null.BoolFrom(proto.PokemonDisplay.Shiny)) + pokemon.SetCp(null.IntFrom(int64(proto.Cp))) + pokemon.SetMove1(null.IntFrom(int64(proto.Move1))) + pokemon.SetMove2(null.IntFrom(int64(proto.Move2))) + pokemon.SetHeight(null.FloatFrom(float64(proto.HeightM))) + pokemon.SetSize(null.IntFrom(int64(proto.Size))) + pokemon.SetWeight(null.FloatFrom(float64(proto.WeightKg))) + + scan := grpc.PokemonScan{ + Weather: int32(pokemon.Weather.Int64), + Strong: pokemon.IsStrong.Bool, + Attack: proto.IndividualAttack, + Defense: proto.IndividualDefense, + Stamina: proto.IndividualStamina, + CellWeather: int32(pokemon.Weather.Int64), + Pokemon: int32(proto.PokemonId), + Costume: int32(proto.PokemonDisplay.Costume), + Gender: int32(proto.PokemonDisplay.Gender), + Form: int32(proto.PokemonDisplay.Form), + } + if scan.CellWeather == int32(pogo.GameplayWeatherProto_NONE) { + weather, unlock, err := peekWeatherRecord(weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon)) + if weather == nil || !weather.GameplayCondition.Valid { + log.Warnf("Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) + } else { + scan.CellWeather = int32(weather.GameplayCondition.Int64) + } + if unlock != nil { + unlock() + } + } + if proto.CpMultiplier < 0.734 { + scan.Level = int32((58.215688455154954*proto.CpMultiplier-2.7012478057856497)*proto.CpMultiplier + 1.3220677708486794) + } else if proto.CpMultiplier < .795 { + scan.Level = int32(171.34093607855277*proto.CpMultiplier - 94.95626666368578) + } else { + scan.Level = int32(199.99995231630976*proto.CpMultiplier - 117.55996066890287) + } + + caughtIv, err := pokemon.detectDitto(&scan) + if err != nil { + caughtIv = &scan + log.Errorf("[POKEMON] Unexpected %d: %s", pokemon.Id, err) + } + if caughtIv == nil { // this can only happen for a 0P Ditto + pokemon.SetLevel(null.IntFrom(int64(scan.Level - 5))) + pokemon.clearIv(false) + } else { + pokemon.SetLevel(null.IntFrom(int64(caughtIv.Level))) + pokemon.calculateIv(int64(caughtIv.Attack), int64(caughtIv.Defense), int64(caughtIv.Stamina)) + } + if err == nil { + newScans := make([]*grpc.PokemonScan, len(pokemon.internal.ScanHistory)+1) + entriesCount := 0 + for _, oldEntry := range pokemon.internal.ScanHistory { + if oldEntry.Strong != scan.Strong || !oldEntry.Strong && + oldEntry.Weather == int32(pogo.GameplayWeatherProto_NONE) != + (scan.Weather == int32(pogo.GameplayWeatherProto_NONE)) { + newScans[entriesCount] = oldEntry + entriesCount++ + } + } + newScans[entriesCount] = &scan + pokemon.internal.ScanHistory = newScans[:entriesCount+1] + } else { + // undo possible changes + scan.Confirmed = false + scan.Weather = int32(pokemon.Weather.Int64) + pokemon.internal.ScanHistory = make([]*grpc.PokemonScan, 1) + pokemon.internal.ScanHistory[0] = &scan + } +} + +func (pokemon *Pokemon) updatePokemonFromEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.EncounterOutProto, username string, timestampMs int64) { + pokemon.SetIsEvent(0) + pokemon.addWildPokemon(ctx, db, encounterData.Pokemon, timestampMs, false) + // tappable encounter can also be available in seen as normal encounter once tapped + if pokemon.isSeenFromTappable() { + pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) + } + pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon.Pokemon, username) + + if pokemon.CellId.Valid == false { + centerCoord := s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon) + cellID := s2.CellIDFromLatLng(centerCoord).Parent(15) + pokemon.SetCellId(null.IntFrom(int64(cellID))) + } +} + +func (pokemon *Pokemon) isSeenFromTappable() bool { + return pokemon.SeenType.ValueOrZero() != SeenType_TappableEncounter && pokemon.SeenType.ValueOrZero() != SeenType_TappableLureEncounter +} + +func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.DiskEncounterOutProto, username string) { + pokemon.SetIsEvent(0) + pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) + pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) + pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) +} + +func (pokemon *Pokemon) updatePokemonFromTappableEncounterProto(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounterData *pogo.TappableEncounterProto, username string, timestampMs int64) { + pokemon.SetIsEvent(0) + pokemon.SetLat(request.LocationHintLat) + pokemon.SetLon(request.LocationHintLng) + + if spawnpointId := request.GetLocation().GetSpawnpointId(); spawnpointId != "" { + pokemon.SetSeenType(null.StringFrom(SeenType_TappableEncounter)) + + spawnId, err := strconv.ParseInt(spawnpointId, 16, 64) + if err != nil { + panic(err) + } + + pokemon.SetSpawnId(null.IntFrom(spawnId)) + pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, false) + } else if fortId := request.GetLocation().GetFortId(); fortId != "" { + pokemon.SetSeenType(null.StringFrom(SeenType_TappableLureEncounter)) + + pokemon.SetPokestopId(null.StringFrom(fortId)) + // we don't know any despawn times from lured/fort tappables + pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) + pokemon.SetExpireTimestampVerified(false) + } + if !pokemon.Username.Valid { + pokemon.SetUsername(null.StringFrom(username)) + } + pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) + pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) +} + +func (pokemon *Pokemon) setPokemonDisplay(pokemonId int16, display *pogo.PokemonDisplayProto) { + if !pokemon.isNewRecord() { + // If we would like to support detect A/B spawn in the future, fill in more code here from Chuck + var oldId int16 + if pokemon.IsDitto { + oldId = int16(pokemon.DisplayPokemonId.ValueOrZero()) + } else { + oldId = pokemon.PokemonId + } + if oldId != pokemonId || pokemon.Form != null.IntFrom(int64(display.Form)) || + pokemon.Costume != null.IntFrom(int64(display.Costume)) || + pokemon.Gender != null.IntFrom(int64(display.Gender)) || + pokemon.IsStrong.ValueOrZero() != display.IsStrongPokemon { + log.Debugf("Pokemon %d changed from (%d,%d,%d,%d,%t) to (%d,%d,%d,%d,%t)", pokemon.Id, oldId, + pokemon.Form.ValueOrZero(), pokemon.Costume.ValueOrZero(), pokemon.Gender.ValueOrZero(), + pokemon.IsStrong.ValueOrZero(), + pokemonId, display.Form, display.Costume, display.Gender, display.IsStrongPokemon) + pokemon.SetWeight(null.NewFloat(0, false)) + pokemon.SetHeight(null.NewFloat(0, false)) + pokemon.SetSize(null.NewInt(0, false)) + pokemon.SetMove1(null.NewInt(0, false)) + pokemon.SetMove2(null.NewInt(0, false)) + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetShiny(null.NewBool(false, false)) + pokemon.SetIsDitto(false) + pokemon.SetDisplayPokemonId(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) + } + } + if pokemon.isNewRecord() || !pokemon.IsDitto { + pokemon.SetPokemonId(pokemonId) + } + pokemon.SetGender(null.IntFrom(int64(display.Gender))) + pokemon.SetForm(null.IntFrom(int64(display.Form))) + pokemon.SetCostume(null.IntFrom(int64(display.Costume))) + if !pokemon.isNewRecord() { + pokemon.repopulateIv(int64(display.WeatherBoostedCondition), display.IsStrongPokemon) + } + pokemon.SetWeather(null.IntFrom(int64(display.WeatherBoostedCondition))) + pokemon.SetIsStrong(null.BoolFrom(display.IsStrongPokemon)) +} + +func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { + var isBoosted bool + if !pokemon.IsDitto { + isBoosted = weather != int64(pogo.GameplayWeatherProto_NONE) + if isStrong == pokemon.IsStrong.ValueOrZero() && + pokemon.Weather.ValueOrZero() != int64(pogo.GameplayWeatherProto_NONE) == isBoosted { + return + } + } else if isStrong { + log.Errorf("Strong Ditto??? I can't handle this fml %d", pokemon.Id) + pokemon.clearIv(true) + return + } else { + isBoosted = weather == int64(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + // both Ditto and disguise are boosted and Ditto was not boosted: none -> boosted + // or both Ditto and disguise were boosted and Ditto is not boosted: boosted -> none + if pokemon.Weather.ValueOrZero() == int64(pogo.GameplayWeatherProto_PARTLY_CLOUDY) == isBoosted { + return + } + } + matchingScan, isBoostedMatches := pokemon.locateScan(isStrong, isBoosted) + var oldAtk, oldDef, oldSta int64 + if matchingScan == nil { + pokemon.SetLevel(null.NewInt(0, false)) + pokemon.clearIv(true) + } else { + oldLevel := pokemon.Level.ValueOrZero() + if pokemon.AtkIv.Valid { + oldAtk = pokemon.AtkIv.Int64 + oldDef = pokemon.DefIv.Int64 + oldSta = pokemon.StaIv.Int64 + } else { + oldAtk = -1 + oldDef = -1 + oldSta = -1 + } + newLevel := int64(matchingScan.Level) + if isBoostedMatches || isStrong { // strong Pokemon IV is unaffected by weather + pokemon.calculateIv(int64(matchingScan.Attack), int64(matchingScan.Defense), int64(matchingScan.Stamina)) + switch pokemon.SeenType.ValueOrZero() { + case SeenType_LureWild: + pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) + case SeenType_Wild: + pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) + } + } else { + pokemon.clearIv(true) + } + if !isBoostedMatches { + if isBoosted { + newLevel += 5 + } else { + newLevel -= 5 + } + } + pokemon.SetLevel(null.IntFrom(newLevel)) + if newLevel != oldLevel || pokemon.AtkIv.Valid && + (pokemon.AtkIv.Int64 != oldAtk || pokemon.DefIv.Int64 != oldDef || pokemon.StaIv.Int64 != oldSta) { + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) + } + } +} + +func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition) { + if pokemon.Cp.Valid || ohbem == nil { + return + } + var displayPokemon int + shouldOverrideIv := false + var overrideIv *grpc.PokemonScan + if pokemon.IsDitto { + displayPokemon = int(pokemon.DisplayPokemonId.Int64) + if pokemon.Weather.Int64 == int64(pogo.GameplayWeatherProto_NONE) { + cellId := weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon) + cellWeather, found := weather[cellId] + if !found { + record, unlock, err := getWeatherRecordReadOnly(ctx, db, cellId) + if err != nil || record == nil || !record.GameplayCondition.Valid { + log.Warnf("[POKEMON] Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) + } else { + log.Warnf("[POKEMON] Weather not found locally for %d at %d", pokemon.Id, cellId) + cellWeather = pogo.GameplayWeatherProto_WeatherCondition(record.GameplayCondition.Int64) + found = true + } + if unlock != nil { + unlock() + } + } + if found && cellWeather == pogo.GameplayWeatherProto_PARTLY_CLOUDY { + shouldOverrideIv = true + scan, isBoostedMatches := pokemon.locateScan(false, false) + if scan != nil && isBoostedMatches { + overrideIv = scan + } + } + } + } else { + displayPokemon = int(pokemon.PokemonId) + } + var cp int + var err error + if shouldOverrideIv { + if overrideIv == nil { + return + } + // You should see boosted IV for 0P Ditto + cp, err = ohbem.CalculateCp(displayPokemon, int(pokemon.Form.ValueOrZero()), 0, + int(overrideIv.Attack), int(overrideIv.Defense), int(overrideIv.Stamina), float64(overrideIv.Level)) + } else { + if !pokemon.AtkIv.Valid || !pokemon.Level.Valid { + return + } + cp, err = ohbem.CalculateCp(displayPokemon, int(pokemon.Form.ValueOrZero()), 0, + int(pokemon.AtkIv.Int64), int(pokemon.DefIv.Int64), int(pokemon.StaIv.Int64), + float64(pokemon.Level.Int64)) + } + if err == nil { + pokemon.SetCp(null.IntFrom(int64(cp))) + } else { + log.Warnf("Pokemon %d %d CP unset due to error %s", pokemon.Id, displayPokemon, err) + } +} diff --git a/decoder/pokemon_process.go b/decoder/pokemon_process.go new file mode 100644 index 00000000..0d9585ad --- /dev/null +++ b/decoder/pokemon_process.go @@ -0,0 +1,92 @@ +package decoder + +import ( + "context" + "fmt" + "time" + + "golbat/db" + "golbat/pogo" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" +) + +func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, encounter *pogo.EncounterOutProto, username string, timestamp int64) string { + if encounter.Pokemon == nil { + return "No encounter" + } + + encounterId := encounter.Pokemon.EncounterId + + // Remove from pending queue - encounter arrived so no need for delayed wild update + if pokemonPendingQueue != nil { + pokemonPendingQueue.Remove(encounterId) + } + + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Errorf("Error pokemon [%d]: %s", encounterId, err) + return fmt.Sprintf("Error finding pokemon %s", err) + } + defer unlock() + + pokemon.updatePokemonFromEncounterProto(ctx, db, encounter, username, timestamp) + savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, timestamp/1000) + // updateEncounterStats() should only be called for encounters, and called + // even if we have the pokemon record already. + updateEncounterStats(pokemon) + + return fmt.Sprintf("%d %d Pokemon %d CP%d", encounter.Pokemon.EncounterId, encounterId, pokemon.PokemonId, encounter.Pokemon.Pokemon.Cp) +} + +func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDetails, encounter *pogo.DiskEncounterOutProto, username string) string { + if encounter.Pokemon == nil { + return "No encounter" + } + + encounterId := uint64(encounter.Pokemon.PokemonDisplay.DisplayId) + + pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) + if err != nil { + log.Errorf("Error pokemon [%d]: %s", encounterId, err) + return fmt.Sprintf("Error finding pokemon %s", err) + } + + if pokemon == nil || pokemon.isNewRecord() { + // No pokemon found - unlock not set when pokemon is nil + if unlock != nil { + unlock() + } + diskEncounterCache.Set(encounterId, encounter, ttlcache.DefaultTTL) + return fmt.Sprintf("%d Disk encounter without previous GMO - Pokemon stored for later", encounterId) + } + defer unlock() + + pokemon.updatePokemonFromDiskEncounterProto(ctx, db, encounter, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) + // updateEncounterStats() should only be called for encounters, and called + // even if we have the pokemon record already. + updateEncounterStats(pokemon) + + return fmt.Sprintf("%d Disk Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp) +} + +func UpdatePokemonRecordWithTappableEncounter(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounter *pogo.TappableEncounterProto, username string, timestampMs int64) string { + encounterId := request.GetEncounterId() + + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Errorf("Error pokemon [%d]: %s", encounterId, err) + return fmt.Sprintf("Error finding pokemon %s", err) + } + defer unlock() + + pokemon.updatePokemonFromTappableEncounterProto(ctx, db, request, encounter, username, timestampMs) + savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) + // updateEncounterStats() should only be called for encounters, and called + // even if we have the pokemon record already. + updateEncounterStats(pokemon) + + return fmt.Sprintf("%d Tappable Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp) +} diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go new file mode 100644 index 00000000..e135dbbe --- /dev/null +++ b/decoder/pokemon_state.go @@ -0,0 +1,486 @@ +package decoder + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strconv" + + "golbat/config" + "golbat/db" + "golbat/geo" + "golbat/webhooks" + + "github.com/UnownHash/gohbem" + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" +) + +// peekPokemonRecordReadOnly acquires lock, does NOT take snapshot. +// Use for read-only checks which will not cause a backing database lookup +// Caller must use returned unlock function +func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { + if item := pokemonCache.Get(encounterId); item != nil { + pokemon := item.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil + } + + return nil, nil, nil +} + +func loadPokemonFromDatabase(ctx context.Context, db db.DbDetails, encounterId uint64, pokemon *Pokemon) error { + err := db.PokemonDb.GetContext(ctx, pokemon, + "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ + "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ + "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ + "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ + "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) + statsCollector.IncDbQuery("select pokemon", err) + + return err +} + +// getPokemonRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks, but will cause a backing database lookup +// Caller MUST call returned unlock function. +func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + // If we are in-memory only, this is identical to peek + if config.Config.PokemonMemoryOnly { + return peekPokemonRecordReadOnly(encounterId) + } + + // Check cache first + if item := pokemonCache.Get(encounterId); item != nil { + pokemon := item.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil + } + + dbPokemon := Pokemon{} + err := loadPokemonFromDatabase(ctx, db, encounterId, &dbPokemon) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbPokemon.ClearDirty() + + // Atomically cache the loaded Pokemon - if another goroutine raced us, + // we'll get their Pokemon and use that instead (ensuring same mutex) + existingPokemon, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + // Only called if key doesn't exist - our Pokemon wins + pokemonRtreeUpdatePokemonOnGet(&dbPokemon) + return &dbPokemon + }) + + pokemon := existingPokemon.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil +} + +// getPokemonRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Pokemon. +// Caller MUST call returned unlock function. +func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) + if err != nil || pokemon == nil { + return nil, nil, err + } + pokemon.snapshotOldValues() + return pokemon, unlock, nil +} + +// getOrCreatePokemonRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + // Create new Pokemon atomically - function only called if key doesn't exist + pokemonItem, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + return &Pokemon{Id: encounterId, newRecord: true} + }) + + pokemon := pokemonItem.Value() + pokemon.Lock() + + if config.Config.PokemonMemoryOnly { + pokemon.snapshotOldValues() + return pokemon, func() { pokemon.Unlock() }, nil + } + + if pokemon.newRecord { + // We should attempt to load from database + err := loadPokemonFromDatabase(ctx, db, encounterId, pokemon) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + pokemon.Unlock() + return nil, nil, err + } + } else { + // We loaded + pokemon.newRecord = false + pokemon.ClearDirty() + pokemonRtreeUpdatePokemonOnGet(pokemon) + } + } + + pokemon.snapshotOldValues() + return pokemon, func() { pokemon.Unlock() }, nil +} + +// hasChangesPokemon compares two Pokemon structs +// Ignored: Username, Iv, Pvp +// Float tolerance: Lat, Lon +// Null Float tolerance: Weight, Height, Capture1, Capture2, Capture3 +func hasChangesPokemon(old *Pokemon, new *Pokemon) bool { + return old.Id != new.Id || + old.PokestopId != new.PokestopId || + old.SpawnId != new.SpawnId || + old.Size != new.Size || + old.ExpireTimestamp != new.ExpireTimestamp || + old.Updated != new.Updated || + old.PokemonId != new.PokemonId || + old.Move1 != new.Move1 || + old.Move2 != new.Move2 || + old.Gender != new.Gender || + old.Cp != new.Cp || + old.AtkIv != new.AtkIv || + old.DefIv != new.DefIv || + old.StaIv != new.StaIv || + old.Form != new.Form || + old.Level != new.Level || + old.IsStrong != new.IsStrong || + old.Weather != new.Weather || + old.Costume != new.Costume || + old.FirstSeenTimestamp != new.FirstSeenTimestamp || + old.Changed != new.Changed || + old.CellId != new.CellId || + old.ExpireTimestampVerified != new.ExpireTimestampVerified || + old.DisplayPokemonId != new.DisplayPokemonId || + old.IsDitto != new.IsDitto || + old.SeenType != new.SeenType || + old.Shiny != new.Shiny || + old.IsEvent != new.IsEvent || + !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || + !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) || + !nullFloatAlmostEqual(old.Weight, new.Weight, floatTolerance) || + !nullFloatAlmostEqual(old.Height, new.Height, floatTolerance) || + !nullFloatAlmostEqual(old.Capture1, new.Capture1, floatTolerance) || + !nullFloatAlmostEqual(old.Capture2, new.Capture2, floatTolerance) || + !nullFloatAlmostEqual(old.Capture3, new.Capture3, floatTolerance) +} + +func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Pokemon, isEncounter, writeDB, webhook bool, now int64) { + if !pokemon.newRecord && !pokemon.IsDirty() { + return + } + + // uncomment to debug excessive writes + //if !pokemon.isNewRecord() && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { + // log.Errorf("Why are we updating this? %s", cmp.Diff(oldPokemon, pokemon, cmp.Options{ + // ignoreNearFloats, ignoreNearNullFloats, + // cmpopts.IgnoreFields(Pokemon{}, "Username", "Iv", "Pvp"), + // })) + //} + + if pokemon.FirstSeenTimestamp == 0 { + pokemon.FirstSeenTimestamp = now + } + + pokemon.Updated = null.IntFrom(now) + if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId || pokemon.oldValues.Cp != pokemon.Cp { + pokemon.Changed = now + } + + changePvpField := false + var pvpResults map[string][]gohbem.PokemonEntry + if ohbem != nil { + // Calculating PVP data - check for changes in pokemon properties that affect PVP rankings + // For new records, always calculate; for existing, check if relevant fields changed + shouldCalculatePvp := pokemon.AtkIv.Valid && (pokemon.isNewRecord() || pokemon.IsDirty()) + if shouldCalculatePvp { + pvp, err := ohbem.QueryPvPRank(int(pokemon.PokemonId), + int(pokemon.Form.ValueOrZero()), + int(pokemon.Costume.ValueOrZero()), + int(pokemon.Gender.ValueOrZero()), + int(pokemon.AtkIv.ValueOrZero()), + int(pokemon.DefIv.ValueOrZero()), + int(pokemon.StaIv.ValueOrZero()), + float64(pokemon.Level.ValueOrZero())) + + if err == nil { + pvpBytes, _ := json.Marshal(pvp) + pokemon.Pvp = null.StringFrom(string(pvpBytes)) + changePvpField = true + pvpResults = pvp + } + } + if !pokemon.AtkIv.Valid && pokemon.isNewRecord() { + pokemon.Pvp = null.NewString("", false) + changePvpField = true + } + } + + var oldSeenType string + if !pokemon.oldValues.SeenType.Valid { + oldSeenType = "n/a" + } else { + oldSeenType = pokemon.oldValues.SeenType.ValueOrZero() + } + + log.Debugf("Updating pokemon [%d] from %s->%s - newRecord: %t", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero(), pokemon.isNewRecord()) + //log.Println(cmp.Diff(oldPokemon, pokemon)) + + if writeDB && !config.Config.PokemonMemoryOnly { + if isEncounter && config.Config.PokemonInternalToDb { + unboosted, boosted, strong := pokemon.locateAllScans() + if unboosted != nil && boosted != nil { + unboosted.RemoveDittoAuxInfo() + boosted.RemoveDittoAuxInfo() + } + if strong != nil { + strong.RemoveDittoAuxInfo() + } + marshaled, err := proto.Marshal(&pokemon.internal) + if err == nil { + pokemon.GolbatInternal = marshaled + } else { + log.Errorf("[POKEMON] Failed to marshal internal data for %d, data may be lost: %s", pokemon.Id, err) + } + } + if pokemon.isNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } + pvpField, pvpValue := "", "" + if changePvpField { + pvpField, pvpValue = "pvp, ", ":pvp, " + } + res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("INSERT INTO pokemon (id, pokemon_id, lat, lon,"+ + "spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, move_1, move_2,"+ + "gender, form, cp, level, strong, weather, costume, weight, height, size,"+ + "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id,"+ + "expire_timestamp_verified, shiny, username, %s is_event, seen_type) "+ + "VALUES (\"%d\", :pokemon_id, :lat, :lon, :spawn_id, :expire_timestamp, :atk_iv, :def_iv, :sta_iv,"+ + ":golbat_internal, :iv, :move_1, :move_2, :gender, :form, :cp, :level, :strong, :weather, :costume,"+ + ":weight, :height, :size, :display_pokemon_id, :is_ditto, :pokestop_id, :updated,"+ + ":first_seen_timestamp, :changed, :cell_id, :expire_timestamp_verified, :shiny, :username, %s :is_event,"+ + ":seen_type)", pvpField, pokemon.Id, pvpValue), pokemon) + + statsCollector.IncDbQuery("insert pokemon", err) + if err != nil { + log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) + log.Errorf("Full structure: %+v", pokemon) + pokemonCache.Delete(pokemon.Id) + // Force reload of pokemon from database + return + } + + rows, rowsErr := res.RowsAffected() + log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } + pvpUpdate := "" + if changePvpField { + pvpUpdate = "pvp = :pvp, " + } + res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("UPDATE pokemon SET "+ + "pokestop_id = :pokestop_id, "+ + "spawn_id = :spawn_id, "+ + "lat = :lat, "+ + "lon = :lon, "+ + "weight = :weight, "+ + "height = :height, "+ + "size = :size, "+ + "expire_timestamp = :expire_timestamp, "+ + "updated = :updated, "+ + "pokemon_id = :pokemon_id, "+ + "move_1 = :move_1, "+ + "move_2 = :move_2, "+ + "gender = :gender, "+ + "cp = :cp, "+ + "atk_iv = :atk_iv, "+ + "def_iv = :def_iv, "+ + "sta_iv = :sta_iv, "+ + "golbat_internal = :golbat_internal,"+ + "iv = :iv,"+ + "form = :form, "+ + "level = :level, "+ + "strong = :strong, "+ + "weather = :weather, "+ + "costume = :costume, "+ + "first_seen_timestamp = :first_seen_timestamp, "+ + "changed = :changed, "+ + "cell_id = :cell_id, "+ + "expire_timestamp_verified = :expire_timestamp_verified, "+ + "display_pokemon_id = :display_pokemon_id, "+ + "is_ditto = :is_ditto, "+ + "seen_type = :seen_type, "+ + "shiny = :shiny, "+ + "username = :username, "+ + "%s"+ + "is_event = :is_event "+ + "WHERE id = \"%d\"", pvpUpdate, pokemon.Id), pokemon, + ) + statsCollector.IncDbQuery("update pokemon", err) + if err != nil { + log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) + log.Errorf("Full structure: %+v", pokemon) + pokemonCache.Delete(pokemon.Id) + // Force reload of pokemon from database + + return + } + rows, rowsErr := res.RowsAffected() + log.Debugf("Updating pokemon [%d] after update res = %d %v", pokemon.Id, rows, rowsErr) + } + } + + // Update pokemon rtree + if pokemon.isNewRecord() { + addPokemonToTree(pokemon) + } else if pokemon.Lat != pokemon.oldValues.Lat || pokemon.Lon != pokemon.oldValues.Lon { + // Position changed - update R-tree by removing from old position and adding to new + removePokemonFromTree(pokemon.Id, pokemon.oldValues.Lat, pokemon.oldValues.Lon) + addPokemonToTree(pokemon) + } + + updatePokemonLookup(pokemon, changePvpField, pvpResults) + + areas := MatchStatsGeofence(pokemon.Lat, pokemon.Lon) + if webhook { + createPokemonWebhooks(ctx, db, pokemon, areas) + } + updatePokemonStats(pokemon, areas, now) + + if dbDebugEnabled { + pokemon.changedFields = pokemon.changedFields[:0] + } + pokemon.newRecord = false // After saving, it's no longer a new record + pokemon.ClearDirty() + + pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache + + if db.UsePokemonCache { + pokemonCache.Set(pokemon.Id, pokemon, pokemon.remainingDuration(now)) + } +} + +type PokemonWebhook struct { + SpawnpointId string `json:"spawnpoint_id"` + PokestopId string `json:"pokestop_id"` + PokestopName *string `json:"pokestop_name"` + EncounterId string `json:"encounter_id"` + PokemonId int16 `json:"pokemon_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + DisappearTime int64 `json:"disappear_time"` + DisappearTimeVerified bool `json:"disappear_time_verified"` + FirstSeen int64 `json:"first_seen"` + LastModifiedTime null.Int `json:"last_modified_time"` + Gender null.Int `json:"gender"` + Cp null.Int `json:"cp"` + Form null.Int `json:"form"` + Costume null.Int `json:"costume"` + IndividualAttack null.Int `json:"individual_attack"` + IndividualDefense null.Int `json:"individual_defense"` + IndividualStamina null.Int `json:"individual_stamina"` + PokemonLevel null.Int `json:"pokemon_level"` + Move1 null.Int `json:"move_1"` + Move2 null.Int `json:"move_2"` + Weight null.Float `json:"weight"` + Size null.Int `json:"size"` + Height null.Float `json:"height"` + Weather null.Int `json:"weather"` + Capture1 float64 `json:"capture_1"` + Capture2 float64 `json:"capture_2"` + Capture3 float64 `json:"capture_3"` + Shiny null.Bool `json:"shiny"` + Username null.String `json:"username"` + DisplayPokemonId null.Int `json:"display_pokemon_id"` + IsEvent int8 `json:"is_event"` + SeenType null.String `json:"seen_type"` + Pvp json.RawMessage `json:"pvp"` +} + +func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemon, areas []geo.AreaName) { + if pokemon.isNewRecord() || + pokemon.oldValues.PokemonId != pokemon.PokemonId || + pokemon.oldValues.Weather != pokemon.Weather || + pokemon.oldValues.Cp != pokemon.Cp { + + spawnpointId := "None" + if pokemon.SpawnId.Valid { + spawnpointId = strconv.FormatInt(pokemon.SpawnId.ValueOrZero(), 16) + } + + pokestopId := "None" + if pokemon.PokestopId.Valid { + pokestopId = pokemon.PokestopId.ValueOrZero() + } + + var pokestopName *string + if pokemon.PokestopId.Valid { + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokemon.PokestopId.String) + name := "Unknown" + if pokestop != nil { + name = pokestop.Name.ValueOrZero() + unlock() + } + pokestopName = &name + } + + var pvp json.RawMessage + if pokemon.Pvp.Valid { + pvp = json.RawMessage(pokemon.Pvp.ValueOrZero()) + } + + pokemonHook := PokemonWebhook{ + SpawnpointId: spawnpointId, + PokestopId: pokestopId, + PokestopName: pokestopName, + EncounterId: strconv.FormatUint(pokemon.Id, 10), + PokemonId: pokemon.PokemonId, + Latitude: pokemon.Lat, + Longitude: pokemon.Lon, + DisappearTime: pokemon.ExpireTimestamp.ValueOrZero(), + DisappearTimeVerified: pokemon.ExpireTimestampVerified, + FirstSeen: pokemon.FirstSeenTimestamp, + LastModifiedTime: pokemon.Updated, + Gender: pokemon.Gender, + Cp: pokemon.Cp, + Form: pokemon.Form, + Costume: pokemon.Costume, + IndividualAttack: pokemon.AtkIv, + IndividualDefense: pokemon.DefIv, + IndividualStamina: pokemon.StaIv, + PokemonLevel: pokemon.Level, + Move1: pokemon.Move1, + Move2: pokemon.Move2, + Weight: pokemon.Weight, + Size: pokemon.Size, + Height: pokemon.Height, + Weather: pokemon.Weather, + Capture1: pokemon.Capture1.ValueOrZero(), + Capture2: pokemon.Capture2.ValueOrZero(), + Capture3: pokemon.Capture3.ValueOrZero(), + Shiny: pokemon.Shiny, + Username: pokemon.Username, + DisplayPokemonId: pokemon.DisplayPokemonId, + IsEvent: pokemon.IsEvent, + SeenType: pokemon.SeenType, + Pvp: pvp, + } + + if pokemon.AtkIv.Valid && pokemon.DefIv.Valid && pokemon.StaIv.Valid { + webhooksSender.AddMessage(webhooks.PokemonIV, pokemonHook, areas) + } else { + webhooksSender.AddMessage(webhooks.PokemonNoIV, pokemonHook, areas) + } + } +} diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 8e25200f..cad7e262 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1,26 +1,9 @@ package decoder import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "strings" "sync" - "time" - "github.com/jellydator/ttlcache/v3" - "github.com/paulmach/orb/geojson" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" - - "golbat/config" - "golbat/db" - "golbat/pogo" - "golbat/tz" - "golbat/util" - "golbat/webhooks" + "github.com/guregu/null/v6" ) // Pokestop struct. @@ -593,997 +576,3 @@ func (p *Pokestop) SetShowcaseRankings(v null.String) { } } } - -type QuestWebhook struct { - PokestopId string `json:"pokestop_id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - PokestopName string `json:"pokestop_name"` - Type null.Int `json:"type"` - Target null.Int `json:"target"` - Template null.String `json:"template"` - Title null.String `json:"title"` - Conditions json.RawMessage `json:"conditions"` - Rewards json.RawMessage `json:"rewards"` - Updated int64 `json:"updated"` - ArScanEligible int64 `json:"ar_scan_eligible"` - PokestopUrl string `json:"pokestop_url"` - WithAr bool `json:"with_ar"` -} - -type PokestopWebhook struct { - PokestopId string `json:"pokestop_id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Name string `json:"name"` - Url string `json:"url"` - LureExpiration int64 `json:"lure_expiration"` - LastModified int64 `json:"last_modified"` - Enabled bool `json:"enabled"` - LureId int16 `json:"lure_id"` - ArScanEligible int64 `json:"ar_scan_eligible"` - PowerUpLevel int64 `json:"power_up_level"` - PowerUpPoints int64 `json:"power_up_points"` - PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` - Updated int64 `json:"updated"` - ShowcaseFocus null.String `json:"showcase_focus"` - ShowcasePokemonId null.Int `json:"showcase_pokemon_id"` - ShowcasePokemonFormId null.Int `json:"showcase_pokemon_form_id"` - ShowcasePokemonTypeId null.Int `json:"showcase_pokemon_type_id"` - ShowcaseRankingStandard null.Int `json:"showcase_ranking_standard"` - ShowcaseExpiry null.Int `json:"showcase_expiry"` - ShowcaseRankings json.RawMessage `json:"showcase_rankings"` -} - -func loadPokestopFromDatabase(ctx context.Context, db db.DbDetails, fortId string, pokestop *Pokestop) error { - err := db.GeneralDb.GetContext(ctx, pokestop, - `SELECT pokestop.id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, - pokestop.updated, quest_type, quest_timestamp, quest_target, quest_conditions, - quest_rewards, quest_template, quest_title, - alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, - alternative_quest_conditions, alternative_quest_rewards, - alternative_quest_template, alternative_quest_title, cell_id, deleted, lure_id, sponsor_id, partner_id, - ar_scan_eligible, power_up_points, power_up_level, power_up_end_timestamp, - quest_expiry, alternative_quest_expiry, description, showcase_pokemon_id, showcase_pokemon_form_id, - showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings - FROM pokestop - WHERE pokestop.id = ? `, fortId) - statsCollector.IncDbQuery("select pokestop", err) - return err -} - -// PeekPokestopRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func PeekPokestopRecord(fortId string) (*Pokestop, func(), error) { - if item := pokestopCache.Get(fortId); item != nil { - pokestop := item.Value() - pokestop.Lock() - return pokestop, func() { pokestop.Unlock() }, nil - } - return nil, nil, nil -} - -// getPokestopRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { - // Check cache first - if item := pokestopCache.Get(fortId); item != nil { - pokestop := item.Value() - pokestop.Lock() - return pokestop, func() { pokestop.Unlock() }, nil - } - - dbPokestop := Pokestop{} - err := loadPokestopFromDatabase(ctx, db, fortId, &dbPokestop) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - - dbPokestop.ClearDirty() - - // Atomically cache the loaded Pokestop - if another goroutine raced us, - // we'll get their Pokestop and use that instead (ensuring same mutex) - existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { - // Only called if key doesn't exist - our Pokestop wins - if config.Config.TestFortInMemory { - fortRtreeUpdatePokestopOnGet(&dbPokestop) - } - return &dbPokestop - }) - - pokestop := existingPokestop.Value() - pokestop.Lock() - return pokestop, func() { pokestop.Unlock() }, nil -} - -// getPokestopRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Use when modifying the Pokestop. -// Caller MUST call returned unlock function if non-nil. -func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { - pokestop, unlock, err := getPokestopRecordReadOnly(ctx, db, fortId) - if err != nil || pokestop == nil { - return nil, nil, err - } - pokestop.snapshotOldValues() - return pokestop, unlock, nil -} - -// getOrCreatePokestopRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { - // Create new Pokestop atomically - function only called if key doesn't exist - pokestopItem, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { - return &Pokestop{Id: fortId, newRecord: true} - }) - - pokestop := pokestopItem.Value() - pokestop.Lock() - - if pokestop.newRecord { - // We should attempt to load from database - err := loadPokestopFromDatabase(ctx, db, fortId, pokestop) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - pokestop.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - pokestop.newRecord = false - pokestop.ClearDirty() - if config.Config.TestFortInMemory { - fortRtreeUpdatePokestopOnGet(pokestop) - } - } - } - - pokestop.snapshotOldValues() - return pokestop, func() { pokestop.Unlock() }, nil -} - -var LureTime int64 = 1800 - -func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, cellId uint64, now int64) *Pokestop { - stop.SetId(fortData.FortId) - stop.SetLat(fortData.Latitude) - stop.SetLon(fortData.Longitude) - - stop.SetPartnerId(null.NewString(fortData.PartnerId, fortData.PartnerId != "")) - stop.SetSponsorId(null.IntFrom(int64(fortData.Sponsor))) - stop.SetEnabled(null.BoolFrom(fortData.Enabled)) - stop.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) - stop.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) - powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) - stop.SetPowerUpLevel(powerUpLevel) - stop.SetPowerUpEndTimestamp(powerUpEndTimestamp) - - // lasModifiedMs is also modified when incident happens - lastModifiedTimestamp := fortData.LastModifiedMs / 1000 - stop.SetLastModifiedTimestamp(null.IntFrom(lastModifiedTimestamp)) - - if len(fortData.ActiveFortModifier) > 0 { - lureId := int16(fortData.ActiveFortModifier[0]) - if lureId >= 501 && lureId <= 510 { - lureEnd := lastModifiedTimestamp + LureTime - oldLureEnd := stop.LureExpireTimestamp.ValueOrZero() - if stop.LureId != lureId { - stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) - stop.SetLureId(lureId) - } else { - // wait some time after lure end before a restart in case of timing issue - if now > oldLureEnd+30 { - for now > lureEnd { - lureEnd += LureTime - } - // lure needs to be restarted - stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) - } - } - } - } - - if fortData.ImageUrl != "" { - stop.SetUrl(null.StringFrom(fortData.ImageUrl)) - } - stop.SetCellId(null.IntFrom(int64(cellId))) - - if stop.Deleted { - stop.SetDeleted(false) - log.Warnf("Cleared Stop with id '%s' is found again in GMO, therefore un-deleted", stop.Id) - // Restore in fort tracker if enabled - if fortTracker != nil { - fortTracker.RestoreFort(stop.Id, cellId, false, time.Now().Unix()) - } - } - return stop -} - -func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOutProto, haveAr bool) string { - - if questProto.ChallengeQuest == nil { - log.Debugf("Received blank quest") - return "Blank quest" - } - questData := questProto.ChallengeQuest.Quest - questTitle := questProto.ChallengeQuest.QuestDisplay.Description - questType := int64(questData.QuestType) - questTarget := int64(questData.Goal.Target) - questTemplate := strings.ToLower(questData.TemplateId) - - conditions := []map[string]any{} - rewards := []map[string]any{} - - for _, conditionData := range questData.Goal.Condition { - condition := make(map[string]any) - infoData := make(map[string]any) - condition["type"] = int(conditionData.Type) - switch conditionData.Type { - case pogo.QuestConditionProto_WITH_BADGE_TYPE: - info := conditionData.GetWithBadgeType() - infoData["amount"] = info.Amount - infoData["badge_rank"] = info.BadgeRank - badgeTypeById := []int{} - for _, badge := range info.BadgeType { - badgeTypeById = append(badgeTypeById, int(badge)) - } - infoData["badge_types"] = badgeTypeById - - case pogo.QuestConditionProto_WITH_ITEM: - info := conditionData.GetWithItem() - if int(info.Item) != 0 { - infoData["item_id"] = int(info.Item) - } - case pogo.QuestConditionProto_WITH_RAID_LEVEL: - info := conditionData.GetWithRaidLevel() - raidLevelById := []int{} - for _, raidLevel := range info.RaidLevel { - raidLevelById = append(raidLevelById, int(raidLevel)) - } - infoData["raid_levels"] = raidLevelById - case pogo.QuestConditionProto_WITH_POKEMON_TYPE: - info := conditionData.GetWithPokemonType() - pokemonTypesById := []int{} - for _, t := range info.PokemonType { - pokemonTypesById = append(pokemonTypesById, int(t)) - } - infoData["pokemon_type_ids"] = pokemonTypesById - case pogo.QuestConditionProto_WITH_POKEMON_CATEGORY: - info := conditionData.GetWithPokemonCategory() - if info.CategoryName != "" { - infoData["category_name"] = info.CategoryName - } - pokemonById := []int{} - for _, pokemon := range info.PokemonIds { - pokemonById = append(pokemonById, int(pokemon)) - } - infoData["pokemon_ids"] = pokemonById - case pogo.QuestConditionProto_WITH_WIN_RAID_STATUS: - case pogo.QuestConditionProto_WITH_THROW_TYPE: - info := conditionData.GetWithThrowType() - if int(info.GetThrowType()) != 0 { // TODO: RDM has ThrowType here, ensure it is the same thing - infoData["throw_type_id"] = int(info.GetThrowType()) - } - infoData["hit"] = info.GetHit() - case pogo.QuestConditionProto_WITH_THROW_TYPE_IN_A_ROW: - info := conditionData.GetWithThrowType() - if int(info.GetThrowType()) != 0 { - infoData["throw_type_id"] = int(info.GetThrowType()) - } - infoData["hit"] = info.GetHit() - case pogo.QuestConditionProto_WITH_LOCATION: - info := conditionData.GetWithLocation() - infoData["cell_ids"] = info.S2CellId - case pogo.QuestConditionProto_WITH_DISTANCE: - info := conditionData.GetWithDistance() - infoData["distance"] = info.DistanceKm - case pogo.QuestConditionProto_WITH_POKEMON_ALIGNMENT: - info := conditionData.GetWithPokemonAlignment() - alignmentIds := []int{} - for _, alignment := range info.Alignment { - alignmentIds = append(alignmentIds, int(alignment)) - } - infoData["alignment_ids"] = alignmentIds - case pogo.QuestConditionProto_WITH_INVASION_CHARACTER: - info := conditionData.GetWithInvasionCharacter() - characterCategoryIds := []int{} - for _, characterCategory := range info.Category { - characterCategoryIds = append(characterCategoryIds, int(characterCategory)) - } - infoData["character_category_ids"] = characterCategoryIds - case pogo.QuestConditionProto_WITH_NPC_COMBAT: - info := conditionData.GetWithNpcCombat() - infoData["win"] = info.RequiresWin - infoData["template_ids"] = info.CombatNpcTrainerId - case pogo.QuestConditionProto_WITH_PLAYER_LEVEL: - info := conditionData.GetWithPlayerLevel() - infoData["level"] = info.Level - case pogo.QuestConditionProto_WITH_BUDDY: - info := conditionData.GetWithBuddy() - if info != nil { - infoData["min_buddy_level"] = int(info.MinBuddyLevel) - infoData["must_be_on_map"] = info.MustBeOnMap - } else { - infoData["min_buddy_level"] = 0 - infoData["must_be_on_map"] = false - } - case pogo.QuestConditionProto_WITH_DAILY_BUDDY_AFFECTION: - info := conditionData.GetWithDailyBuddyAffection() - infoData["min_buddy_affection_earned_today"] = info.MinBuddyAffectionEarnedToday - case pogo.QuestConditionProto_WITH_TEMP_EVO_POKEMON: - info := conditionData.GetWithTempEvoId() - tempEvoIds := []int{} - for _, evolution := range info.MegaForm { - tempEvoIds = append(tempEvoIds, int(evolution)) - } - infoData["raid_pokemon_evolutions"] = tempEvoIds - case pogo.QuestConditionProto_WITH_ITEM_TYPE: - info := conditionData.GetWithItemType() - itemTypes := []int{} - for _, itemType := range info.ItemType { - itemTypes = append(itemTypes, int(itemType)) - } - infoData["item_type_ids"] = itemTypes - case pogo.QuestConditionProto_WITH_RAID_ELAPSED_TIME: - info := conditionData.GetWithElapsedTime() - infoData["time"] = int64(info.ElapsedTimeMs) / 1000 - case pogo.QuestConditionProto_WITH_WIN_GYM_BATTLE_STATUS: - case pogo.QuestConditionProto_WITH_SUPER_EFFECTIVE_CHARGE: - case pogo.QuestConditionProto_WITH_UNIQUE_POKESTOP: - case pogo.QuestConditionProto_WITH_QUEST_CONTEXT: - case pogo.QuestConditionProto_WITH_WIN_BATTLE_STATUS: - case pogo.QuestConditionProto_WITH_CURVE_BALL: - case pogo.QuestConditionProto_WITH_NEW_FRIEND: - case pogo.QuestConditionProto_WITH_DAYS_IN_A_ROW: - case pogo.QuestConditionProto_WITH_WEATHER_BOOST: - case pogo.QuestConditionProto_WITH_DAILY_CAPTURE_BONUS: - case pogo.QuestConditionProto_WITH_DAILY_SPIN_BONUS: - case pogo.QuestConditionProto_WITH_UNIQUE_POKEMON: - case pogo.QuestConditionProto_WITH_BUDDY_INTERESTING_POI: - case pogo.QuestConditionProto_WITH_POKEMON_LEVEL: - case pogo.QuestConditionProto_WITH_SINGLE_DAY: - case pogo.QuestConditionProto_WITH_UNIQUE_POKEMON_TEAM: - case pogo.QuestConditionProto_WITH_MAX_CP: - case pogo.QuestConditionProto_WITH_LUCKY_POKEMON: - case pogo.QuestConditionProto_WITH_LEGENDARY_POKEMON: - case pogo.QuestConditionProto_WITH_GBL_RANK: - case pogo.QuestConditionProto_WITH_CATCHES_IN_A_ROW: - case pogo.QuestConditionProto_WITH_ENCOUNTER_TYPE: - case pogo.QuestConditionProto_WITH_COMBAT_TYPE: - case pogo.QuestConditionProto_WITH_GEOTARGETED_POI: - case pogo.QuestConditionProto_WITH_FRIEND_LEVEL: - case pogo.QuestConditionProto_WITH_STICKER: - case pogo.QuestConditionProto_WITH_POKEMON_CP: - case pogo.QuestConditionProto_WITH_RAID_LOCATION: - case pogo.QuestConditionProto_WITH_FRIENDS_RAID: - case pogo.QuestConditionProto_WITH_POKEMON_COSTUME: - default: - break - } - - if infoData != nil { - condition["info"] = infoData - } - conditions = append(conditions, condition) - } - - for _, rewardData := range questData.QuestRewards { - reward := make(map[string]any) - infoData := make(map[string]any) - reward["type"] = int(rewardData.Type) - switch rewardData.Type { - case pogo.QuestRewardProto_EXPERIENCE: - infoData["amount"] = rewardData.GetExp() - case pogo.QuestRewardProto_ITEM: - info := rewardData.GetItem() - infoData["amount"] = info.Amount - infoData["item_id"] = int(info.Item) - case pogo.QuestRewardProto_STARDUST: - infoData["amount"] = rewardData.GetStardust() - case pogo.QuestRewardProto_CANDY: - info := rewardData.GetCandy() - infoData["amount"] = info.Amount - infoData["pokemon_id"] = int(info.PokemonId) - case pogo.QuestRewardProto_XL_CANDY: - info := rewardData.GetXlCandy() - infoData["amount"] = info.Amount - infoData["pokemon_id"] = int(info.PokemonId) - case pogo.QuestRewardProto_POKEMON_ENCOUNTER: - info := rewardData.GetPokemonEncounter() - if info.IsHiddenDitto { - infoData["pokemon_id"] = 132 - infoData["pokemon_id_display"] = int(info.GetPokemonId()) - } else { - infoData["pokemon_id"] = int(info.GetPokemonId()) - } - if info.ShinyProbability > 0.0 { - infoData["shiny_probability"] = info.ShinyProbability - } - if display := info.PokemonDisplay; display != nil { - if costumeId := int(display.Costume); costumeId != 0 { - infoData["costume_id"] = costumeId - } - if formId := int(display.Form); formId != 0 { - infoData["form_id"] = formId - } - if genderId := int(display.Gender); genderId != 0 { - infoData["gender_id"] = genderId - } - if display.Shiny { - infoData["shiny"] = display.Shiny - } - if background := util.ExtractBackgroundFromDisplay(display); background != nil { - infoData["background"] = background - } - if breadMode := int(display.BreadModeEnum); breadMode != 0 { - infoData["bread_mode"] = breadMode - } - } else { - - } - case pogo.QuestRewardProto_POKECOIN: - infoData["amount"] = rewardData.GetPokecoin() - case pogo.QuestRewardProto_STICKER: - info := rewardData.GetSticker() - infoData["amount"] = info.Amount - infoData["sticker_id"] = info.StickerId - case pogo.QuestRewardProto_MEGA_RESOURCE: - info := rewardData.GetMegaResource() - infoData["amount"] = info.Amount - infoData["pokemon_id"] = int(info.PokemonId) - case pogo.QuestRewardProto_AVATAR_CLOTHING: - case pogo.QuestRewardProto_QUEST: - case pogo.QuestRewardProto_LEVEL_CAP: - case pogo.QuestRewardProto_INCIDENT: - case pogo.QuestRewardProto_PLAYER_ATTRIBUTE: - default: - break - - } - reward["info"] = infoData - rewards = append(rewards, reward) - } - - questConditions, _ := json.Marshal(conditions) - questRewards, _ := json.Marshal(rewards) - questTimestamp := time.Now().Unix() - - questExpiry := null.NewInt(0, false) - - stopTimezone := tz.SearchTimezone(stop.Lat, stop.Lon) - if stopTimezone != "" { - loc, err := time.LoadLocation(stopTimezone) - if err != nil { - log.Warnf("Unrecognised time zone %s at %f,%f", stopTimezone, stop.Lat, stop.Lon) - } else { - year, month, day := time.Now().In(loc).Date() - t := time.Date(year, month, day, 0, 0, 0, 0, loc).AddDate(0, 0, 1) - unixTime := t.Unix() - questExpiry = null.IntFrom(unixTime) - } - } - - if questExpiry.Valid == false { - questExpiry = null.IntFrom(time.Now().Unix() + 24*60*60) // Set expiry to 24 hours from now - } - - if !haveAr { - stop.SetAlternativeQuestType(null.IntFrom(questType)) - stop.SetAlternativeQuestTarget(null.IntFrom(questTarget)) - stop.SetAlternativeQuestTemplate(null.StringFrom(questTemplate)) - stop.SetAlternativeQuestTitle(null.StringFrom(questTitle)) - stop.SetAlternativeQuestConditions(null.StringFrom(string(questConditions))) - stop.SetAlternativeQuestRewards(null.StringFrom(string(questRewards))) - stop.SetAlternativeQuestTimestamp(null.IntFrom(questTimestamp)) - stop.SetAlternativeQuestExpiry(questExpiry) - } else { - stop.SetQuestType(null.IntFrom(questType)) - stop.SetQuestTarget(null.IntFrom(questTarget)) - stop.SetQuestTemplate(null.StringFrom(questTemplate)) - stop.SetQuestTitle(null.StringFrom(questTitle)) - stop.SetQuestConditions(null.StringFrom(string(questConditions))) - stop.SetQuestRewards(null.StringFrom(string(questRewards))) - stop.SetQuestTimestamp(null.IntFrom(questTimestamp)) - stop.SetQuestExpiry(questExpiry) - } - - return questTitle -} - -func (stop *Pokestop) updatePokestopFromFortDetailsProto(fortData *pogo.FortDetailsOutProto) *Pokestop { - stop.SetId(fortData.Id) - stop.SetLat(fortData.Latitude) - stop.SetLon(fortData.Longitude) - if len(fortData.ImageUrl) > 0 { - stop.SetUrl(null.StringFrom(fortData.ImageUrl[0])) - } - stop.SetName(null.StringFrom(fortData.Name)) - - if fortData.Description == "" { - stop.SetDescription(null.NewString("", false)) - } else { - stop.SetDescription(null.StringFrom(fortData.Description)) - } - - if fortData.Modifier != nil && len(fortData.Modifier) > 0 { - // DeployingPlayerCodename contains the name of the player if we want that - lureId := int16(fortData.Modifier[0].ModifierType) - lureExpiry := fortData.Modifier[0].ExpirationTimeMs / 1000 - - stop.SetLureId(lureId) - stop.SetLureExpireTimestamp(null.IntFrom(lureExpiry)) - } - - return stop -} - -func (stop *Pokestop) updatePokestopFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto) *Pokestop { - stop.SetId(fortData.Id) - stop.SetLat(fortData.Latitude) - stop.SetLon(fortData.Longitude) - - if len(fortData.Image) > 0 { - stop.SetUrl(null.StringFrom(fortData.Image[0].Url)) - } - stop.SetName(null.StringFrom(fortData.Name)) - if stop.Deleted { - log.Debugf("Cleared Stop with id '%s' is found again in GMF, therefore kept deleted", stop.Id) - } - return stop -} - -func (stop *Pokestop) updatePokestopFromGetContestDataOutProto(contest *pogo.ContestProto) { - stop.SetShowcaseRankingStandard(null.IntFrom(int64(contest.GetMetric().GetRankingStandard()))) - stop.SetShowcaseExpiry(null.IntFrom(contest.GetSchedule().GetContestCycle().GetEndTimeMs() / 1000)) - - focusStore := createFocusStoreFromContestProto(contest) - - if len(focusStore) > 1 { - log.Warnf("SHOWCASE: we got more than one showcase focus: %v", focusStore) - } - - for key, focus := range focusStore { - focus["type"] = key - jsonBytes, err := json.Marshal(focus) - if err != nil { - log.Errorf("SHOWCASE: Stop '%s' - Focus '%v' marshalling failed: %s", stop.Id, focus, err) - } - stop.SetShowcaseFocus(null.StringFrom(string(jsonBytes))) - // still support old format - probably still required to filter in external tools - stop.extractShowcasePokemonInfoDeprecated(key, focus) - } -} - -func (stop *Pokestop) updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) { - type contestEntry struct { - Rank int `json:"rank"` - Score float64 `json:"score"` - PokemonId int `json:"pokemon_id"` - Form int `json:"form"` - Costume int `json:"costume"` - Gender int `json:"gender"` - Shiny bool `json:"shiny"` - TempEvolution int `json:"temp_evolution"` - TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms"` - Alignment int `json:"alignment"` - Badge int `json:"badge"` - Background *int64 `json:"background,omitempty"` - } - type contestJson struct { - TotalEntries int `json:"total_entries"` - LastUpdate int64 `json:"last_update"` - ContestEntries []contestEntry `json:"contest_entries"` - } - - j := contestJson{LastUpdate: time.Now().Unix()} - j.TotalEntries = int(contestData.TotalEntries) - - for _, entry := range contestData.GetContestEntries() { - rank := entry.GetRank() - if rank > 3 { - break - } - j.ContestEntries = append(j.ContestEntries, contestEntry{ - Rank: int(rank), - Score: entry.GetScore(), - PokemonId: int(entry.GetPokedexId()), - Form: int(entry.GetPokemonDisplay().Form), - Costume: int(entry.GetPokemonDisplay().Costume), - Gender: int(entry.GetPokemonDisplay().Gender), - Shiny: entry.GetPokemonDisplay().Shiny, - TempEvolution: int(entry.GetPokemonDisplay().CurrentTempEvolution), - TempEvolutionFinishMs: entry.GetPokemonDisplay().TemporaryEvolutionFinishMs, - Alignment: int(entry.GetPokemonDisplay().Alignment), - Badge: int(entry.GetPokemonDisplay().PokemonBadge), - Background: util.ExtractBackgroundFromDisplay(entry.PokemonDisplay), - }) - - } - jsonString, _ := json.Marshal(j) - stop.SetShowcaseRankings(null.StringFrom(string(jsonString))) -} - -func createPokestopFortWebhooks(stop *Pokestop) { - fort := InitWebHookFortFromPokestop(stop) - if stop.newRecord { - CreateFortWebHooks(nil, fort, NEW) - } else { - // Build old fort from saved old values - oldFort := &FortWebhook{ - Type: POKESTOP.String(), - Id: stop.Id, - Name: stop.oldValues.Name.Ptr(), - ImageUrl: stop.oldValues.Url.Ptr(), - Description: stop.oldValues.Description.Ptr(), - Location: Location{Latitude: stop.oldValues.Lat, Longitude: stop.oldValues.Lon}, - } - CreateFortWebHooks(oldFort, fort, EDIT) - } -} - -func createPokestopWebhooks(stop *Pokestop) { - - areas := MatchStatsGeofence(stop.Lat, stop.Lon) - - pokestopName := "Unknown" - if stop.Name.Valid { - pokestopName = stop.Name.String - } - - if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldValues.AlternativeQuestType) { - questHook := QuestWebhook{ - PokestopId: stop.Id, - Latitude: stop.Lat, - Longitude: stop.Lon, - PokestopName: pokestopName, - Type: stop.AlternativeQuestType, - Target: stop.AlternativeQuestTarget, - Template: stop.AlternativeQuestTemplate, - Title: stop.AlternativeQuestTitle, - Conditions: json.RawMessage(stop.AlternativeQuestConditions.ValueOrZero()), - Rewards: json.RawMessage(stop.AlternativeQuestRewards.ValueOrZero()), - Updated: stop.Updated, - ArScanEligible: stop.ArScanEligible.ValueOrZero(), - PokestopUrl: stop.Url.ValueOrZero(), - WithAr: false, - } - webhooksSender.AddMessage(webhooks.Quest, questHook, areas) - } - - if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldValues.QuestType) { - questHook := QuestWebhook{ - PokestopId: stop.Id, - Latitude: stop.Lat, - Longitude: stop.Lon, - PokestopName: pokestopName, - Type: stop.QuestType, - Target: stop.QuestTarget, - Template: stop.QuestTemplate, - Title: stop.QuestTitle, - Conditions: json.RawMessage(stop.QuestConditions.ValueOrZero()), - Rewards: json.RawMessage(stop.QuestRewards.ValueOrZero()), - Updated: stop.Updated, - ArScanEligible: stop.ArScanEligible.ValueOrZero(), - PokestopUrl: stop.Url.ValueOrZero(), - WithAr: true, - } - webhooksSender.AddMessage(webhooks.Quest, questHook, areas) - } - if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldValues.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldValues.PowerUpEndTimestamp)) { - var showcaseRankings json.RawMessage - if stop.ShowcaseRankings.Valid { - showcaseRankings = json.RawMessage(stop.ShowcaseRankings.ValueOrZero()) - } - - pokestopHook := PokestopWebhook{ - PokestopId: stop.Id, - Latitude: stop.Lat, - Longitude: stop.Lon, - Name: pokestopName, - Url: stop.Url.ValueOrZero(), - LureExpiration: stop.LureExpireTimestamp.ValueOrZero(), - LastModified: stop.LastModifiedTimestamp.ValueOrZero(), - Enabled: stop.Enabled.ValueOrZero(), - LureId: stop.LureId, - ArScanEligible: stop.ArScanEligible.ValueOrZero(), - PowerUpLevel: stop.PowerUpLevel.ValueOrZero(), - PowerUpPoints: stop.PowerUpPoints.ValueOrZero(), - PowerUpEndTimestamp: stop.PowerUpEndTimestamp.ValueOrZero(), - Updated: stop.Updated, - ShowcaseFocus: stop.ShowcaseFocus, - ShowcasePokemonId: stop.ShowcasePokemon, - ShowcasePokemonFormId: stop.ShowcasePokemonForm, - ShowcasePokemonTypeId: stop.ShowcasePokemonType, - ShowcaseRankingStandard: stop.ShowcaseRankingStandard, - ShowcaseExpiry: stop.ShowcaseExpiry, - ShowcaseRankings: showcaseRankings, - } - - webhooksSender.AddMessage(webhooks.Pokestop, pokestopHook, areas) - } -} - -func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { - now := time.Now().Unix() - if !pokestop.IsNewRecord() && !pokestop.IsDirty() { - // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. - if pokestop.Updated > now-GetUpdateThreshold(900) { - // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again - return - } - } - pokestop.Updated = now - - if pokestop.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, ` - INSERT INTO pokestop ( - id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, - quest_timestamp, quest_target, quest_conditions, quest_rewards, quest_template, quest_title, - alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, - alternative_quest_conditions, alternative_quest_rewards, alternative_quest_template, - alternative_quest_title, cell_id, lure_id, sponsor_id, partner_id, ar_scan_eligible, - power_up_points, power_up_level, power_up_end_timestamp, updated, first_seen_timestamp, - quest_expiry, alternative_quest_expiry, description, showcase_focus, showcase_pokemon_id, - showcase_pokemon_form_id, showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings - ) - VALUES ( - :id, :lat, :lon, :name, :url, :enabled, :lure_expire_timestamp, :last_modified_timestamp, :quest_type, - :quest_timestamp, :quest_target, :quest_conditions, :quest_rewards, :quest_template, :quest_title, - :alternative_quest_type, :alternative_quest_timestamp, :alternative_quest_target, - :alternative_quest_conditions, :alternative_quest_rewards, :alternative_quest_template, - :alternative_quest_title, :cell_id, :lure_id, :sponsor_id, :partner_id, :ar_scan_eligible, - :power_up_points, :power_up_level, :power_up_end_timestamp, - UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), - :quest_expiry, :alternative_quest_expiry, :description, :showcase_focus, :showcase_pokemon_id, - :showcase_pokemon_form_id, :showcase_pokemon_type_id, :showcase_ranking_standard, :showcase_expiry, :showcase_rankings)`, - pokestop) - - statsCollector.IncDbQuery("insert pokestop", err) - //log.Debugf("Insert pokestop %s %+v", pokestop.Id, pokestop) - if err != nil { - log.Errorf("insert pokestop: %s", err) - return - } - - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, ` - UPDATE pokestop SET - lat = :lat, - lon = :lon, - name = :name, - url = :url, - enabled = :enabled, - lure_expire_timestamp = :lure_expire_timestamp, - last_modified_timestamp = :last_modified_timestamp, - updated = :updated, - quest_type = :quest_type, - quest_timestamp = :quest_timestamp, - quest_target = :quest_target, - quest_conditions = :quest_conditions, - quest_rewards = :quest_rewards, - quest_template = :quest_template, - quest_title = :quest_title, - alternative_quest_type = :alternative_quest_type, - alternative_quest_timestamp = :alternative_quest_timestamp, - alternative_quest_target = :alternative_quest_target, - alternative_quest_conditions = :alternative_quest_conditions, - alternative_quest_rewards = :alternative_quest_rewards, - alternative_quest_template = :alternative_quest_template, - alternative_quest_title = :alternative_quest_title, - cell_id = :cell_id, - lure_id = :lure_id, - deleted = :deleted, - sponsor_id = :sponsor_id, - partner_id = :partner_id, - ar_scan_eligible = :ar_scan_eligible, - power_up_points = :power_up_points, - power_up_level = :power_up_level, - power_up_end_timestamp = :power_up_end_timestamp, - quest_expiry = :quest_expiry, - alternative_quest_expiry = :alternative_quest_expiry, - description = :description, - showcase_focus = :showcase_focus, - showcase_pokemon_id = :showcase_pokemon_id, - showcase_pokemon_form_id = :showcase_pokemon_form_id, - showcase_pokemon_type_id = :showcase_pokemon_type_id, - showcase_ranking_standard = :showcase_ranking_standard, - showcase_expiry = :showcase_expiry, - showcase_rankings = :showcase_rankings - WHERE id = :id`, - pokestop, - ) - statsCollector.IncDbQuery("update pokestop", err) - //log.Debugf("Update pokestop %s %+v", pokestop.Id, pokestop) - if err != nil { - log.Errorf("update pokestop %s: %s", pokestop.Id, err) - return - } - _ = res - } - //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) - if dbDebugEnabled { - pokestop.changedFields = pokestop.changedFields[:0] - } - - createPokestopWebhooks(pokestop) - createPokestopFortWebhooks(pokestop) - if pokestop.IsNewRecord() { - pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) - pokestop.newRecord = false - } - pokestop.ClearDirty() - -} - -func updatePokestopGetMapFortCache(pokestop *Pokestop) { - storedGetMapFort := getMapFortsCache.Get(pokestop.Id) - if storedGetMapFort != nil { - getMapFort := storedGetMapFort.Value() - getMapFortsCache.Delete(pokestop.Id) - pokestop.updatePokestopFromGetMapFortsOutProto(getMapFort) - log.Debugf("Updated Gym using stored getMapFort: %s", pokestop.Id) - } -} - -func UpdatePokestopRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { - pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fort.Id) - if err != nil { - log.Printf("Update pokestop %s", err) - return fmt.Sprintf("Error %s", err) - } - defer unlock() - - pokestop.updatePokestopFromFortDetailsProto(fort) - - updatePokestopGetMapFortCache(pokestop) - savePokestopRecord(ctx, db, pokestop) - return fmt.Sprintf("%s %s", fort.Id, fort.Name) -} - -func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.FortSearchOutProto, haveAr bool) string { - haveArStr := "NoAR" - if haveAr { - haveArStr = "AR" - } - - if quest.ChallengeQuest == nil { - statsCollector.IncDecodeQuest("error", "no_quest") - return fmt.Sprintf("%s %s Blank quest", quest.FortId, haveArStr) - } - - statsCollector.IncDecodeQuest("ok", haveArStr) - - pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, quest.FortId) - if err != nil { - log.Printf("Update quest %s", err) - return fmt.Sprintf("error %s", err) - } - defer unlock() - - questTitle := pokestop.updatePokestopFromQuestProto(quest, haveAr) - - updatePokestopGetMapFortCache(pokestop) - savePokestopRecord(ctx, db, pokestop) - - areas := MatchStatsGeofence(pokestop.Lat, pokestop.Lon) - updateQuestStats(pokestop, haveAr, areas) - - return fmt.Sprintf("%s %s %s", quest.FortId, haveArStr, questTitle) -} - -func ClearQuestsWithinGeofence(ctx context.Context, dbDetails db.DbDetails, geofence *geojson.Feature) { - started := time.Now() - rows, err := db.RemoveQuests(ctx, dbDetails, geofence) - if err != nil { - log.Errorf("ClearQuest: Error removing quests: %s", err) - return - } - ClearPokestopCache() - log.Infof("ClearQuest: Removed quests from %d pokestops in %s", rows, time.Since(started)) -} - -func GetQuestStatusWithGeofence(dbDetails db.DbDetails, geofence *geojson.Feature) db.QuestStatus { - res, err := db.GetQuestStatus(dbDetails, geofence) - if err != nil { - log.Errorf("QuestStatus: Error retrieving quests: %s", err) - return db.QuestStatus{} - } - return res -} - -func UpdatePokestopRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { - pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, mapFort.Id) - if err != nil { - log.Printf("Update pokestop %s", err) - return false, fmt.Sprintf("Error %s", err) - } - - if pokestop == nil { - return false, "" - } - defer unlock() - - pokestop.updatePokestopFromGetMapFortsOutProto(mapFort) - savePokestopRecord(ctx, db, pokestop) - return true, fmt.Sprintf("%s %s", mapFort.Id, mapFort.Name) -} - -func GetPokestopPositions(details db.DbDetails, geofence *geojson.Feature) ([]db.QuestLocation, error) { - return db.GetPokestopPositions(details, geofence) -} - -func UpdatePokestopWithContestData(ctx context.Context, db db.DbDetails, request *pogo.GetContestDataProto, contestData *pogo.GetContestDataOutProto) string { - if contestData.ContestIncident == nil || len(contestData.ContestIncident.Contests) == 0 { - return "No contests found" - } - - var fortId string - if request != nil { - fortId = request.FortId - } else { - fortId = getFortIdFromContest(contestData.ContestIncident.Contests[0].ContestId) - } - - if fortId == "" { - return "No fortId found" - } - - if len(contestData.ContestIncident.Contests) > 1 { - log.Errorf("More than one contest found") - return fmt.Sprintf("More than one contest found in %s", fortId) - } - - contest := contestData.ContestIncident.Contests[0] - - pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) - if err != nil { - log.Printf("Get pokestop %s", err) - return "Error getting pokestop" - } - - if pokestop == nil { - log.Infof("Contest data for pokestop %s not found", fortId) - return fmt.Sprintf("Contest data for pokestop %s not found", fortId) - } - defer unlock() - - pokestop.updatePokestopFromGetContestDataOutProto(contest) - savePokestopRecord(ctx, db, pokestop) - - return fmt.Sprintf("Contest %s", fortId) -} - -func getFortIdFromContest(id string) string { - return strings.Split(id, "-")[0] -} - -func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDetails, request *pogo.GetPokemonSizeLeaderboardEntryProto, contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) string { - fortId := getFortIdFromContest(request.GetContestId()) - - pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) - if err != nil { - log.Printf("Get pokestop %s", err) - return "Error getting pokestop" - } - - if pokestop == nil { - log.Infof("Contest data for pokestop %s not found", fortId) - return fmt.Sprintf("Contest data for pokestop %s not found", fortId) - } - defer unlock() - - pokestop.updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData) - savePokestopRecord(ctx, db, pokestop) - - return fmt.Sprintf("Contest Detail %s", fortId) -} diff --git a/decoder/pokestop_decode.go b/decoder/pokestop_decode.go new file mode 100644 index 00000000..87941c73 --- /dev/null +++ b/decoder/pokestop_decode.go @@ -0,0 +1,475 @@ +package decoder + +import ( + "encoding/json" + "strings" + "time" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" + "golbat/tz" + "golbat/util" +) + +var LureTime int64 = 1800 + +func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, cellId uint64, now int64) *Pokestop { + stop.SetId(fortData.FortId) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) + + stop.SetPartnerId(null.NewString(fortData.PartnerId, fortData.PartnerId != "")) + stop.SetSponsorId(null.IntFrom(int64(fortData.Sponsor))) + stop.SetEnabled(null.BoolFrom(fortData.Enabled)) + stop.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) + stop.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) + powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) + stop.SetPowerUpLevel(powerUpLevel) + stop.SetPowerUpEndTimestamp(powerUpEndTimestamp) + + // lasModifiedMs is also modified when incident happens + lastModifiedTimestamp := fortData.LastModifiedMs / 1000 + stop.SetLastModifiedTimestamp(null.IntFrom(lastModifiedTimestamp)) + + if len(fortData.ActiveFortModifier) > 0 { + lureId := int16(fortData.ActiveFortModifier[0]) + if lureId >= 501 && lureId <= 510 { + lureEnd := lastModifiedTimestamp + LureTime + oldLureEnd := stop.LureExpireTimestamp.ValueOrZero() + if stop.LureId != lureId { + stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) + stop.SetLureId(lureId) + } else { + // wait some time after lure end before a restart in case of timing issue + if now > oldLureEnd+30 { + for now > lureEnd { + lureEnd += LureTime + } + // lure needs to be restarted + stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) + } + } + } + } + + if fortData.ImageUrl != "" { + stop.SetUrl(null.StringFrom(fortData.ImageUrl)) + } + stop.SetCellId(null.IntFrom(int64(cellId))) + + if stop.Deleted { + stop.SetDeleted(false) + log.Warnf("Cleared Stop with id '%s' is found again in GMO, therefore un-deleted", stop.Id) + // Restore in fort tracker if enabled + if fortTracker != nil { + fortTracker.RestoreFort(stop.Id, cellId, false, time.Now().Unix()) + } + } + return stop +} + +func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOutProto, haveAr bool) string { + + if questProto.ChallengeQuest == nil { + log.Debugf("Received blank quest") + return "Blank quest" + } + questData := questProto.ChallengeQuest.Quest + questTitle := questProto.ChallengeQuest.QuestDisplay.Description + questType := int64(questData.QuestType) + questTarget := int64(questData.Goal.Target) + questTemplate := strings.ToLower(questData.TemplateId) + + conditions := []map[string]any{} + rewards := []map[string]any{} + + for _, conditionData := range questData.Goal.Condition { + condition := make(map[string]any) + infoData := make(map[string]any) + condition["type"] = int(conditionData.Type) + switch conditionData.Type { + case pogo.QuestConditionProto_WITH_BADGE_TYPE: + info := conditionData.GetWithBadgeType() + infoData["amount"] = info.Amount + infoData["badge_rank"] = info.BadgeRank + badgeTypeById := []int{} + for _, badge := range info.BadgeType { + badgeTypeById = append(badgeTypeById, int(badge)) + } + infoData["badge_types"] = badgeTypeById + + case pogo.QuestConditionProto_WITH_ITEM: + info := conditionData.GetWithItem() + if int(info.Item) != 0 { + infoData["item_id"] = int(info.Item) + } + case pogo.QuestConditionProto_WITH_RAID_LEVEL: + info := conditionData.GetWithRaidLevel() + raidLevelById := []int{} + for _, raidLevel := range info.RaidLevel { + raidLevelById = append(raidLevelById, int(raidLevel)) + } + infoData["raid_levels"] = raidLevelById + case pogo.QuestConditionProto_WITH_POKEMON_TYPE: + info := conditionData.GetWithPokemonType() + pokemonTypesById := []int{} + for _, t := range info.PokemonType { + pokemonTypesById = append(pokemonTypesById, int(t)) + } + infoData["pokemon_type_ids"] = pokemonTypesById + case pogo.QuestConditionProto_WITH_POKEMON_CATEGORY: + info := conditionData.GetWithPokemonCategory() + if info.CategoryName != "" { + infoData["category_name"] = info.CategoryName + } + pokemonById := []int{} + for _, pokemon := range info.PokemonIds { + pokemonById = append(pokemonById, int(pokemon)) + } + infoData["pokemon_ids"] = pokemonById + case pogo.QuestConditionProto_WITH_WIN_RAID_STATUS: + case pogo.QuestConditionProto_WITH_THROW_TYPE: + info := conditionData.GetWithThrowType() + if int(info.GetThrowType()) != 0 { // TODO: RDM has ThrowType here, ensure it is the same thing + infoData["throw_type_id"] = int(info.GetThrowType()) + } + infoData["hit"] = info.GetHit() + case pogo.QuestConditionProto_WITH_THROW_TYPE_IN_A_ROW: + info := conditionData.GetWithThrowType() + if int(info.GetThrowType()) != 0 { + infoData["throw_type_id"] = int(info.GetThrowType()) + } + infoData["hit"] = info.GetHit() + case pogo.QuestConditionProto_WITH_LOCATION: + info := conditionData.GetWithLocation() + infoData["cell_ids"] = info.S2CellId + case pogo.QuestConditionProto_WITH_DISTANCE: + info := conditionData.GetWithDistance() + infoData["distance"] = info.DistanceKm + case pogo.QuestConditionProto_WITH_POKEMON_ALIGNMENT: + info := conditionData.GetWithPokemonAlignment() + alignmentIds := []int{} + for _, alignment := range info.Alignment { + alignmentIds = append(alignmentIds, int(alignment)) + } + infoData["alignment_ids"] = alignmentIds + case pogo.QuestConditionProto_WITH_INVASION_CHARACTER: + info := conditionData.GetWithInvasionCharacter() + characterCategoryIds := []int{} + for _, characterCategory := range info.Category { + characterCategoryIds = append(characterCategoryIds, int(characterCategory)) + } + infoData["character_category_ids"] = characterCategoryIds + case pogo.QuestConditionProto_WITH_NPC_COMBAT: + info := conditionData.GetWithNpcCombat() + infoData["win"] = info.RequiresWin + infoData["template_ids"] = info.CombatNpcTrainerId + case pogo.QuestConditionProto_WITH_PLAYER_LEVEL: + info := conditionData.GetWithPlayerLevel() + infoData["level"] = info.Level + case pogo.QuestConditionProto_WITH_BUDDY: + info := conditionData.GetWithBuddy() + if info != nil { + infoData["min_buddy_level"] = int(info.MinBuddyLevel) + infoData["must_be_on_map"] = info.MustBeOnMap + } else { + infoData["min_buddy_level"] = 0 + infoData["must_be_on_map"] = false + } + case pogo.QuestConditionProto_WITH_DAILY_BUDDY_AFFECTION: + info := conditionData.GetWithDailyBuddyAffection() + infoData["min_buddy_affection_earned_today"] = info.MinBuddyAffectionEarnedToday + case pogo.QuestConditionProto_WITH_TEMP_EVO_POKEMON: + info := conditionData.GetWithTempEvoId() + tempEvoIds := []int{} + for _, evolution := range info.MegaForm { + tempEvoIds = append(tempEvoIds, int(evolution)) + } + infoData["raid_pokemon_evolutions"] = tempEvoIds + case pogo.QuestConditionProto_WITH_ITEM_TYPE: + info := conditionData.GetWithItemType() + itemTypes := []int{} + for _, itemType := range info.ItemType { + itemTypes = append(itemTypes, int(itemType)) + } + infoData["item_type_ids"] = itemTypes + case pogo.QuestConditionProto_WITH_RAID_ELAPSED_TIME: + info := conditionData.GetWithElapsedTime() + infoData["time"] = int64(info.ElapsedTimeMs) / 1000 + case pogo.QuestConditionProto_WITH_WIN_GYM_BATTLE_STATUS: + case pogo.QuestConditionProto_WITH_SUPER_EFFECTIVE_CHARGE: + case pogo.QuestConditionProto_WITH_UNIQUE_POKESTOP: + case pogo.QuestConditionProto_WITH_QUEST_CONTEXT: + case pogo.QuestConditionProto_WITH_WIN_BATTLE_STATUS: + case pogo.QuestConditionProto_WITH_CURVE_BALL: + case pogo.QuestConditionProto_WITH_NEW_FRIEND: + case pogo.QuestConditionProto_WITH_DAYS_IN_A_ROW: + case pogo.QuestConditionProto_WITH_WEATHER_BOOST: + case pogo.QuestConditionProto_WITH_DAILY_CAPTURE_BONUS: + case pogo.QuestConditionProto_WITH_DAILY_SPIN_BONUS: + case pogo.QuestConditionProto_WITH_UNIQUE_POKEMON: + case pogo.QuestConditionProto_WITH_BUDDY_INTERESTING_POI: + case pogo.QuestConditionProto_WITH_POKEMON_LEVEL: + case pogo.QuestConditionProto_WITH_SINGLE_DAY: + case pogo.QuestConditionProto_WITH_UNIQUE_POKEMON_TEAM: + case pogo.QuestConditionProto_WITH_MAX_CP: + case pogo.QuestConditionProto_WITH_LUCKY_POKEMON: + case pogo.QuestConditionProto_WITH_LEGENDARY_POKEMON: + case pogo.QuestConditionProto_WITH_GBL_RANK: + case pogo.QuestConditionProto_WITH_CATCHES_IN_A_ROW: + case pogo.QuestConditionProto_WITH_ENCOUNTER_TYPE: + case pogo.QuestConditionProto_WITH_COMBAT_TYPE: + case pogo.QuestConditionProto_WITH_GEOTARGETED_POI: + case pogo.QuestConditionProto_WITH_FRIEND_LEVEL: + case pogo.QuestConditionProto_WITH_STICKER: + case pogo.QuestConditionProto_WITH_POKEMON_CP: + case pogo.QuestConditionProto_WITH_RAID_LOCATION: + case pogo.QuestConditionProto_WITH_FRIENDS_RAID: + case pogo.QuestConditionProto_WITH_POKEMON_COSTUME: + default: + break + } + + if infoData != nil { + condition["info"] = infoData + } + conditions = append(conditions, condition) + } + + for _, rewardData := range questData.QuestRewards { + reward := make(map[string]any) + infoData := make(map[string]any) + reward["type"] = int(rewardData.Type) + switch rewardData.Type { + case pogo.QuestRewardProto_EXPERIENCE: + infoData["amount"] = rewardData.GetExp() + case pogo.QuestRewardProto_ITEM: + info := rewardData.GetItem() + infoData["amount"] = info.Amount + infoData["item_id"] = int(info.Item) + case pogo.QuestRewardProto_STARDUST: + infoData["amount"] = rewardData.GetStardust() + case pogo.QuestRewardProto_CANDY: + info := rewardData.GetCandy() + infoData["amount"] = info.Amount + infoData["pokemon_id"] = int(info.PokemonId) + case pogo.QuestRewardProto_XL_CANDY: + info := rewardData.GetXlCandy() + infoData["amount"] = info.Amount + infoData["pokemon_id"] = int(info.PokemonId) + case pogo.QuestRewardProto_POKEMON_ENCOUNTER: + info := rewardData.GetPokemonEncounter() + if info.IsHiddenDitto { + infoData["pokemon_id"] = 132 + infoData["pokemon_id_display"] = int(info.GetPokemonId()) + } else { + infoData["pokemon_id"] = int(info.GetPokemonId()) + } + if info.ShinyProbability > 0.0 { + infoData["shiny_probability"] = info.ShinyProbability + } + if display := info.PokemonDisplay; display != nil { + if costumeId := int(display.Costume); costumeId != 0 { + infoData["costume_id"] = costumeId + } + if formId := int(display.Form); formId != 0 { + infoData["form_id"] = formId + } + if genderId := int(display.Gender); genderId != 0 { + infoData["gender_id"] = genderId + } + if display.Shiny { + infoData["shiny"] = display.Shiny + } + if background := util.ExtractBackgroundFromDisplay(display); background != nil { + infoData["background"] = background + } + if breadMode := int(display.BreadModeEnum); breadMode != 0 { + infoData["bread_mode"] = breadMode + } + } else { + + } + case pogo.QuestRewardProto_POKECOIN: + infoData["amount"] = rewardData.GetPokecoin() + case pogo.QuestRewardProto_STICKER: + info := rewardData.GetSticker() + infoData["amount"] = info.Amount + infoData["sticker_id"] = info.StickerId + case pogo.QuestRewardProto_MEGA_RESOURCE: + info := rewardData.GetMegaResource() + infoData["amount"] = info.Amount + infoData["pokemon_id"] = int(info.PokemonId) + case pogo.QuestRewardProto_AVATAR_CLOTHING: + case pogo.QuestRewardProto_QUEST: + case pogo.QuestRewardProto_LEVEL_CAP: + case pogo.QuestRewardProto_INCIDENT: + case pogo.QuestRewardProto_PLAYER_ATTRIBUTE: + default: + break + + } + reward["info"] = infoData + rewards = append(rewards, reward) + } + + questConditions, _ := json.Marshal(conditions) + questRewards, _ := json.Marshal(rewards) + questTimestamp := time.Now().Unix() + + questExpiry := null.NewInt(0, false) + + stopTimezone := tz.SearchTimezone(stop.Lat, stop.Lon) + if stopTimezone != "" { + loc, err := time.LoadLocation(stopTimezone) + if err != nil { + log.Warnf("Unrecognised time zone %s at %f,%f", stopTimezone, stop.Lat, stop.Lon) + } else { + year, month, day := time.Now().In(loc).Date() + t := time.Date(year, month, day, 0, 0, 0, 0, loc).AddDate(0, 0, 1) + unixTime := t.Unix() + questExpiry = null.IntFrom(unixTime) + } + } + + if questExpiry.Valid == false { + questExpiry = null.IntFrom(time.Now().Unix() + 24*60*60) // Set expiry to 24 hours from now + } + + if !haveAr { + stop.SetAlternativeQuestType(null.IntFrom(questType)) + stop.SetAlternativeQuestTarget(null.IntFrom(questTarget)) + stop.SetAlternativeQuestTemplate(null.StringFrom(questTemplate)) + stop.SetAlternativeQuestTitle(null.StringFrom(questTitle)) + stop.SetAlternativeQuestConditions(null.StringFrom(string(questConditions))) + stop.SetAlternativeQuestRewards(null.StringFrom(string(questRewards))) + stop.SetAlternativeQuestTimestamp(null.IntFrom(questTimestamp)) + stop.SetAlternativeQuestExpiry(questExpiry) + } else { + stop.SetQuestType(null.IntFrom(questType)) + stop.SetQuestTarget(null.IntFrom(questTarget)) + stop.SetQuestTemplate(null.StringFrom(questTemplate)) + stop.SetQuestTitle(null.StringFrom(questTitle)) + stop.SetQuestConditions(null.StringFrom(string(questConditions))) + stop.SetQuestRewards(null.StringFrom(string(questRewards))) + stop.SetQuestTimestamp(null.IntFrom(questTimestamp)) + stop.SetQuestExpiry(questExpiry) + } + + return questTitle +} + +func (stop *Pokestop) updatePokestopFromFortDetailsProto(fortData *pogo.FortDetailsOutProto) *Pokestop { + stop.SetId(fortData.Id) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) + if len(fortData.ImageUrl) > 0 { + stop.SetUrl(null.StringFrom(fortData.ImageUrl[0])) + } + stop.SetName(null.StringFrom(fortData.Name)) + + if fortData.Description == "" { + stop.SetDescription(null.NewString("", false)) + } else { + stop.SetDescription(null.StringFrom(fortData.Description)) + } + + if fortData.Modifier != nil && len(fortData.Modifier) > 0 { + // DeployingPlayerCodename contains the name of the player if we want that + lureId := int16(fortData.Modifier[0].ModifierType) + lureExpiry := fortData.Modifier[0].ExpirationTimeMs / 1000 + + stop.SetLureId(lureId) + stop.SetLureExpireTimestamp(null.IntFrom(lureExpiry)) + } + + return stop +} + +func (stop *Pokestop) updatePokestopFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto) *Pokestop { + stop.SetId(fortData.Id) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) + + if len(fortData.Image) > 0 { + stop.SetUrl(null.StringFrom(fortData.Image[0].Url)) + } + stop.SetName(null.StringFrom(fortData.Name)) + if stop.Deleted { + log.Debugf("Cleared Stop with id '%s' is found again in GMF, therefore kept deleted", stop.Id) + } + return stop +} + +func (stop *Pokestop) updatePokestopFromGetContestDataOutProto(contest *pogo.ContestProto) { + stop.SetShowcaseRankingStandard(null.IntFrom(int64(contest.GetMetric().GetRankingStandard()))) + stop.SetShowcaseExpiry(null.IntFrom(contest.GetSchedule().GetContestCycle().GetEndTimeMs() / 1000)) + + focusStore := createFocusStoreFromContestProto(contest) + + if len(focusStore) > 1 { + log.Warnf("SHOWCASE: we got more than one showcase focus: %v", focusStore) + } + + for key, focus := range focusStore { + focus["type"] = key + jsonBytes, err := json.Marshal(focus) + if err != nil { + log.Errorf("SHOWCASE: Stop '%s' - Focus '%v' marshalling failed: %s", stop.Id, focus, err) + } + stop.SetShowcaseFocus(null.StringFrom(string(jsonBytes))) + // still support old format - probably still required to filter in external tools + stop.extractShowcasePokemonInfoDeprecated(key, focus) + } +} + +func (stop *Pokestop) updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) { + type contestEntry struct { + Rank int `json:"rank"` + Score float64 `json:"score"` + PokemonId int `json:"pokemon_id"` + Form int `json:"form"` + Costume int `json:"costume"` + Gender int `json:"gender"` + Shiny bool `json:"shiny"` + TempEvolution int `json:"temp_evolution"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms"` + Alignment int `json:"alignment"` + Badge int `json:"badge"` + Background *int64 `json:"background,omitempty"` + } + type contestJson struct { + TotalEntries int `json:"total_entries"` + LastUpdate int64 `json:"last_update"` + ContestEntries []contestEntry `json:"contest_entries"` + } + + j := contestJson{LastUpdate: time.Now().Unix()} + j.TotalEntries = int(contestData.TotalEntries) + + for _, entry := range contestData.GetContestEntries() { + rank := entry.GetRank() + if rank > 3 { + break + } + j.ContestEntries = append(j.ContestEntries, contestEntry{ + Rank: int(rank), + Score: entry.GetScore(), + PokemonId: int(entry.GetPokedexId()), + Form: int(entry.GetPokemonDisplay().Form), + Costume: int(entry.GetPokemonDisplay().Costume), + Gender: int(entry.GetPokemonDisplay().Gender), + Shiny: entry.GetPokemonDisplay().Shiny, + TempEvolution: int(entry.GetPokemonDisplay().CurrentTempEvolution), + TempEvolutionFinishMs: entry.GetPokemonDisplay().TemporaryEvolutionFinishMs, + Alignment: int(entry.GetPokemonDisplay().Alignment), + Badge: int(entry.GetPokemonDisplay().PokemonBadge), + Background: util.ExtractBackgroundFromDisplay(entry.PokemonDisplay), + }) + + } + jsonString, _ := json.Marshal(j) + stop.SetShowcaseRankings(null.StringFrom(string(jsonString))) +} diff --git a/decoder/pokestop_process.go b/decoder/pokestop_process.go new file mode 100644 index 00000000..69dd78da --- /dev/null +++ b/decoder/pokestop_process.go @@ -0,0 +1,167 @@ +package decoder + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/paulmach/orb/geojson" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func UpdatePokestopRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fort.Id) + if err != nil { + log.Printf("Update pokestop %s", err) + return fmt.Sprintf("Error %s", err) + } + defer unlock() + + pokestop.updatePokestopFromFortDetailsProto(fort) + + updatePokestopGetMapFortCache(pokestop) + savePokestopRecord(ctx, db, pokestop) + return fmt.Sprintf("%s %s", fort.Id, fort.Name) +} + +func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.FortSearchOutProto, haveAr bool) string { + haveArStr := "NoAR" + if haveAr { + haveArStr = "AR" + } + + if quest.ChallengeQuest == nil { + statsCollector.IncDecodeQuest("error", "no_quest") + return fmt.Sprintf("%s %s Blank quest", quest.FortId, haveArStr) + } + + statsCollector.IncDecodeQuest("ok", haveArStr) + + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, quest.FortId) + if err != nil { + log.Printf("Update quest %s", err) + return fmt.Sprintf("error %s", err) + } + defer unlock() + + questTitle := pokestop.updatePokestopFromQuestProto(quest, haveAr) + + updatePokestopGetMapFortCache(pokestop) + savePokestopRecord(ctx, db, pokestop) + + areas := MatchStatsGeofence(pokestop.Lat, pokestop.Lon) + updateQuestStats(pokestop, haveAr, areas) + + return fmt.Sprintf("%s %s %s", quest.FortId, haveArStr, questTitle) +} + +func ClearQuestsWithinGeofence(ctx context.Context, dbDetails db.DbDetails, geofence *geojson.Feature) { + started := time.Now() + rows, err := db.RemoveQuests(ctx, dbDetails, geofence) + if err != nil { + log.Errorf("ClearQuest: Error removing quests: %s", err) + return + } + ClearPokestopCache() + log.Infof("ClearQuest: Removed quests from %d pokestops in %s", rows, time.Since(started)) +} + +func GetQuestStatusWithGeofence(dbDetails db.DbDetails, geofence *geojson.Feature) db.QuestStatus { + res, err := db.GetQuestStatus(dbDetails, geofence) + if err != nil { + log.Errorf("QuestStatus: Error retrieving quests: %s", err) + return db.QuestStatus{} + } + return res +} + +func UpdatePokestopRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, mapFort.Id) + if err != nil { + log.Printf("Update pokestop %s", err) + return false, fmt.Sprintf("Error %s", err) + } + + if pokestop == nil { + return false, "" + } + defer unlock() + + pokestop.updatePokestopFromGetMapFortsOutProto(mapFort) + savePokestopRecord(ctx, db, pokestop) + return true, fmt.Sprintf("%s %s", mapFort.Id, mapFort.Name) +} + +func GetPokestopPositions(details db.DbDetails, geofence *geojson.Feature) ([]db.QuestLocation, error) { + return db.GetPokestopPositions(details, geofence) +} + +func UpdatePokestopWithContestData(ctx context.Context, db db.DbDetails, request *pogo.GetContestDataProto, contestData *pogo.GetContestDataOutProto) string { + if contestData.ContestIncident == nil || len(contestData.ContestIncident.Contests) == 0 { + return "No contests found" + } + + var fortId string + if request != nil { + fortId = request.FortId + } else { + fortId = getFortIdFromContest(contestData.ContestIncident.Contests[0].ContestId) + } + + if fortId == "" { + return "No fortId found" + } + + if len(contestData.ContestIncident.Contests) > 1 { + log.Errorf("More than one contest found") + return fmt.Sprintf("More than one contest found in %s", fortId) + } + + contest := contestData.ContestIncident.Contests[0] + + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) + if err != nil { + log.Printf("Get pokestop %s", err) + return "Error getting pokestop" + } + + if pokestop == nil { + log.Infof("Contest data for pokestop %s not found", fortId) + return fmt.Sprintf("Contest data for pokestop %s not found", fortId) + } + defer unlock() + + pokestop.updatePokestopFromGetContestDataOutProto(contest) + savePokestopRecord(ctx, db, pokestop) + + return fmt.Sprintf("Contest %s", fortId) +} + +func getFortIdFromContest(id string) string { + return strings.Split(id, "-")[0] +} + +func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDetails, request *pogo.GetPokemonSizeLeaderboardEntryProto, contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) string { + fortId := getFortIdFromContest(request.GetContestId()) + + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) + if err != nil { + log.Printf("Get pokestop %s", err) + return "Error getting pokestop" + } + + if pokestop == nil { + log.Infof("Contest data for pokestop %s not found", fortId) + return fmt.Sprintf("Contest data for pokestop %s not found", fortId) + } + defer unlock() + + pokestop.updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData) + savePokestopRecord(ctx, db, pokestop) + + return fmt.Sprintf("Contest Detail %s", fortId) +} diff --git a/decoder/pokestop_showcase.go b/decoder/pokestop_showcase.go index 42876295..44fba2ba 100644 --- a/decoder/pokestop_showcase.go +++ b/decoder/pokestop_showcase.go @@ -3,8 +3,8 @@ package decoder import ( "golbat/pogo" + "github.com/guregu/null/v6" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) type contestFocusType string diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go new file mode 100644 index 00000000..f2116179 --- /dev/null +++ b/decoder/pokestop_state.go @@ -0,0 +1,397 @@ +package decoder + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "time" + + "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/config" + "golbat/db" + "golbat/webhooks" +) + +func loadPokestopFromDatabase(ctx context.Context, db db.DbDetails, fortId string, pokestop *Pokestop) error { + err := db.GeneralDb.GetContext(ctx, pokestop, + `SELECT pokestop.id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, + pokestop.updated, quest_type, quest_timestamp, quest_target, quest_conditions, + quest_rewards, quest_template, quest_title, + alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, + alternative_quest_conditions, alternative_quest_rewards, + alternative_quest_template, alternative_quest_title, cell_id, deleted, lure_id, sponsor_id, partner_id, + ar_scan_eligible, power_up_points, power_up_level, power_up_end_timestamp, + quest_expiry, alternative_quest_expiry, description, showcase_pokemon_id, showcase_pokemon_form_id, + showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings + FROM pokestop + WHERE pokestop.id = ? `, fortId) + statsCollector.IncDbQuery("select pokestop", err) + return err +} + +// PeekPokestopRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekPokestopRecord(fortId string) (*Pokestop, func(), error) { + if item := pokestopCache.Get(fortId); item != nil { + pokestop := item.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil + } + return nil, nil, nil +} + +// getPokestopRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + // Check cache first + if item := pokestopCache.Get(fortId); item != nil { + pokestop := item.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil + } + + dbPokestop := Pokestop{} + err := loadPokestopFromDatabase(ctx, db, fortId, &dbPokestop) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + + dbPokestop.ClearDirty() + + // Atomically cache the loaded Pokestop - if another goroutine raced us, + // we'll get their Pokestop and use that instead (ensuring same mutex) + existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + // Only called if key doesn't exist - our Pokestop wins + if config.Config.TestFortInMemory { + fortRtreeUpdatePokestopOnGet(&dbPokestop) + } + return &dbPokestop + }) + + pokestop := existingPokestop.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil +} + +// getPokestopRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Pokestop. +// Caller MUST call returned unlock function if non-nil. +func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + pokestop, unlock, err := getPokestopRecordReadOnly(ctx, db, fortId) + if err != nil || pokestop == nil { + return nil, nil, err + } + pokestop.snapshotOldValues() + return pokestop, unlock, nil +} + +// getOrCreatePokestopRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + // Create new Pokestop atomically - function only called if key doesn't exist + pokestopItem, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + return &Pokestop{Id: fortId, newRecord: true} + }) + + pokestop := pokestopItem.Value() + pokestop.Lock() + + if pokestop.newRecord { + // We should attempt to load from database + err := loadPokestopFromDatabase(ctx, db, fortId, pokestop) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + pokestop.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + pokestop.newRecord = false + pokestop.ClearDirty() + if config.Config.TestFortInMemory { + fortRtreeUpdatePokestopOnGet(pokestop) + } + } + } + + pokestop.snapshotOldValues() + return pokestop, func() { pokestop.Unlock() }, nil +} + +type QuestWebhook struct { + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + PokestopName string `json:"pokestop_name"` + Type null.Int `json:"type"` + Target null.Int `json:"target"` + Template null.String `json:"template"` + Title null.String `json:"title"` + Conditions json.RawMessage `json:"conditions"` + Rewards json.RawMessage `json:"rewards"` + Updated int64 `json:"updated"` + ArScanEligible int64 `json:"ar_scan_eligible"` + PokestopUrl string `json:"pokestop_url"` + WithAr bool `json:"with_ar"` +} + +type PokestopWebhook struct { + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + Url string `json:"url"` + LureExpiration int64 `json:"lure_expiration"` + LastModified int64 `json:"last_modified"` + Enabled bool `json:"enabled"` + LureId int16 `json:"lure_id"` + ArScanEligible int64 `json:"ar_scan_eligible"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + Updated int64 `json:"updated"` + ShowcaseFocus null.String `json:"showcase_focus"` + ShowcasePokemonId null.Int `json:"showcase_pokemon_id"` + ShowcasePokemonFormId null.Int `json:"showcase_pokemon_form_id"` + ShowcasePokemonTypeId null.Int `json:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `json:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `json:"showcase_expiry"` + ShowcaseRankings json.RawMessage `json:"showcase_rankings"` +} + +func createPokestopFortWebhooks(stop *Pokestop) { + fort := InitWebHookFortFromPokestop(stop) + if stop.newRecord { + CreateFortWebHooks(nil, fort, NEW) + } else { + // Build old fort from saved old values + oldFort := &FortWebhook{ + Type: POKESTOP.String(), + Id: stop.Id, + Name: stop.oldValues.Name.Ptr(), + ImageUrl: stop.oldValues.Url.Ptr(), + Description: stop.oldValues.Description.Ptr(), + Location: Location{Latitude: stop.oldValues.Lat, Longitude: stop.oldValues.Lon}, + } + CreateFortWebHooks(oldFort, fort, EDIT) + } +} + +func createPokestopWebhooks(stop *Pokestop) { + + areas := MatchStatsGeofence(stop.Lat, stop.Lon) + + pokestopName := "Unknown" + if stop.Name.Valid { + pokestopName = stop.Name.String + } + + if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldValues.AlternativeQuestType) { + questHook := QuestWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Type: stop.AlternativeQuestType, + Target: stop.AlternativeQuestTarget, + Template: stop.AlternativeQuestTemplate, + Title: stop.AlternativeQuestTitle, + Conditions: json.RawMessage(stop.AlternativeQuestConditions.ValueOrZero()), + Rewards: json.RawMessage(stop.AlternativeQuestRewards.ValueOrZero()), + Updated: stop.Updated, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PokestopUrl: stop.Url.ValueOrZero(), + WithAr: false, + } + webhooksSender.AddMessage(webhooks.Quest, questHook, areas) + } + + if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldValues.QuestType) { + questHook := QuestWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Type: stop.QuestType, + Target: stop.QuestTarget, + Template: stop.QuestTemplate, + Title: stop.QuestTitle, + Conditions: json.RawMessage(stop.QuestConditions.ValueOrZero()), + Rewards: json.RawMessage(stop.QuestRewards.ValueOrZero()), + Updated: stop.Updated, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PokestopUrl: stop.Url.ValueOrZero(), + WithAr: true, + } + webhooksSender.AddMessage(webhooks.Quest, questHook, areas) + } + if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldValues.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldValues.PowerUpEndTimestamp)) { + var showcaseRankings json.RawMessage + if stop.ShowcaseRankings.Valid { + showcaseRankings = json.RawMessage(stop.ShowcaseRankings.ValueOrZero()) + } + + pokestopHook := PokestopWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + Name: pokestopName, + Url: stop.Url.ValueOrZero(), + LureExpiration: stop.LureExpireTimestamp.ValueOrZero(), + LastModified: stop.LastModifiedTimestamp.ValueOrZero(), + Enabled: stop.Enabled.ValueOrZero(), + LureId: stop.LureId, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PowerUpLevel: stop.PowerUpLevel.ValueOrZero(), + PowerUpPoints: stop.PowerUpPoints.ValueOrZero(), + PowerUpEndTimestamp: stop.PowerUpEndTimestamp.ValueOrZero(), + Updated: stop.Updated, + ShowcaseFocus: stop.ShowcaseFocus, + ShowcasePokemonId: stop.ShowcasePokemon, + ShowcasePokemonFormId: stop.ShowcasePokemonForm, + ShowcasePokemonTypeId: stop.ShowcasePokemonType, + ShowcaseRankingStandard: stop.ShowcaseRankingStandard, + ShowcaseExpiry: stop.ShowcaseExpiry, + ShowcaseRankings: showcaseRankings, + } + + webhooksSender.AddMessage(webhooks.Pokestop, pokestopHook, areas) + } +} + +func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { + now := time.Now().Unix() + if !pokestop.IsNewRecord() && !pokestop.IsDirty() { + // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. + if pokestop.Updated > now-GetUpdateThreshold(900) { + // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again + return + } + } + pokestop.Updated = now + + if pokestop.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, ` + INSERT INTO pokestop ( + id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, + quest_timestamp, quest_target, quest_conditions, quest_rewards, quest_template, quest_title, + alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, + alternative_quest_conditions, alternative_quest_rewards, alternative_quest_template, + alternative_quest_title, cell_id, lure_id, sponsor_id, partner_id, ar_scan_eligible, + power_up_points, power_up_level, power_up_end_timestamp, updated, first_seen_timestamp, + quest_expiry, alternative_quest_expiry, description, showcase_focus, showcase_pokemon_id, + showcase_pokemon_form_id, showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings + ) + VALUES ( + :id, :lat, :lon, :name, :url, :enabled, :lure_expire_timestamp, :last_modified_timestamp, :quest_type, + :quest_timestamp, :quest_target, :quest_conditions, :quest_rewards, :quest_template, :quest_title, + :alternative_quest_type, :alternative_quest_timestamp, :alternative_quest_target, + :alternative_quest_conditions, :alternative_quest_rewards, :alternative_quest_template, + :alternative_quest_title, :cell_id, :lure_id, :sponsor_id, :partner_id, :ar_scan_eligible, + :power_up_points, :power_up_level, :power_up_end_timestamp, + UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), + :quest_expiry, :alternative_quest_expiry, :description, :showcase_focus, :showcase_pokemon_id, + :showcase_pokemon_form_id, :showcase_pokemon_type_id, :showcase_ranking_standard, :showcase_expiry, :showcase_rankings)`, + pokestop) + + statsCollector.IncDbQuery("insert pokestop", err) + //log.Debugf("Insert pokestop %s %+v", pokestop.Id, pokestop) + if err != nil { + log.Errorf("insert pokestop: %s", err) + return + } + + _, _ = res, err + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, ` + UPDATE pokestop SET + lat = :lat, + lon = :lon, + name = :name, + url = :url, + enabled = :enabled, + lure_expire_timestamp = :lure_expire_timestamp, + last_modified_timestamp = :last_modified_timestamp, + updated = :updated, + quest_type = :quest_type, + quest_timestamp = :quest_timestamp, + quest_target = :quest_target, + quest_conditions = :quest_conditions, + quest_rewards = :quest_rewards, + quest_template = :quest_template, + quest_title = :quest_title, + alternative_quest_type = :alternative_quest_type, + alternative_quest_timestamp = :alternative_quest_timestamp, + alternative_quest_target = :alternative_quest_target, + alternative_quest_conditions = :alternative_quest_conditions, + alternative_quest_rewards = :alternative_quest_rewards, + alternative_quest_template = :alternative_quest_template, + alternative_quest_title = :alternative_quest_title, + cell_id = :cell_id, + lure_id = :lure_id, + deleted = :deleted, + sponsor_id = :sponsor_id, + partner_id = :partner_id, + ar_scan_eligible = :ar_scan_eligible, + power_up_points = :power_up_points, + power_up_level = :power_up_level, + power_up_end_timestamp = :power_up_end_timestamp, + quest_expiry = :quest_expiry, + alternative_quest_expiry = :alternative_quest_expiry, + description = :description, + showcase_focus = :showcase_focus, + showcase_pokemon_id = :showcase_pokemon_id, + showcase_pokemon_form_id = :showcase_pokemon_form_id, + showcase_pokemon_type_id = :showcase_pokemon_type_id, + showcase_ranking_standard = :showcase_ranking_standard, + showcase_expiry = :showcase_expiry, + showcase_rankings = :showcase_rankings + WHERE id = :id`, + pokestop, + ) + statsCollector.IncDbQuery("update pokestop", err) + //log.Debugf("Update pokestop %s %+v", pokestop.Id, pokestop) + if err != nil { + log.Errorf("update pokestop %s: %s", pokestop.Id, err) + return + } + _ = res + } + //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + if dbDebugEnabled { + pokestop.changedFields = pokestop.changedFields[:0] + } + + createPokestopWebhooks(pokestop) + createPokestopFortWebhooks(pokestop) + if pokestop.IsNewRecord() { + pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + pokestop.newRecord = false + } + pokestop.ClearDirty() + +} + +func updatePokestopGetMapFortCache(pokestop *Pokestop) { + storedGetMapFort := getMapFortsCache.Get(pokestop.Id) + if storedGetMapFort != nil { + getMapFort := storedGetMapFort.Value() + getMapFortsCache.Delete(pokestop.Id) + pokestop.updatePokestopFromGetMapFortsOutProto(getMapFort) + log.Debugf("Updated Gym using stored getMapFort: %s", pokestop.Id) + } +} diff --git a/decoder/routes.go b/decoder/routes.go index e707b806..214d7812 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -1,21 +1,9 @@ package decoder import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" "sync" - "time" - "golbat/db" - "golbat/pogo" - "golbat/util" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" ) // Route struct. @@ -292,238 +280,3 @@ func (r *Route) SetWaypoints(v string) { } } } - -func loadRouteFromDatabase(ctx context.Context, db db.DbDetails, routeId string, route *Route) error { - err := db.GeneralDb.GetContext(ctx, route, - `SELECT * FROM route WHERE route.id = ?`, routeId) - statsCollector.IncDbQuery("select route", err) - return err -} - -// peekRouteRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func peekRouteRecord(routeId string) (*Route, func(), error) { - if item := routeCache.Get(routeId); item != nil { - route := item.Value() - route.Lock() - return route, func() { route.Unlock() }, nil - } - return nil, nil, nil -} - -// getRouteRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getRouteRecordReadOnly(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { - // Check cache first - if item := routeCache.Get(routeId); item != nil { - route := item.Value() - route.Lock() - return route, func() { route.Unlock() }, nil - } - - dbRoute := Route{} - err := loadRouteFromDatabase(ctx, db, routeId, &dbRoute) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbRoute.ClearDirty() - - // Atomically cache the loaded Route - if another goroutine raced us, - // we'll get their Route and use that instead (ensuring same mutex) - existingRoute, _ := routeCache.GetOrSetFunc(routeId, func() *Route { - return &dbRoute - }) - - route := existingRoute.Value() - route.Lock() - return route, func() { route.Unlock() }, nil -} - -// getRouteRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Caller MUST call returned unlock function if non-nil. -func getRouteRecordForUpdate(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { - route, unlock, err := getRouteRecordReadOnly(ctx, db, routeId) - if err != nil || route == nil { - return nil, nil, err - } - route.snapshotOldValues() - return route, unlock, nil -} - -// getOrCreateRouteRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreateRouteRecord(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { - // Create new Route atomically - function only called if key doesn't exist - routeItem, _ := routeCache.GetOrSetFunc(routeId, func() *Route { - return &Route{Id: routeId, newRecord: true} - }) - - route := routeItem.Value() - route.Lock() - - if route.newRecord { - // We should attempt to load from database - err := loadRouteFromDatabase(ctx, db, routeId, route) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - route.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - route.newRecord = false - route.ClearDirty() - } - } - - route.snapshotOldValues() - return route, func() { route.Unlock() }, nil -} - -func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { - // Skip save if not dirty and not new, unless 15-minute debounce expired - if !route.IsDirty() && !route.IsNewRecord() { - if route.Updated > time.Now().Unix()-GetUpdateThreshold(900) { - // if a route is unchanged, but we did see it again after 15 minutes, then save again - return nil - } - } - - route.Updated = time.Now().Unix() - - if route.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Route", route.Id, route.changedFields) - } - _, err := db.GeneralDb.NamedExecContext(ctx, - ` - INSERT INTO route ( - id, name, shortcode, description, distance_meters, - duration_seconds, end_fort_id, end_image, - end_lat, end_lon, image, image_border_color, - reversible, start_fort_id, start_image, - start_lat, start_lon, tags, type, - updated, version, waypoints - ) - VALUES - ( - :id, :name, :shortcode, :description, :distance_meters, - :duration_seconds, :end_fort_id, - :end_image, :end_lat, :end_lon, :image, - :image_border_color, :reversible, - :start_fort_id, :start_image, :start_lat, - :start_lon, :tags, :type, :updated, - :version, :waypoints - ) - `, - route, - ) - - statsCollector.IncDbQuery("insert route", err) - if err != nil { - return fmt.Errorf("insert route error: %w", err) - } - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) - } - _, err := db.GeneralDb.NamedExecContext(ctx, - ` - UPDATE route SET - name = :name, - shortcode = :shortcode, - description = :description, - distance_meters = :distance_meters, - duration_seconds = :duration_seconds, - end_fort_id = :end_fort_id, - end_image = :end_image, - end_lat = :end_lat, - end_lon = :end_lon, - image = :image, - image_border_color = :image_border_color, - reversible = :reversible, - start_fort_id = :start_fort_id, - start_image = :start_image, - start_lat = :start_lat, - start_lon = :start_lon, - tags = :tags, - type = :type, - updated = :updated, - version = :version, - waypoints = :waypoints - WHERE id = :id`, - route, - ) - - statsCollector.IncDbQuery("update route", err) - if err != nil { - return fmt.Errorf("update route error %w", err) - } - } - - if dbDebugEnabled { - route.changedFields = route.changedFields[:0] - } - route.ClearDirty() - if route.IsNewRecord() { - routeCache.Set(route.Id, route, ttlcache.DefaultTTL) - route.newRecord = false - } - return nil -} - -func (route *Route) updateFromSharedRouteProto(sharedRouteProto *pogo.SharedRouteProto) { - route.SetName(sharedRouteProto.GetName()) - if sharedRouteProto.GetShortCode() != "" { - route.SetShortcode(sharedRouteProto.GetShortCode()) - } - description := sharedRouteProto.GetDescription() - // NOTE: Some descriptions have more than 255 runes, which won't fit in our - // varchar(255). - if truncateStr, truncated := util.TruncateUTF8(description, 255); truncated { - log.Warnf("truncating description for route id '%s'. Orig description: %s", - route.Id, - description, - ) - description = truncateStr - } - route.SetDescription(description) - route.SetDistanceMeters(sharedRouteProto.GetRouteDistanceMeters()) - route.SetDurationSeconds(sharedRouteProto.GetRouteDurationSeconds()) - route.SetEndFortId(sharedRouteProto.GetEndPoi().GetAnchor().GetFortId()) - route.SetEndImage(sharedRouteProto.GetEndPoi().GetImageUrl()) - route.SetEndLat(sharedRouteProto.GetEndPoi().GetAnchor().GetLatDegrees()) - route.SetEndLon(sharedRouteProto.GetEndPoi().GetAnchor().GetLngDegrees()) - route.SetImage(sharedRouteProto.GetImage().GetImageUrl()) - route.SetImageBorderColor(sharedRouteProto.GetImage().GetBorderColorHex()) - route.SetReversible(sharedRouteProto.GetReversible()) - route.SetStartFortId(sharedRouteProto.GetStartPoi().GetAnchor().GetFortId()) - route.SetStartImage(sharedRouteProto.GetStartPoi().GetImageUrl()) - route.SetStartLat(sharedRouteProto.GetStartPoi().GetAnchor().GetLatDegrees()) - route.SetStartLon(sharedRouteProto.GetStartPoi().GetAnchor().GetLngDegrees()) - route.SetType(int8(sharedRouteProto.GetType())) - route.SetVersion(sharedRouteProto.GetVersion()) - waypoints, _ := json.Marshal(sharedRouteProto.GetWaypoints()) - route.SetWaypoints(string(waypoints)) - - if len(sharedRouteProto.GetTags()) > 0 { - tags, _ := json.Marshal(sharedRouteProto.GetTags()) - route.SetTags(null.StringFrom(string(tags))) - } -} - -func UpdateRouteRecordWithSharedRouteProto(ctx context.Context, db db.DbDetails, sharedRouteProto *pogo.SharedRouteProto) error { - route, unlock, err := getOrCreateRouteRecord(ctx, db, sharedRouteProto.GetId()) - if err != nil { - return err - } - defer unlock() - - route.updateFromSharedRouteProto(sharedRouteProto) - saveError := saveRouteRecord(ctx, db, route) - return saveError -} diff --git a/decoder/routes_decode.go b/decoder/routes_decode.go new file mode 100644 index 00000000..cb72fd5a --- /dev/null +++ b/decoder/routes_decode.go @@ -0,0 +1,51 @@ +package decoder + +import ( + "encoding/json" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" + "golbat/util" +) + +func (route *Route) updateFromSharedRouteProto(sharedRouteProto *pogo.SharedRouteProto) { + route.SetName(sharedRouteProto.GetName()) + if sharedRouteProto.GetShortCode() != "" { + route.SetShortcode(sharedRouteProto.GetShortCode()) + } + description := sharedRouteProto.GetDescription() + // NOTE: Some descriptions have more than 255 runes, which won't fit in our + // varchar(255). + if truncateStr, truncated := util.TruncateUTF8(description, 255); truncated { + log.Warnf("truncating description for route id '%s'. Orig description: %s", + route.Id, + description, + ) + description = truncateStr + } + route.SetDescription(description) + route.SetDistanceMeters(sharedRouteProto.GetRouteDistanceMeters()) + route.SetDurationSeconds(sharedRouteProto.GetRouteDurationSeconds()) + route.SetEndFortId(sharedRouteProto.GetEndPoi().GetAnchor().GetFortId()) + route.SetEndImage(sharedRouteProto.GetEndPoi().GetImageUrl()) + route.SetEndLat(sharedRouteProto.GetEndPoi().GetAnchor().GetLatDegrees()) + route.SetEndLon(sharedRouteProto.GetEndPoi().GetAnchor().GetLngDegrees()) + route.SetImage(sharedRouteProto.GetImage().GetImageUrl()) + route.SetImageBorderColor(sharedRouteProto.GetImage().GetBorderColorHex()) + route.SetReversible(sharedRouteProto.GetReversible()) + route.SetStartFortId(sharedRouteProto.GetStartPoi().GetAnchor().GetFortId()) + route.SetStartImage(sharedRouteProto.GetStartPoi().GetImageUrl()) + route.SetStartLat(sharedRouteProto.GetStartPoi().GetAnchor().GetLatDegrees()) + route.SetStartLon(sharedRouteProto.GetStartPoi().GetAnchor().GetLngDegrees()) + route.SetType(int8(sharedRouteProto.GetType())) + route.SetVersion(sharedRouteProto.GetVersion()) + waypoints, _ := json.Marshal(sharedRouteProto.GetWaypoints()) + route.SetWaypoints(string(waypoints)) + + if len(sharedRouteProto.GetTags()) > 0 { + tags, _ := json.Marshal(sharedRouteProto.GetTags()) + route.SetTags(null.StringFrom(string(tags))) + } +} diff --git a/decoder/routes_process.go b/decoder/routes_process.go new file mode 100644 index 00000000..a0c5c83b --- /dev/null +++ b/decoder/routes_process.go @@ -0,0 +1,20 @@ +package decoder + +import ( + "context" + + "golbat/db" + "golbat/pogo" +) + +func UpdateRouteRecordWithSharedRouteProto(ctx context.Context, db db.DbDetails, sharedRouteProto *pogo.SharedRouteProto) error { + route, unlock, err := getOrCreateRouteRecord(ctx, db, sharedRouteProto.GetId()) + if err != nil { + return err + } + defer unlock() + + route.updateFromSharedRouteProto(sharedRouteProto) + saveError := saveRouteRecord(ctx, db, route) + return saveError +} diff --git a/decoder/routes_state.go b/decoder/routes_state.go new file mode 100644 index 00000000..8538cc86 --- /dev/null +++ b/decoder/routes_state.go @@ -0,0 +1,196 @@ +package decoder + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/jellydator/ttlcache/v3" + + "golbat/db" +) + +func loadRouteFromDatabase(ctx context.Context, db db.DbDetails, routeId string, route *Route) error { + err := db.GeneralDb.GetContext(ctx, route, + `SELECT * FROM route WHERE route.id = ?`, routeId) + statsCollector.IncDbQuery("select route", err) + return err +} + +// peekRouteRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekRouteRecord(routeId string) (*Route, func(), error) { + if item := routeCache.Get(routeId); item != nil { + route := item.Value() + route.Lock() + return route, func() { route.Unlock() }, nil + } + return nil, nil, nil +} + +// getRouteRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getRouteRecordReadOnly(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + // Check cache first + if item := routeCache.Get(routeId); item != nil { + route := item.Value() + route.Lock() + return route, func() { route.Unlock() }, nil + } + + dbRoute := Route{} + err := loadRouteFromDatabase(ctx, db, routeId, &dbRoute) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbRoute.ClearDirty() + + // Atomically cache the loaded Route - if another goroutine raced us, + // we'll get their Route and use that instead (ensuring same mutex) + existingRoute, _ := routeCache.GetOrSetFunc(routeId, func() *Route { + return &dbRoute + }) + + route := existingRoute.Value() + route.Lock() + return route, func() { route.Unlock() }, nil +} + +// getRouteRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getRouteRecordForUpdate(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + route, unlock, err := getRouteRecordReadOnly(ctx, db, routeId) + if err != nil || route == nil { + return nil, nil, err + } + route.snapshotOldValues() + return route, unlock, nil +} + +// getOrCreateRouteRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateRouteRecord(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + // Create new Route atomically - function only called if key doesn't exist + routeItem, _ := routeCache.GetOrSetFunc(routeId, func() *Route { + return &Route{Id: routeId, newRecord: true} + }) + + route := routeItem.Value() + route.Lock() + + if route.newRecord { + // We should attempt to load from database + err := loadRouteFromDatabase(ctx, db, routeId, route) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + route.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + route.newRecord = false + route.ClearDirty() + } + } + + route.snapshotOldValues() + return route, func() { route.Unlock() }, nil +} + +func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { + // Skip save if not dirty and not new, unless 15-minute debounce expired + if !route.IsDirty() && !route.IsNewRecord() { + if route.Updated > time.Now().Unix()-GetUpdateThreshold(900) { + // if a route is unchanged, but we did see it again after 15 minutes, then save again + return nil + } + } + + route.Updated = time.Now().Unix() + + if route.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Route", route.Id, route.changedFields) + } + _, err := db.GeneralDb.NamedExecContext(ctx, + ` + INSERT INTO route ( + id, name, shortcode, description, distance_meters, + duration_seconds, end_fort_id, end_image, + end_lat, end_lon, image, image_border_color, + reversible, start_fort_id, start_image, + start_lat, start_lon, tags, type, + updated, version, waypoints + ) + VALUES + ( + :id, :name, :shortcode, :description, :distance_meters, + :duration_seconds, :end_fort_id, + :end_image, :end_lat, :end_lon, :image, + :image_border_color, :reversible, + :start_fort_id, :start_image, :start_lat, + :start_lon, :tags, :type, :updated, + :version, :waypoints + ) + `, + route, + ) + + statsCollector.IncDbQuery("insert route", err) + if err != nil { + return fmt.Errorf("insert route error: %w", err) + } + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) + } + _, err := db.GeneralDb.NamedExecContext(ctx, + ` + UPDATE route SET + name = :name, + shortcode = :shortcode, + description = :description, + distance_meters = :distance_meters, + duration_seconds = :duration_seconds, + end_fort_id = :end_fort_id, + end_image = :end_image, + end_lat = :end_lat, + end_lon = :end_lon, + image = :image, + image_border_color = :image_border_color, + reversible = :reversible, + start_fort_id = :start_fort_id, + start_image = :start_image, + start_lat = :start_lat, + start_lon = :start_lon, + tags = :tags, + type = :type, + updated = :updated, + version = :version, + waypoints = :waypoints + WHERE id = :id`, + route, + ) + + statsCollector.IncDbQuery("update route", err) + if err != nil { + return fmt.Errorf("update route error %w", err) + } + } + + if dbDebugEnabled { + route.changedFields = route.changedFields[:0] + } + route.ClearDirty() + if route.IsNewRecord() { + routeCache.Set(route.Id, route, ttlcache.DefaultTTL) + route.newRecord = false + } + return nil +} diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 4f59ebaf..736b302f 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -9,9 +9,9 @@ import ( "golbat/db" "github.com/golang/geo/s2" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) type S2Cell struct { diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 7e1d8811..f858cc85 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -11,9 +11,9 @@ import ( "golbat/db" "golbat/pogo" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) // Spawnpoint struct. diff --git a/decoder/station.go b/decoder/station.go index bbf5c79a..9feea5c1 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -1,22 +1,9 @@ package decoder import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" "sync" - "time" - "golbat/db" - "golbat/pogo" - "golbat/util" - "golbat/webhooks" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" ) // Station struct. @@ -372,376 +359,3 @@ func (station *Station) SetStationedPokemon(v null.String) { } } } - -type StationWebhook struct { - Id string `json:"id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Name string `json:"name"` - StartTime int64 `json:"start_time"` - EndTime int64 `json:"end_time"` - IsBattleAvailable bool `json:"is_battle_available"` - BattleLevel null.Int `json:"battle_level"` - BattleStart null.Int `json:"battle_start"` - BattleEnd null.Int `json:"battle_end"` - BattlePokemonId null.Int `json:"battle_pokemon_id"` - BattlePokemonForm null.Int `json:"battle_pokemon_form"` - BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` - BattlePokemonGender null.Int `json:"battle_pokemon_gender"` - BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` - BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` - BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` - BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` - TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` - TotalStationedGmax null.Int `json:"total_stationed_gmax"` - Updated int64 `json:"updated"` -} - -func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId string, station *Station) error { - err := db.GeneralDb.GetContext(ctx, station, - `SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon - FROM station WHERE id = ?`, stationId) - statsCollector.IncDbQuery("select station", err) - return err -} - -// peekStationRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func peekStationRecord(stationId string) (*Station, func(), error) { - if item := stationCache.Get(stationId); item != nil { - station := item.Value() - station.Lock() - return station, func() { station.Unlock() }, nil - } - return nil, nil, nil -} - -// getStationRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { - // Check cache first - if item := stationCache.Get(stationId); item != nil { - station := item.Value() - station.Lock() - return station, func() { station.Unlock() }, nil - } - - dbStation := Station{} - err := loadStationFromDatabase(ctx, db, stationId, &dbStation) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbStation.ClearDirty() - - // Atomically cache the loaded Station - if another goroutine raced us, - // we'll get their Station and use that instead (ensuring same mutex) - existingStation, _ := stationCache.GetOrSetFunc(stationId, func() *Station { - return &dbStation - }) - - station := existingStation.Value() - station.Lock() - return station, func() { station.Unlock() }, nil -} - -// getStationRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Caller MUST call returned unlock function if non-nil. -func getStationRecordForUpdate(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { - station, unlock, err := getStationRecordReadOnly(ctx, db, stationId) - if err != nil || station == nil { - return nil, nil, err - } - station.snapshotOldValues() - return station, unlock, nil -} - -// getOrCreateStationRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { - // Create new Station atomically - function only called if key doesn't exist - stationItem, _ := stationCache.GetOrSetFunc(stationId, func() *Station { - return &Station{Id: stationId, newRecord: true} - }) - - station := stationItem.Value() - station.Lock() - - if station.newRecord { - // We should attempt to load from database - err := loadStationFromDatabase(ctx, db, stationId, station) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - station.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - station.newRecord = false - station.ClearDirty() - } - } - - station.snapshotOldValues() - return station, func() { station.Unlock() }, nil -} - -func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { - now := time.Now().Unix() - - // Skip save if not dirty and was updated recently (15-min debounce) - if !station.IsDirty() && !station.IsNewRecord() { - if station.Updated > now-GetUpdateThreshold(900) { - return - } - } - - station.Updated = now - - if station.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Station", station.Id, station.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, - ` - INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) - VALUES (:id,:lat,:lon,:name,:cell_id,:start_time,:end_time,:cooldown_complete,:is_battle_available,:is_inactive,:updated,:battle_level,:battle_start,:battle_end,:battle_pokemon_id,:battle_pokemon_form,:battle_pokemon_costume,:battle_pokemon_gender,:battle_pokemon_alignment,:battle_pokemon_bread_mode,:battle_pokemon_move_1,:battle_pokemon_move_2,:battle_pokemon_stamina,:battle_pokemon_cp_multiplier,:total_stationed_pokemon,:total_stationed_gmax,:stationed_pokemon) - `, station) - - statsCollector.IncDbQuery("insert station", err) - if err != nil { - log.Errorf("insert station: %s", err) - return - } - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Station", station.Id, station.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, ` - UPDATE station - SET - lat = :lat, - lon = :lon, - name = :name, - cell_id = :cell_id, - start_time = :start_time, - end_time = :end_time, - cooldown_complete = :cooldown_complete, - is_battle_available = :is_battle_available, - is_inactive = :is_inactive, - updated = :updated, - battle_level = :battle_level, - battle_start = :battle_start, - battle_end = :battle_end, - battle_pokemon_id = :battle_pokemon_id, - battle_pokemon_form = :battle_pokemon_form, - battle_pokemon_costume = :battle_pokemon_costume, - battle_pokemon_gender = :battle_pokemon_gender, - battle_pokemon_alignment = :battle_pokemon_alignment, - battle_pokemon_bread_mode = :battle_pokemon_bread_mode, - battle_pokemon_move_1 = :battle_pokemon_move_1, - battle_pokemon_move_2 = :battle_pokemon_move_2, - battle_pokemon_stamina = :battle_pokemon_stamina, - battle_pokemon_cp_multiplier = :battle_pokemon_cp_multiplier, - total_stationed_pokemon = :total_stationed_pokemon, - total_stationed_gmax = :total_stationed_gmax, - stationed_pokemon = :stationed_pokemon - WHERE id = :id - `, station, - ) - statsCollector.IncDbQuery("update station", err) - if err != nil { - log.Errorf("Update station %s", err) - } - _, _ = res, err - } - - if dbDebugEnabled { - station.changedFields = station.changedFields[:0] - } - station.ClearDirty() - createStationWebhooks(station) - if station.IsNewRecord() { - stationCache.Set(station.Id, station, ttlcache.DefaultTTL) - station.newRecord = false - } -} - -func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, cellId uint64) *Station { - station.SetId(stationProto.Id) - name := stationProto.Name - // NOTE: Some names have more than 255 runes, which won't fit in our - // varchar(255). - if truncateStr, truncated := util.TruncateUTF8(stationProto.Name, 255); truncated { - log.Warnf("truncating name for station id '%s'. Orig name: %s", - stationProto.Id, - stationProto.Name, - ) - name = truncateStr - } - station.SetName(name) - station.SetLat(stationProto.Lat) - station.SetLon(stationProto.Lng) - station.SetStartTime(stationProto.StartTimeMs / 1000) - station.SetEndTime(stationProto.EndTimeMs / 1000) - station.SetCooldownComplete(stationProto.CooldownCompleteMs) - station.SetIsBattleAvailable(stationProto.IsBreadBattleAvailable) - if battleDetails := stationProto.BattleDetails; battleDetails != nil { - station.SetBattleLevel(null.IntFrom(int64(battleDetails.BattleLevel))) - station.SetBattleStart(null.IntFrom(battleDetails.BattleWindowStartMs / 1000)) - station.SetBattleEnd(null.IntFrom(battleDetails.BattleWindowEndMs / 1000)) - if pokemon := battleDetails.BattlePokemon; pokemon != nil { - station.SetBattlePokemonId(null.IntFrom(int64(pokemon.PokemonId))) - station.SetBattlePokemonMove1(null.IntFrom(int64(pokemon.Move1))) - station.SetBattlePokemonMove2(null.IntFrom(int64(pokemon.Move2))) - station.SetBattlePokemonForm(null.IntFrom(int64(pokemon.PokemonDisplay.Form))) - station.SetBattlePokemonCostume(null.IntFrom(int64(pokemon.PokemonDisplay.Costume))) - station.SetBattlePokemonGender(null.IntFrom(int64(pokemon.PokemonDisplay.Gender))) - station.SetBattlePokemonAlignment(null.IntFrom(int64(pokemon.PokemonDisplay.Alignment))) - station.SetBattlePokemonBreadMode(null.IntFrom(int64(pokemon.PokemonDisplay.BreadModeEnum))) - station.SetBattlePokemonStamina(null.IntFrom(int64(pokemon.Stamina))) - station.SetBattlePokemonCpMultiplier(null.FloatFrom(float64(pokemon.CpMultiplier))) - if rewardPokemon := battleDetails.RewardPokemon; rewardPokemon != nil && pokemon.PokemonId != rewardPokemon.PokemonId { - log.Infof("[DYNAMAX] Pokemon reward differs from battle: Battle %v - Reward %v", pokemon, rewardPokemon) - } - } - } - station.SetCellId(int64(cellId)) - return station -} - -func (station *Station) updateFromGetStationedPokemonDetailsOutProto(stationProto *pogo.GetStationedPokemonDetailsOutProto) *Station { - type stationedPokemonDetail struct { - PokemonId int `json:"pokemon_id"` - Form int `json:"form"` - Costume int `json:"costume"` - Gender int `json:"gender"` - Shiny bool `json:"shiny,omitempty"` - TempEvolution int `json:"temp_evolution,omitempty"` - TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` - Alignment int `json:"alignment,omitempty"` - Badge int `json:"badge,omitempty"` - Background *int64 `json:"background,omitempty"` - BreadMode int `json:"bread_mode"` - } - - var stationedPokemon []stationedPokemonDetail - stationedGmax := int64(0) - for _, stationedPokemonDetails := range stationProto.StationedPokemons { - pokemon := stationedPokemonDetails.Pokemon - display := pokemon.PokemonDisplay - stationedPokemon = append(stationedPokemon, stationedPokemonDetail{ - PokemonId: int(pokemon.PokemonId), - Form: int(display.Form), - Costume: int(display.Costume), - Gender: int(display.Gender), - Shiny: display.Shiny, - TempEvolution: int(display.CurrentTempEvolution), - TempEvolutionFinishMs: display.TemporaryEvolutionFinishMs, - Alignment: int(display.Alignment), - Badge: int(display.PokemonBadge), - Background: util.ExtractBackgroundFromDisplay(display), - BreadMode: int(display.BreadModeEnum), - }) - if display.BreadModeEnum == pogo.BreadModeEnum_BREAD_DOUGH_MODE || display.BreadModeEnum == pogo.BreadModeEnum_BREAD_DOUGH_MODE_2 { - stationedGmax++ - } - } - jsonString, _ := json.Marshal(stationedPokemon) - station.SetStationedPokemon(null.StringFrom(string(jsonString))) - station.SetTotalStationedPokemon(null.IntFrom(int64(stationProto.TotalNumStationedPokemon))) - station.SetTotalStationedGmax(null.IntFrom(stationedGmax)) - return station -} - -func (station *Station) resetStationedPokemonFromStationDetailsNotFound() *Station { - jsonString, _ := json.Marshal([]string{}) - station.SetStationedPokemon(null.StringFrom(string(jsonString))) - station.SetTotalStationedPokemon(null.IntFrom(0)) - station.SetTotalStationedGmax(null.IntFrom(0)) - return station -} - -func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto) string { - stationId := request.StationId - - station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) - if err != nil { - log.Printf("Get station %s", err) - return "Error getting station" - } - - if station == nil { - log.Infof("Stationed pokemon details for station %s not found", stationId) - return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) - } - defer unlock() - - station.resetStationedPokemonFromStationDetailsNotFound() - saveStationRecord(ctx, db, station) - return fmt.Sprintf("StationedPokemonDetails %s", stationId) -} - -func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto, stationDetails *pogo.GetStationedPokemonDetailsOutProto) string { - stationId := request.StationId - - station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) - if err != nil { - log.Printf("Get station %s", err) - return "Error getting station" - } - - if station == nil { - log.Infof("Stationed pokemon details for station %s not found", stationId) - return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) - } - defer unlock() - - station.updateFromGetStationedPokemonDetailsOutProto(stationDetails) - saveStationRecord(ctx, db, station) - return fmt.Sprintf("StationedPokemonDetails %s", stationId) -} - -func createStationWebhooks(station *Station) { - old := &station.oldValues - isNew := station.IsNewRecord() - - if isNew || station.BattlePokemonId.Valid && (old.EndTime != station.EndTime || - old.BattleEnd != station.BattleEnd || - old.BattlePokemonId != station.BattlePokemonId || - old.BattlePokemonForm != station.BattlePokemonForm || - old.BattlePokemonCostume != station.BattlePokemonCostume || - old.BattlePokemonGender != station.BattlePokemonGender || - old.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { - stationHook := StationWebhook{ - Id: station.Id, - Latitude: station.Lat, - Longitude: station.Lon, - Name: station.Name, - StartTime: station.StartTime, - EndTime: station.EndTime, - IsBattleAvailable: station.IsBattleAvailable, - BattleLevel: station.BattleLevel, - BattleStart: station.BattleStart, - BattleEnd: station.BattleEnd, - BattlePokemonId: station.BattlePokemonId, - BattlePokemonForm: station.BattlePokemonForm, - BattlePokemonCostume: station.BattlePokemonCostume, - BattlePokemonGender: station.BattlePokemonGender, - BattlePokemonAlignment: station.BattlePokemonAlignment, - BattlePokemonBreadMode: station.BattlePokemonBreadMode, - BattlePokemonMove1: station.BattlePokemonMove1, - BattlePokemonMove2: station.BattlePokemonMove2, - TotalStationedPokemon: station.TotalStationedPokemon, - TotalStationedGmax: station.TotalStationedGmax, - Updated: station.Updated, - } - areas := MatchStatsGeofence(station.Lat, station.Lon) - webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) - statsCollector.UpdateMaxBattleCount(areas, station.BattleLevel.ValueOrZero()) - } -} diff --git a/decoder/station_decode.go b/decoder/station_decode.go new file mode 100644 index 00000000..da341b9a --- /dev/null +++ b/decoder/station_decode.go @@ -0,0 +1,106 @@ +package decoder + +import ( + "encoding/json" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" + "golbat/util" +) + +func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, cellId uint64) *Station { + station.SetId(stationProto.Id) + name := stationProto.Name + // NOTE: Some names have more than 255 runes, which won't fit in our + // varchar(255). + if truncateStr, truncated := util.TruncateUTF8(stationProto.Name, 255); truncated { + log.Warnf("truncating name for station id '%s'. Orig name: %s", + stationProto.Id, + stationProto.Name, + ) + name = truncateStr + } + station.SetName(name) + station.SetLat(stationProto.Lat) + station.SetLon(stationProto.Lng) + station.SetStartTime(stationProto.StartTimeMs / 1000) + station.SetEndTime(stationProto.EndTimeMs / 1000) + station.SetCooldownComplete(stationProto.CooldownCompleteMs) + station.SetIsBattleAvailable(stationProto.IsBreadBattleAvailable) + if battleDetails := stationProto.BattleDetails; battleDetails != nil { + station.SetBattleLevel(null.IntFrom(int64(battleDetails.BattleLevel))) + station.SetBattleStart(null.IntFrom(battleDetails.BattleWindowStartMs / 1000)) + station.SetBattleEnd(null.IntFrom(battleDetails.BattleWindowEndMs / 1000)) + if pokemon := battleDetails.BattlePokemon; pokemon != nil { + station.SetBattlePokemonId(null.IntFrom(int64(pokemon.PokemonId))) + station.SetBattlePokemonMove1(null.IntFrom(int64(pokemon.Move1))) + station.SetBattlePokemonMove2(null.IntFrom(int64(pokemon.Move2))) + station.SetBattlePokemonForm(null.IntFrom(int64(pokemon.PokemonDisplay.Form))) + station.SetBattlePokemonCostume(null.IntFrom(int64(pokemon.PokemonDisplay.Costume))) + station.SetBattlePokemonGender(null.IntFrom(int64(pokemon.PokemonDisplay.Gender))) + station.SetBattlePokemonAlignment(null.IntFrom(int64(pokemon.PokemonDisplay.Alignment))) + station.SetBattlePokemonBreadMode(null.IntFrom(int64(pokemon.PokemonDisplay.BreadModeEnum))) + station.SetBattlePokemonStamina(null.IntFrom(int64(pokemon.Stamina))) + station.SetBattlePokemonCpMultiplier(null.FloatFrom(float64(pokemon.CpMultiplier))) + if rewardPokemon := battleDetails.RewardPokemon; rewardPokemon != nil && pokemon.PokemonId != rewardPokemon.PokemonId { + log.Infof("[DYNAMAX] Pokemon reward differs from battle: Battle %v - Reward %v", pokemon, rewardPokemon) + } + } + } + station.SetCellId(int64(cellId)) + return station +} + +func (station *Station) updateFromGetStationedPokemonDetailsOutProto(stationProto *pogo.GetStationedPokemonDetailsOutProto) *Station { + type stationedPokemonDetail struct { + PokemonId int `json:"pokemon_id"` + Form int `json:"form"` + Costume int `json:"costume"` + Gender int `json:"gender"` + Shiny bool `json:"shiny,omitempty"` + TempEvolution int `json:"temp_evolution,omitempty"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` + Alignment int `json:"alignment,omitempty"` + Badge int `json:"badge,omitempty"` + Background *int64 `json:"background,omitempty"` + BreadMode int `json:"bread_mode"` + } + + var stationedPokemon []stationedPokemonDetail + stationedGmax := int64(0) + for _, stationedPokemonDetails := range stationProto.StationedPokemons { + pokemon := stationedPokemonDetails.Pokemon + display := pokemon.PokemonDisplay + stationedPokemon = append(stationedPokemon, stationedPokemonDetail{ + PokemonId: int(pokemon.PokemonId), + Form: int(display.Form), + Costume: int(display.Costume), + Gender: int(display.Gender), + Shiny: display.Shiny, + TempEvolution: int(display.CurrentTempEvolution), + TempEvolutionFinishMs: display.TemporaryEvolutionFinishMs, + Alignment: int(display.Alignment), + Badge: int(display.PokemonBadge), + Background: util.ExtractBackgroundFromDisplay(display), + BreadMode: int(display.BreadModeEnum), + }) + if display.BreadModeEnum == pogo.BreadModeEnum_BREAD_DOUGH_MODE || display.BreadModeEnum == pogo.BreadModeEnum_BREAD_DOUGH_MODE_2 { + stationedGmax++ + } + } + jsonString, _ := json.Marshal(stationedPokemon) + station.SetStationedPokemon(null.StringFrom(string(jsonString))) + station.SetTotalStationedPokemon(null.IntFrom(int64(stationProto.TotalNumStationedPokemon))) + station.SetTotalStationedGmax(null.IntFrom(stationedGmax)) + return station +} + +func (station *Station) resetStationedPokemonFromStationDetailsNotFound() *Station { + jsonString, _ := json.Marshal([]string{}) + station.SetStationedPokemon(null.StringFrom(string(jsonString))) + station.SetTotalStationedPokemon(null.IntFrom(0)) + station.SetTotalStationedGmax(null.IntFrom(0)) + return station +} diff --git a/decoder/station_process.go b/decoder/station_process.go new file mode 100644 index 00000000..05cda289 --- /dev/null +++ b/decoder/station_process.go @@ -0,0 +1,51 @@ +package decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto) string { + stationId := request.StationId + + station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) + if err != nil { + log.Printf("Get station %s", err) + return "Error getting station" + } + + if station == nil { + log.Infof("Stationed pokemon details for station %s not found", stationId) + return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) + } + defer unlock() + + station.resetStationedPokemonFromStationDetailsNotFound() + saveStationRecord(ctx, db, station) + return fmt.Sprintf("StationedPokemonDetails %s", stationId) +} + +func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto, stationDetails *pogo.GetStationedPokemonDetailsOutProto) string { + stationId := request.StationId + + station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) + if err != nil { + log.Printf("Get station %s", err) + return "Error getting station" + } + + if station == nil { + log.Infof("Stationed pokemon details for station %s not found", stationId) + return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) + } + defer unlock() + + station.updateFromGetStationedPokemonDetailsOutProto(stationDetails) + saveStationRecord(ctx, db, station) + return fmt.Sprintf("StationedPokemonDetails %s", stationId) +} diff --git a/decoder/station_state.go b/decoder/station_state.go new file mode 100644 index 00000000..d4e4e06a --- /dev/null +++ b/decoder/station_state.go @@ -0,0 +1,253 @@ +package decoder + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/webhooks" +) + +type StationWebhook struct { + Id string `json:"id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + IsBattleAvailable bool `json:"is_battle_available"` + BattleLevel null.Int `json:"battle_level"` + BattleStart null.Int `json:"battle_start"` + BattleEnd null.Int `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` + TotalStationedGmax null.Int `json:"total_stationed_gmax"` + Updated int64 `json:"updated"` +} + +func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId string, station *Station) error { + err := db.GeneralDb.GetContext(ctx, station, + `SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon + FROM station WHERE id = ?`, stationId) + statsCollector.IncDbQuery("select station", err) + return err +} + +// peekStationRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekStationRecord(stationId string) (*Station, func(), error) { + if item := stationCache.Get(stationId); item != nil { + station := item.Value() + station.Lock() + return station, func() { station.Unlock() }, nil + } + return nil, nil, nil +} + +// getStationRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + // Check cache first + if item := stationCache.Get(stationId); item != nil { + station := item.Value() + station.Lock() + return station, func() { station.Unlock() }, nil + } + + dbStation := Station{} + err := loadStationFromDatabase(ctx, db, stationId, &dbStation) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbStation.ClearDirty() + + // Atomically cache the loaded Station - if another goroutine raced us, + // we'll get their Station and use that instead (ensuring same mutex) + existingStation, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + return &dbStation + }) + + station := existingStation.Value() + station.Lock() + return station, func() { station.Unlock() }, nil +} + +// getStationRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getStationRecordForUpdate(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + station, unlock, err := getStationRecordReadOnly(ctx, db, stationId) + if err != nil || station == nil { + return nil, nil, err + } + station.snapshotOldValues() + return station, unlock, nil +} + +// getOrCreateStationRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + // Create new Station atomically - function only called if key doesn't exist + stationItem, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + return &Station{Id: stationId, newRecord: true} + }) + + station := stationItem.Value() + station.Lock() + + if station.newRecord { + // We should attempt to load from database + err := loadStationFromDatabase(ctx, db, stationId, station) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + station.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + station.newRecord = false + station.ClearDirty() + } + } + + station.snapshotOldValues() + return station, func() { station.Unlock() }, nil +} + +func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { + now := time.Now().Unix() + + // Skip save if not dirty and was updated recently (15-min debounce) + if !station.IsDirty() && !station.IsNewRecord() { + if station.Updated > now-GetUpdateThreshold(900) { + return + } + } + + station.Updated = now + + if station.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Station", station.Id, station.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, + ` + INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) + VALUES (:id,:lat,:lon,:name,:cell_id,:start_time,:end_time,:cooldown_complete,:is_battle_available,:is_inactive,:updated,:battle_level,:battle_start,:battle_end,:battle_pokemon_id,:battle_pokemon_form,:battle_pokemon_costume,:battle_pokemon_gender,:battle_pokemon_alignment,:battle_pokemon_bread_mode,:battle_pokemon_move_1,:battle_pokemon_move_2,:battle_pokemon_stamina,:battle_pokemon_cp_multiplier,:total_stationed_pokemon,:total_stationed_gmax,:stationed_pokemon) + `, station) + + statsCollector.IncDbQuery("insert station", err) + if err != nil { + log.Errorf("insert station: %s", err) + return + } + _, _ = res, err + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Station", station.Id, station.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, ` + UPDATE station + SET + lat = :lat, + lon = :lon, + name = :name, + cell_id = :cell_id, + start_time = :start_time, + end_time = :end_time, + cooldown_complete = :cooldown_complete, + is_battle_available = :is_battle_available, + is_inactive = :is_inactive, + updated = :updated, + battle_level = :battle_level, + battle_start = :battle_start, + battle_end = :battle_end, + battle_pokemon_id = :battle_pokemon_id, + battle_pokemon_form = :battle_pokemon_form, + battle_pokemon_costume = :battle_pokemon_costume, + battle_pokemon_gender = :battle_pokemon_gender, + battle_pokemon_alignment = :battle_pokemon_alignment, + battle_pokemon_bread_mode = :battle_pokemon_bread_mode, + battle_pokemon_move_1 = :battle_pokemon_move_1, + battle_pokemon_move_2 = :battle_pokemon_move_2, + battle_pokemon_stamina = :battle_pokemon_stamina, + battle_pokemon_cp_multiplier = :battle_pokemon_cp_multiplier, + total_stationed_pokemon = :total_stationed_pokemon, + total_stationed_gmax = :total_stationed_gmax, + stationed_pokemon = :stationed_pokemon + WHERE id = :id + `, station, + ) + statsCollector.IncDbQuery("update station", err) + if err != nil { + log.Errorf("Update station %s", err) + } + _, _ = res, err + } + + if dbDebugEnabled { + station.changedFields = station.changedFields[:0] + } + station.ClearDirty() + createStationWebhooks(station) + if station.IsNewRecord() { + stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + station.newRecord = false + } +} + +func createStationWebhooks(station *Station) { + old := &station.oldValues + isNew := station.IsNewRecord() + + if isNew || station.BattlePokemonId.Valid && (old.EndTime != station.EndTime || + old.BattleEnd != station.BattleEnd || + old.BattlePokemonId != station.BattlePokemonId || + old.BattlePokemonForm != station.BattlePokemonForm || + old.BattlePokemonCostume != station.BattlePokemonCostume || + old.BattlePokemonGender != station.BattlePokemonGender || + old.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { + stationHook := StationWebhook{ + Id: station.Id, + Latitude: station.Lat, + Longitude: station.Lon, + Name: station.Name, + StartTime: station.StartTime, + EndTime: station.EndTime, + IsBattleAvailable: station.IsBattleAvailable, + BattleLevel: station.BattleLevel, + BattleStart: station.BattleStart, + BattleEnd: station.BattleEnd, + BattlePokemonId: station.BattlePokemonId, + BattlePokemonForm: station.BattlePokemonForm, + BattlePokemonCostume: station.BattlePokemonCostume, + BattlePokemonGender: station.BattlePokemonGender, + BattlePokemonAlignment: station.BattlePokemonAlignment, + BattlePokemonBreadMode: station.BattlePokemonBreadMode, + BattlePokemonMove1: station.BattlePokemonMove1, + BattlePokemonMove2: station.BattlePokemonMove2, + TotalStationedPokemon: station.TotalStationedPokemon, + TotalStationedGmax: station.TotalStationedGmax, + Updated: station.Updated, + } + areas := MatchStatsGeofence(station.Lat, station.Lon) + webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) + statsCollector.UpdateMaxBattleCount(areas, station.BattleLevel.ValueOrZero()) + } +} diff --git a/decoder/tappable.go b/decoder/tappable.go index 3a498ef2..57884bfd 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -1,20 +1,9 @@ package decoder import ( - "context" - "database/sql" - "errors" - "fmt" - "strconv" "sync" - "time" - "golbat/db" - "golbat/pogo" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" ) // Tappable struct. @@ -166,278 +155,3 @@ func (ta *Tappable) SetExpireTimestampVerified(v bool) { } } } - -func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.DbDetails, tappable *pogo.ProcessTappableOutProto, request *pogo.ProcessTappableProto, timestampMs int64) { - // update from request - ta.Id = request.EncounterId // Id is primary key, don't track as dirty - location := request.GetLocation() - if spawnPointId := location.GetSpawnpointId(); spawnPointId != "" { - spawnId, err := strconv.ParseInt(spawnPointId, 16, 64) - if err != nil { - panic(err) - } - ta.SetSpawnId(null.IntFrom(spawnId)) - } - if fortId := location.GetFortId(); fortId != "" { - ta.SetFortId(null.StringFrom(fortId)) - } - ta.SetType(request.TappableTypeId) - ta.SetLat(request.LocationHintLat) - ta.SetLon(request.LocationHintLng) - ta.setExpireTimestamp(ctx, db, timestampMs) - - // update from tappable - if encounter := tappable.GetEncounter(); encounter != nil { - // tappable is a Pokèmon, encounter is sent in a separate proto - // we store this to link tappable with Pokèmon from encounter proto - ta.SetEncounter(null.IntFrom(int64(encounter.Pokemon.PokemonId))) - } else if reward := tappable.GetReward(); reward != nil { - for _, lootProto := range reward { - for _, itemProto := range lootProto.GetLootItem() { - switch t := itemProto.Type.(type) { - case *pogo.LootItemProto_Item: - ta.SetItemId(null.IntFrom(int64(t.Item))) - ta.SetCount(null.IntFrom(int64(itemProto.Count))) - case *pogo.LootItemProto_Stardust: - log.Warnf("[TAPPABLE] Reward is Stardust: %t", t.Stardust) - case *pogo.LootItemProto_Pokecoin: - log.Warnf("[TAPPABLE] Reward is Pokecoin: %t", t.Pokecoin) - case *pogo.LootItemProto_PokemonCandy: - log.Warnf("[TAPPABLE] Reward is Pokemon Candy: %v", t.PokemonCandy) - case *pogo.LootItemProto_Experience: - log.Warnf("[TAPPABLE] Reward is Experience: %t", t.Experience) - case *pogo.LootItemProto_PokemonEgg: - log.Warnf("[TAPPABLE] Reward is a Pokemon Egg: %v", t.PokemonEgg) - case *pogo.LootItemProto_AvatarTemplateId: - log.Warnf("[TAPPABLE] Reward is an Avatar Template ID: %v", t.AvatarTemplateId) - case *pogo.LootItemProto_StickerId: - log.Warnf("[TAPPABLE] Reward is a Sticker ID: %s", t.StickerId) - case *pogo.LootItemProto_MegaEnergyPokemonId: - log.Warnf("[TAPPABLE] Reward is Mega Energy Pokemon ID: %v", t.MegaEnergyPokemonId) - case *pogo.LootItemProto_XlCandy: - log.Warnf("[TAPPABLE] Reward is XL Candy: %v", t.XlCandy) - case *pogo.LootItemProto_FollowerPokemon: - log.Warnf("[TAPPABLE] Reward is a Follower Pokemon: %v", t.FollowerPokemon) - case *pogo.LootItemProto_NeutralAvatarTemplateId: - log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Template ID: %v", t.NeutralAvatarTemplateId) - case *pogo.LootItemProto_NeutralAvatarItemTemplate: - log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Item Template: %v", t.NeutralAvatarItemTemplate) - case *pogo.LootItemProto_NeutralAvatarItemDisplay: - log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Item Display: %v", t.NeutralAvatarItemDisplay) - default: - log.Warnf("Unknown or unset Type") - } - } - } - } -} - -func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, timestampMs int64) { - ta.SetExpireTimestampVerified(false) - if spawnId := ta.SpawnId.ValueOrZero(); spawnId != 0 { - spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) - if spawnPoint != nil && spawnPoint.DespawnSec.Valid { - despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) - unlock() - - date := time.Unix(timestampMs/1000, 0) - secondOfHour := date.Second() + date.Minute()*60 - - despawnOffset := despawnSecond - secondOfHour - if despawnOffset < 0 { - despawnOffset += 3600 - } - ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) - ta.SetExpireTimestampVerified(true) - } else { - if unlock != nil { - unlock() - } - ta.setUnknownTimestamp(timestampMs / 1000) - } - } else if fortId := ta.FortId.ValueOrZero(); fortId != "" { - // we don't know any despawn times from lured/fort tappables - ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) - } -} - -func (ta *Tappable) setUnknownTimestamp(now int64) { - if !ta.ExpireTimestamp.Valid { - ta.SetExpireTimestamp(null.IntFrom(now + 20*60)) - } else { - if ta.ExpireTimestamp.Int64 < now { - ta.SetExpireTimestamp(null.IntFrom(now + 10*60)) - } - } -} - -func loadTappableFromDatabase(ctx context.Context, db db.DbDetails, id uint64, tappable *Tappable) error { - err := db.GeneralDb.GetContext(ctx, tappable, - `SELECT id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated - FROM tappable WHERE id = ?`, strconv.FormatUint(id, 10)) - statsCollector.IncDbQuery("select tappable", err) - return err -} - -// peekTappableRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func peekTappableRecord(id uint64) (*Tappable, func(), error) { - if item := tappableCache.Get(id); item != nil { - tappable := item.Value() - tappable.Lock() - return tappable, func() { tappable.Unlock() }, nil - } - return nil, nil, nil -} - -// getTappableRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getTappableRecordReadOnly(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { - // Check cache first - if item := tappableCache.Get(id); item != nil { - tappable := item.Value() - tappable.Lock() - return tappable, func() { tappable.Unlock() }, nil - } - - dbTappable := Tappable{} - err := loadTappableFromDatabase(ctx, db, id, &dbTappable) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbTappable.ClearDirty() - - // Atomically cache the loaded Tappable - if another goroutine raced us, - // we'll get their Tappable and use that instead (ensuring same mutex) - existingTappable, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { - return &dbTappable - }) - - tappable := existingTappable.Value() - tappable.Lock() - return tappable, func() { tappable.Unlock() }, nil -} - -// getOrCreateTappableRecord gets existing or creates new, locked. -// Caller MUST call returned unlock function. -func getOrCreateTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { - // Create new Tappable atomically - function only called if key doesn't exist - tappableItem, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { - return &Tappable{Id: id, newRecord: true} - }) - - tappable := tappableItem.Value() - tappable.Lock() - - if tappable.newRecord { - // We should attempt to load from database - err := loadTappableFromDatabase(ctx, db, id, tappable) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - tappable.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - tappable.newRecord = false - tappable.ClearDirty() - } - } - - return tappable, func() { tappable.Unlock() }, nil -} - -// GetTappableRecord is an exported function for API use. -// Returns a tappable if found, nil if not found. -func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, error) { - tappable, unlock, err := getTappableRecordReadOnly(ctx, db, id) - if err != nil { - return nil, err - } - if tappable == nil { - return nil, nil - } - defer unlock() - return tappable, nil -} - -func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tappable) { - // Skip save if not dirty and not new - if !tappable.IsDirty() && !tappable.IsNewRecord() { - return - } - - now := time.Now().Unix() - tappable.Updated = now - - if tappable.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) - } - res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` - INSERT INTO tappable ( - id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated - ) VALUES ( - "%d", :lat, :lon, :fort_id, :spawn_id, :type, :pokemon_id, :item_id, :count, :expire_timestamp, :expire_timestamp_verified, :updated - ) - `, tappable.Id), tappable) - statsCollector.IncDbQuery("insert tappable", err) - if err != nil { - log.Errorf("insert tappable %d: %s", tappable.Id, err) - return - } - _ = res - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) - } - res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` - UPDATE tappable SET - lat = :lat, - lon = :lon, - fort_id = :fort_id, - spawn_id = :spawn_id, - type = :type, - pokemon_id = :pokemon_id, - item_id = :item_id, - count = :count, - expire_timestamp = :expire_timestamp, - expire_timestamp_verified = :expire_timestamp_verified, - updated = :updated - WHERE id = "%d" - `, tappable.Id), tappable) - statsCollector.IncDbQuery("update tappable", err) - if err != nil { - log.Errorf("update tappable %d: %s", tappable.Id, err) - return - } - _ = res - } - if dbDebugEnabled { - tappable.changedFields = tappable.changedFields[:0] - } - tappable.ClearDirty() - if tappable.IsNewRecord() { - tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) - tappable.newRecord = false - } -} - -func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { - id := request.GetEncounterId() - - tappable, unlock, err := getOrCreateTappableRecord(ctx, db, id) - if err != nil { - log.Printf("getOrCreateTappableRecord: %s", err) - return "Error getting tappable" - } - defer unlock() - - tappable.updateFromProcessTappableProto(ctx, db, tappableDetails, request, timestampMs) - saveTappableRecord(ctx, db, tappable) - return fmt.Sprintf("ProcessTappableOutProto %d", id) -} diff --git a/decoder/tappable_decode.go b/decoder/tappable_decode.go new file mode 100644 index 00000000..2b6f65e9 --- /dev/null +++ b/decoder/tappable_decode.go @@ -0,0 +1,117 @@ +package decoder + +import ( + "context" + "strconv" + "time" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.DbDetails, tappable *pogo.ProcessTappableOutProto, request *pogo.ProcessTappableProto, timestampMs int64) { + // update from request + ta.Id = request.EncounterId // Id is primary key, don't track as dirty + location := request.GetLocation() + if spawnPointId := location.GetSpawnpointId(); spawnPointId != "" { + spawnId, err := strconv.ParseInt(spawnPointId, 16, 64) + if err != nil { + panic(err) + } + ta.SetSpawnId(null.IntFrom(spawnId)) + } + if fortId := location.GetFortId(); fortId != "" { + ta.SetFortId(null.StringFrom(fortId)) + } + ta.SetType(request.TappableTypeId) + ta.SetLat(request.LocationHintLat) + ta.SetLon(request.LocationHintLng) + ta.setExpireTimestamp(ctx, db, timestampMs) + + // update from tappable + if encounter := tappable.GetEncounter(); encounter != nil { + // tappable is a Pokèmon, encounter is sent in a separate proto + // we store this to link tappable with Pokèmon from encounter proto + ta.SetEncounter(null.IntFrom(int64(encounter.Pokemon.PokemonId))) + } else if reward := tappable.GetReward(); reward != nil { + for _, lootProto := range reward { + for _, itemProto := range lootProto.GetLootItem() { + switch t := itemProto.Type.(type) { + case *pogo.LootItemProto_Item: + ta.SetItemId(null.IntFrom(int64(t.Item))) + ta.SetCount(null.IntFrom(int64(itemProto.Count))) + case *pogo.LootItemProto_Stardust: + log.Warnf("[TAPPABLE] Reward is Stardust: %t", t.Stardust) + case *pogo.LootItemProto_Pokecoin: + log.Warnf("[TAPPABLE] Reward is Pokecoin: %t", t.Pokecoin) + case *pogo.LootItemProto_PokemonCandy: + log.Warnf("[TAPPABLE] Reward is Pokemon Candy: %v", t.PokemonCandy) + case *pogo.LootItemProto_Experience: + log.Warnf("[TAPPABLE] Reward is Experience: %t", t.Experience) + case *pogo.LootItemProto_PokemonEgg: + log.Warnf("[TAPPABLE] Reward is a Pokemon Egg: %v", t.PokemonEgg) + case *pogo.LootItemProto_AvatarTemplateId: + log.Warnf("[TAPPABLE] Reward is an Avatar Template ID: %v", t.AvatarTemplateId) + case *pogo.LootItemProto_StickerId: + log.Warnf("[TAPPABLE] Reward is a Sticker ID: %s", t.StickerId) + case *pogo.LootItemProto_MegaEnergyPokemonId: + log.Warnf("[TAPPABLE] Reward is Mega Energy Pokemon ID: %v", t.MegaEnergyPokemonId) + case *pogo.LootItemProto_XlCandy: + log.Warnf("[TAPPABLE] Reward is XL Candy: %v", t.XlCandy) + case *pogo.LootItemProto_FollowerPokemon: + log.Warnf("[TAPPABLE] Reward is a Follower Pokemon: %v", t.FollowerPokemon) + case *pogo.LootItemProto_NeutralAvatarTemplateId: + log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Template ID: %v", t.NeutralAvatarTemplateId) + case *pogo.LootItemProto_NeutralAvatarItemTemplate: + log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Item Template: %v", t.NeutralAvatarItemTemplate) + case *pogo.LootItemProto_NeutralAvatarItemDisplay: + log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Item Display: %v", t.NeutralAvatarItemDisplay) + default: + log.Warnf("Unknown or unset Type") + } + } + } + } +} + +func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, timestampMs int64) { + ta.SetExpireTimestampVerified(false) + if spawnId := ta.SpawnId.ValueOrZero(); spawnId != 0 { + spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) + if spawnPoint != nil && spawnPoint.DespawnSec.Valid { + despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) + unlock() + + date := time.Unix(timestampMs/1000, 0) + secondOfHour := date.Second() + date.Minute()*60 + + despawnOffset := despawnSecond - secondOfHour + if despawnOffset < 0 { + despawnOffset += 3600 + } + ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) + ta.SetExpireTimestampVerified(true) + } else { + if unlock != nil { + unlock() + } + ta.setUnknownTimestamp(timestampMs / 1000) + } + } else if fortId := ta.FortId.ValueOrZero(); fortId != "" { + // we don't know any despawn times from lured/fort tappables + ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) + } +} + +func (ta *Tappable) setUnknownTimestamp(now int64) { + if !ta.ExpireTimestamp.Valid { + ta.SetExpireTimestamp(null.IntFrom(now + 20*60)) + } else { + if ta.ExpireTimestamp.Int64 < now { + ta.SetExpireTimestamp(null.IntFrom(now + 10*60)) + } + } +} diff --git a/decoder/tappable_process.go b/decoder/tappable_process.go new file mode 100644 index 00000000..1001ac5e --- /dev/null +++ b/decoder/tappable_process.go @@ -0,0 +1,26 @@ +package decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { + id := request.GetEncounterId() + + tappable, unlock, err := getOrCreateTappableRecord(ctx, db, id) + if err != nil { + log.Printf("getOrCreateTappableRecord: %s", err) + return "Error getting tappable" + } + defer unlock() + + tappable.updateFromProcessTappableProto(ctx, db, tappableDetails, request, timestampMs) + saveTappableRecord(ctx, db, tappable) + return fmt.Sprintf("ProcessTappableOutProto %d", id) +} diff --git a/decoder/tappable_state.go b/decoder/tappable_state.go new file mode 100644 index 00000000..ff30c42f --- /dev/null +++ b/decoder/tappable_state.go @@ -0,0 +1,157 @@ +package decoder + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strconv" + "time" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/db" +) + +func loadTappableFromDatabase(ctx context.Context, db db.DbDetails, id uint64, tappable *Tappable) error { + err := db.GeneralDb.GetContext(ctx, tappable, + `SELECT id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated + FROM tappable WHERE id = ?`, strconv.FormatUint(id, 10)) + statsCollector.IncDbQuery("select tappable", err) + return err +} + +// PeekTappableRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekTappableRecord(id uint64) (*Tappable, func(), error) { + if item := tappableCache.Get(id); item != nil { + tappable := item.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil + } + return nil, nil, nil +} + +// getTappableRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getTappableRecordReadOnly(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { + // Check cache first + if item := tappableCache.Get(id); item != nil { + tappable := item.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil + } + + dbTappable := Tappable{} + err := loadTappableFromDatabase(ctx, db, id, &dbTappable) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbTappable.ClearDirty() + + // Atomically cache the loaded Tappable - if another goroutine raced us, + // we'll get their Tappable and use that instead (ensuring same mutex) + existingTappable, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { + return &dbTappable + }) + + tappable := existingTappable.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil +} + +// getOrCreateTappableRecord gets existing or creates new, locked. +// Caller MUST call returned unlock function. +func getOrCreateTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { + // Create new Tappable atomically - function only called if key doesn't exist + tappableItem, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { + return &Tappable{Id: id, newRecord: true} + }) + + tappable := tappableItem.Value() + tappable.Lock() + + if tappable.newRecord { + // We should attempt to load from database + err := loadTappableFromDatabase(ctx, db, id, tappable) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + tappable.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + tappable.newRecord = false + tappable.ClearDirty() + } + } + + return tappable, func() { tappable.Unlock() }, nil +} + +func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tappable) { + // Skip save if not dirty and not new + if !tappable.IsDirty() && !tappable.IsNewRecord() { + return + } + + now := time.Now().Unix() + tappable.Updated = now + + if tappable.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } + res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` + INSERT INTO tappable ( + id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated + ) VALUES ( + "%d", :lat, :lon, :fort_id, :spawn_id, :type, :pokemon_id, :item_id, :count, :expire_timestamp, :expire_timestamp_verified, :updated + ) + `, tappable.Id), tappable) + statsCollector.IncDbQuery("insert tappable", err) + if err != nil { + log.Errorf("insert tappable %d: %s", tappable.Id, err) + return + } + _ = res + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } + res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` + UPDATE tappable SET + lat = :lat, + lon = :lon, + fort_id = :fort_id, + spawn_id = :spawn_id, + type = :type, + pokemon_id = :pokemon_id, + item_id = :item_id, + count = :count, + expire_timestamp = :expire_timestamp, + expire_timestamp_verified = :expire_timestamp_verified, + updated = :updated + WHERE id = "%d" + `, tappable.Id), tappable) + statsCollector.IncDbQuery("update tappable", err) + if err != nil { + log.Errorf("update tappable %d: %s", tappable.Id, err) + return + } + _ = res + } + if dbDebugEnabled { + tappable.changedFields = tappable.changedFields[:0] + } + tappable.ClearDirty() + if tappable.IsNewRecord() { + tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) + tappable.newRecord = false + } +} diff --git a/decoder/weather.go b/decoder/weather.go index 7f346761..b75a64a4 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -11,9 +11,9 @@ import ( "golbat/webhooks" "github.com/golang/geo/s2" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) // Weather struct. diff --git a/decoder/weather_iv.go b/decoder/weather_iv.go index 7a85d256..2f55df6b 100644 --- a/decoder/weather_iv.go +++ b/decoder/weather_iv.go @@ -13,8 +13,8 @@ import ( "golbat/pogo" "github.com/golang/geo/s2" + "github.com/guregu/null/v6" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) const masterFileURL = "https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/master/master-latest-rdm.json" diff --git a/go.mod b/go.mod index fc946286..c3b5b86d 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/golang-migrate/migrate/v4 v4.19.1 github.com/golang/geo v0.0.0-20260129164528-943061e2742c github.com/grafana/pyroscope-go v1.2.7 + github.com/guregu/null/v6 v6.0.0 github.com/jellydator/ttlcache/v3 v3.4.0 github.com/jmoiron/sqlx v1.4.0 github.com/knadh/koanf/maps v0.1.2 @@ -21,7 +22,6 @@ require ( github.com/knadh/koanf/providers/file v1.2.1 github.com/knadh/koanf/providers/structs v1.0.0 github.com/knadh/koanf/v2 v2.3.2 - github.com/nmvalera/striped-mutex v0.1.0 github.com/paulmach/orb v0.12.0 github.com/prometheus/client_golang v1.23.2 github.com/puzpuzpuz/xsync/v3 v3.5.1 @@ -32,7 +32,6 @@ require ( github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 - gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index a16aadf3..282db98a 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= +github.com/guregu/null/v6 v6.0.0 h1:N14VRS+4di81i1PXRiprbQJ9EM9gqBa0+KVMeS/QSjQ= +github.com/guregu/null/v6 v6.0.0/go.mod h1:hrMIhIfrOZeLPZhROSn149tpw2gHkidAqxoXNyeX3iQ= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= @@ -161,8 +163,6 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nmvalera/striped-mutex v0.1.0 h1:+fwhVWGAsgkiOqmZGZ0HzDD3p/CEt3S85wf0YnMUpL0= -github.com/nmvalera/striped-mutex v0.1.0/go.mod h1:D22iujuGIgMJ/2xSnQDK0I3kh0t3ZOiao5VPYBbq1O0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -338,8 +338,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg= -gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index bf486171..f838c496 100644 --- a/main.go +++ b/main.go @@ -6,11 +6,18 @@ import ( "net" "net/http" "net/http/pprof" - "strings" "sync" "time" _ "time/tzdata" + "golbat/config" + db2 "golbat/db" + "golbat/decoder" + "golbat/external" + pb "golbat/grpc" + "golbat/stats_collector" + "golbat/webhooks" + "github.com/gin-gonic/gin" "github.com/go-sql-driver/mysql" "github.com/golang-migrate/migrate/v4" @@ -20,16 +27,6 @@ import ( log "github.com/sirupsen/logrus" ginlogrus "github.com/toorop/gin-logrus" "google.golang.org/grpc" - "google.golang.org/protobuf/proto" - - "golbat/config" - db2 "golbat/db" - "golbat/decoder" - "golbat/external" - pb "golbat/grpc" - "golbat/pogo" - "golbat/stats_collector" - "golbat/webhooks" ) var db *sqlx.DB @@ -387,727 +384,3 @@ func main() { log.Info("Golbat exiting!") } - -func decode(ctx context.Context, method int, protoData *ProtoData) { - getMethodName := func(method int, trimString bool) string { - if val, ok := pogo.Method_name[int32(method)]; ok { - if trimString && strings.HasPrefix(val, "METHOD_") { - return strings.TrimPrefix(val, "METHOD_") - } - return val - } - return fmt.Sprintf("#%d", method) - } - - if method != int(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION) && protoData.Level < 30 { - statsCollector.IncDecodeMethods("error", "low_level", getMethodName(method, true)) - log.Debugf("Insufficient Level %d Did not process hook type %s", protoData.Level, pogo.Method(method)) - return - } - - processed := false - ignore := false - start := time.Now() - result := "" - - switch pogo.Method(method) { - case pogo.Method_METHOD_START_INCIDENT: - result = decodeStartIncident(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_INVASION_OPEN_COMBAT_SESSION: - if protoData.Request != nil { - result = decodeOpenInvasion(ctx, protoData.Request, protoData.Data) - processed = true - } - case pogo.Method_METHOD_FORT_DETAILS: - result = decodeFortDetails(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_GET_MAP_OBJECTS: - result = decodeGMO(ctx, protoData, getScanParameters(protoData)) - processed = true - case pogo.Method_METHOD_GYM_GET_INFO: - result = decodeGetGymInfo(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_ENCOUNTER: - if getScanParameters(protoData).ProcessPokemon { - result = decodeEncounter(ctx, protoData.Data, protoData.Account, protoData.TimestampMs) - } - processed = true - case pogo.Method_METHOD_DISK_ENCOUNTER: - result = decodeDiskEncounter(ctx, protoData.Data, protoData.Account) - processed = true - case pogo.Method_METHOD_FORT_SEARCH: - result = decodeQuest(ctx, protoData.Data, protoData.HaveAr) - processed = true - case pogo.Method_METHOD_GET_PLAYER: - ignore = true - case pogo.Method_METHOD_GET_HOLOHOLO_INVENTORY: - ignore = true - case pogo.Method_METHOD_CREATE_COMBAT_CHALLENGE: - ignore = true - case pogo.Method(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION): - if protoData.Request != nil { - result = decodeSocialActionWithRequest(protoData.Request, protoData.Data) - processed = true - } - case pogo.Method_METHOD_GET_MAP_FORTS: - result = decodeGetMapForts(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_GET_ROUTES: - result = decodeGetRoutes(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_GET_CONTEST_DATA: - if getScanParameters(protoData).ProcessPokestops { - // Request helps, but can be decoded without it - result = decodeGetContestData(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_GET_POKEMON_SIZE_CONTEST_ENTRY: - // Request is essential to decode this - if protoData.Request != nil { - if getScanParameters(protoData).ProcessPokestops { - result = decodeGetPokemonSizeContestEntry(ctx, protoData.Request, protoData.Data) - } - processed = true - } - case pogo.Method_METHOD_GET_STATION_DETAILS: - if getScanParameters(protoData).ProcessStations { - // Request is essential to decode this - result = decodeGetStationDetails(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_PROCESS_TAPPABLE: - if getScanParameters(protoData).ProcessTappables { - // Request is essential to decode this - result = decodeTappable(ctx, protoData.Request, protoData.Data, protoData.Account, protoData.TimestampMs) - } - processed = true - case pogo.Method_METHOD_GET_EVENT_RSVPS: - if getScanParameters(protoData).ProcessGyms { - result = decodeGetEventRsvp(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_GET_EVENT_RSVP_COUNT: - if getScanParameters(protoData).ProcessGyms { - result = decodeGetEventRsvpCount(ctx, protoData.Data) - } - processed = true - default: - log.Debugf("Did not know hook type %s", pogo.Method(method)) - } - if !ignore { - elapsed := time.Since(start) - if processed == true { - statsCollector.IncDecodeMethods("ok", "", getMethodName(method, true)) - log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, result) - } else { - log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, "**Did not process**") - statsCollector.IncDecodeMethods("unprocessed", "", getMethodName(method, true)) - } - } -} - -func getScanParameters(protoData *ProtoData) decoder.ScanParameters { - return decoder.FindScanConfiguration(protoData.ScanContext, protoData.Lat, protoData.Lon) -} - -func decodeQuest(ctx context.Context, sDec []byte, haveAr *bool) string { - if haveAr == nil { - statsCollector.IncDecodeQuest("error", "missing_ar_info") - log.Infoln("Cannot determine AR quest - ignoring") - // We should either assume AR quest, or trace inventory like RDM probably - return "No AR quest info" - } - decodedQuest := &pogo.FortSearchOutProto{} - if err := proto.Unmarshal(sDec, decodedQuest); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeQuest("error", "parse") - return "Parse failure" - } - - if decodedQuest.Result != pogo.FortSearchOutProto_SUCCESS { - statsCollector.IncDecodeQuest("error", "non_success") - res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedQuest.Result, - pogo.FortSearchOutProto_Result_name[int32(decodedQuest.Result)]) - return res - } - - return decoder.UpdatePokestopWithQuest(ctx, dbDetails, decodedQuest, *haveAr) - -} - -func decodeSocialActionWithRequest(request []byte, payload []byte) string { - var proxyRequestProto pogo.ProxyRequestProto - - if err := proto.Unmarshal(request, &proxyRequestProto); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeSocialActionWithRequest("error", "request_parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - var proxyResponseProto pogo.ProxyResponseProto - - if err := proto.Unmarshal(payload, &proxyResponseProto); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeSocialActionWithRequest("error", "response_parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED && proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED_AND_REASSIGNED { - statsCollector.IncDecodeSocialActionWithRequest("error", "non_success") - return fmt.Sprintf("unsuccessful proxyResponseProto response %d %s", int(proxyResponseProto.Status), proxyResponseProto.Status) - } - - switch pogo.InternalSocialAction(proxyRequestProto.GetAction()) { - case pogo.InternalSocialAction_SOCIAL_ACTION_LIST_FRIEND_STATUS: - statsCollector.IncDecodeSocialActionWithRequest("ok", "list_friend_status") - return decodeGetFriendDetails(proxyResponseProto.Payload) - case pogo.InternalSocialAction_SOCIAL_ACTION_SEARCH_PLAYER: - statsCollector.IncDecodeSocialActionWithRequest("ok", "search_player") - return decodeSearchPlayer(&proxyRequestProto, proxyResponseProto.Payload) - - } - - statsCollector.IncDecodeSocialActionWithRequest("ok", "unknown") - return fmt.Sprintf("Did not process %s", pogo.InternalSocialAction(proxyRequestProto.GetAction()).String()) -} - -func decodeGetFriendDetails(payload []byte) string { - var getFriendDetailsOutProto pogo.InternalGetFriendDetailsOutProto - getFriendDetailsError := proto.Unmarshal(payload, &getFriendDetailsOutProto) - - if getFriendDetailsError != nil { - statsCollector.IncDecodeGetFriendDetails("error", "parse") - log.Errorf("Failed to parse %s", getFriendDetailsError) - return fmt.Sprintf("Failed to parse %s", getFriendDetailsError) - } - - if getFriendDetailsOutProto.GetResult() != pogo.InternalGetFriendDetailsOutProto_SUCCESS || getFriendDetailsOutProto.GetFriend() == nil { - statsCollector.IncDecodeGetFriendDetails("error", "non_success") - return fmt.Sprintf("unsuccessful get friends details") - } - - failures := 0 - - for _, friend := range getFriendDetailsOutProto.GetFriend() { - player := friend.GetPlayer() - - updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, "", player.GetPlayerId()) - if updatePlayerError != nil { - failures++ - } - } - - statsCollector.IncDecodeGetFriendDetails("ok", "") - return fmt.Sprintf("%d players decoded on %d", len(getFriendDetailsOutProto.GetFriend())-failures, len(getFriendDetailsOutProto.GetFriend())) -} - -func decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byte) string { - var searchPlayerOutProto pogo.InternalSearchPlayerOutProto - searchPlayerOutError := proto.Unmarshal(payload, &searchPlayerOutProto) - - if searchPlayerOutError != nil { - log.Errorf("Failed to parse %s", searchPlayerOutError) - statsCollector.IncDecodeSearchPlayer("error", "parse") - return fmt.Sprintf("Failed to parse %s", searchPlayerOutError) - } - - if searchPlayerOutProto.GetResult() != pogo.InternalSearchPlayerOutProto_SUCCESS || searchPlayerOutProto.GetPlayer() == nil { - statsCollector.IncDecodeSearchPlayer("error", "non_success") - return fmt.Sprintf("unsuccessful search player response") - } - - var searchPlayerProto pogo.InternalSearchPlayerProto - searchPlayerError := proto.Unmarshal(proxyRequestProto.GetPayload(), &searchPlayerProto) - - if searchPlayerError != nil || searchPlayerProto.GetFriendCode() == "" { - statsCollector.IncDecodeSearchPlayer("error", "parse") - return fmt.Sprintf("Failed to parse %s", searchPlayerError) - } - - player := searchPlayerOutProto.GetPlayer() - updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, searchPlayerProto.GetFriendCode(), "") - if updatePlayerError != nil { - statsCollector.IncDecodeSearchPlayer("error", "update") - return fmt.Sprintf("Failed update player %s", updatePlayerError) - } - - statsCollector.IncDecodeSearchPlayer("ok", "") - return fmt.Sprintf("1 player decoded from SearchPlayerProto") -} - -func decodeFortDetails(ctx context.Context, sDec []byte) string { - decodedFort := &pogo.FortDetailsOutProto{} - if err := proto.Unmarshal(sDec, decodedFort); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeFortDetails("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - switch decodedFort.FortType { - case pogo.FortType_CHECKPOINT: - statsCollector.IncDecodeFortDetails("ok", "pokestop") - return decoder.UpdatePokestopRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) - case pogo.FortType_GYM: - statsCollector.IncDecodeFortDetails("ok", "gym") - return decoder.UpdateGymRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) - } - - statsCollector.IncDecodeFortDetails("ok", "unknown") - return "Unknown fort type" -} - -func decodeGetMapForts(ctx context.Context, sDec []byte) string { - decodedMapForts := &pogo.GetMapFortsOutProto{} - if err := proto.Unmarshal(sDec, decodedMapForts); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeGetMapForts("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedMapForts.Status != pogo.GetMapFortsOutProto_SUCCESS { - statsCollector.IncDecodeGetMapForts("error", "non_success") - res := fmt.Sprintf(`GetMapFortsOutProto: Ignored non-success value %d:%s`, decodedMapForts.Status, - pogo.GetMapFortsOutProto_Status_name[int32(decodedMapForts.Status)]) - return res - } - - statsCollector.IncDecodeGetMapForts("ok", "") - var outputString string - processedForts := 0 - - for _, fort := range decodedMapForts.Fort { - status, output := decoder.UpdateFortRecordWithGetMapFortsOutProto(ctx, dbDetails, fort) - if status { - processedForts += 1 - outputString += output + ", " - } - } - - if processedForts > 0 { - return fmt.Sprintf("Updated %d forts: %s", processedForts, outputString) - } - return "No forts updated" -} - -func decodeGetRoutes(ctx context.Context, payload []byte) string { - getRoutesOutProto := &pogo.GetRoutesOutProto{} - if err := proto.Unmarshal(payload, getRoutesOutProto); err != nil { - return fmt.Sprintf("failed to decode GetRoutesOutProto %s", err) - } - - if getRoutesOutProto.Status != pogo.GetRoutesOutProto_SUCCESS { - return fmt.Sprintf("GetRoutesOutProto: Ignored non-success value %d:%s", getRoutesOutProto.Status, getRoutesOutProto.Status.String()) - } - - decodeSuccesses := map[string]bool{} - decodeErrors := map[string]bool{} - - for _, routeMapCell := range getRoutesOutProto.GetRouteMapCell() { - for _, route := range routeMapCell.GetRoute() { - //TODO we need to check the repeated field, for now access last element - routeSubmissionStatus := route.RouteSubmissionStatus[len(route.RouteSubmissionStatus)-1] - if routeSubmissionStatus != nil && routeSubmissionStatus.Status != pogo.RouteSubmissionStatus_PUBLISHED { - log.Warnf("Non published Route found in GetRoutesOutProto, status: %s", routeSubmissionStatus.String()) - continue - } - decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(ctx, dbDetails, route) - if decodeError != nil { - if decodeErrors[route.Id] != true { - decodeErrors[route.Id] = true - } - log.Errorf("Failed to decode route %s", decodeError) - } else if decodeSuccesses[route.Id] != true { - decodeSuccesses[route.Id] = true - } - } - } - - return fmt.Sprintf( - "Decoded %d routes, failed to decode %d routes, from %d cells", - len(decodeSuccesses), - len(decodeErrors), - len(getRoutesOutProto.GetRouteMapCell()), - ) -} - -func decodeGetGymInfo(ctx context.Context, sDec []byte) string { - decodedGymInfo := &pogo.GymGetInfoOutProto{} - if err := proto.Unmarshal(sDec, decodedGymInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeGetGymInfo("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedGymInfo.Result != pogo.GymGetInfoOutProto_SUCCESS { - statsCollector.IncDecodeGetGymInfo("error", "non_success") - res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedGymInfo.Result, - pogo.GymGetInfoOutProto_Result_name[int32(decodedGymInfo.Result)]) - return res - } - - statsCollector.IncDecodeGetGymInfo("ok", "") - return decoder.UpdateGymRecordWithGymInfoProto(ctx, dbDetails, decodedGymInfo) -} - -func decodeEncounter(ctx context.Context, sDec []byte, username string, timestampMs int64) string { - decodedEncounterInfo := &pogo.EncounterOutProto{} - if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeEncounter("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedEncounterInfo.Status != pogo.EncounterOutProto_ENCOUNTER_SUCCESS { - statsCollector.IncDecodeEncounter("error", "non_success") - res := fmt.Sprintf(`EncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Status, - pogo.EncounterOutProto_Status_name[int32(decodedEncounterInfo.Status)]) - return res - } - - statsCollector.IncDecodeEncounter("ok", "") - return decoder.UpdatePokemonRecordWithEncounterProto(ctx, dbDetails, decodedEncounterInfo, username, timestampMs) -} - -func decodeDiskEncounter(ctx context.Context, sDec []byte, username string) string { - decodedEncounterInfo := &pogo.DiskEncounterOutProto{} - if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeDiskEncounter("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedEncounterInfo.Result != pogo.DiskEncounterOutProto_SUCCESS { - statsCollector.IncDecodeDiskEncounter("error", "non_success") - res := fmt.Sprintf(`DiskEncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Result, - pogo.DiskEncounterOutProto_Result_name[int32(decodedEncounterInfo.Result)]) - return res - } - - statsCollector.IncDecodeDiskEncounter("ok", "") - return decoder.UpdatePokemonRecordWithDiskEncounterProto(ctx, dbDetails, decodedEncounterInfo, username) -} - -func decodeStartIncident(ctx context.Context, sDec []byte) string { - decodedIncident := &pogo.StartIncidentOutProto{} - if err := proto.Unmarshal(sDec, decodedIncident); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeStartIncident("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedIncident.Status != pogo.StartIncidentOutProto_SUCCESS { - statsCollector.IncDecodeStartIncident("error", "non_success") - res := fmt.Sprintf(`GiovanniOutProto: Ignored non-success value %d:%s`, decodedIncident.Status, - pogo.StartIncidentOutProto_Status_name[int32(decodedIncident.Status)]) - return res - } - - statsCollector.IncDecodeStartIncident("ok", "") - return decoder.ConfirmIncident(ctx, dbDetails, decodedIncident) -} - -func decodeOpenInvasion(ctx context.Context, request []byte, payload []byte) string { - decodeOpenInvasionRequest := &pogo.OpenInvasionCombatSessionProto{} - - if err := proto.Unmarshal(request, decodeOpenInvasionRequest); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeOpenInvasion("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - if decodeOpenInvasionRequest.IncidentLookup == nil { - return "Invalid OpenInvasionCombatSessionProto received" - } - - decodedOpenInvasionResponse := &pogo.OpenInvasionCombatSessionOutProto{} - if err := proto.Unmarshal(payload, decodedOpenInvasionResponse); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeOpenInvasion("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedOpenInvasionResponse.Status != pogo.InvasionStatus_SUCCESS { - statsCollector.IncDecodeOpenInvasion("error", "non_success") - res := fmt.Sprintf(`InvasionLineupOutProto: Ignored non-success value %d:%s`, decodedOpenInvasionResponse.Status, - pogo.InvasionStatus_Status_name[int32(decodedOpenInvasionResponse.Status)]) - return res - } - - statsCollector.IncDecodeOpenInvasion("ok", "") - return decoder.UpdateIncidentLineup(ctx, dbDetails, decodeOpenInvasionRequest, decodedOpenInvasionResponse) -} - -func decodeGMO(ctx context.Context, protoData *ProtoData, scanParameters decoder.ScanParameters) string { - decodedGmo := &pogo.GetMapObjectsOutProto{} - - if err := proto.Unmarshal(protoData.Data, decodedGmo); err != nil { - statsCollector.IncDecodeGMO("error", "parse") - log.Errorf("Failed to parse %s", err) - } - - if decodedGmo.Status != pogo.GetMapObjectsOutProto_SUCCESS { - statsCollector.IncDecodeGMO("error", "non_success") - res := fmt.Sprintf(`GetMapObjectsOutProto: Ignored non-success value %d:%s`, decodedGmo.Status, - pogo.GetMapObjectsOutProto_Status_name[int32(decodedGmo.Status)]) - return res - } - - var newForts []decoder.RawFortData - var newStations []decoder.RawStationData - var newWildPokemon []decoder.RawWildPokemonData - var newNearbyPokemon []decoder.RawNearbyPokemonData - var newMapPokemon []decoder.RawMapPokemonData - var newMapCells []uint64 - var cellsToBeCleaned []uint64 - - // track forts per cell for memory-based cleanup (only if tracker enabled) - cellForts := make(map[uint64]*decoder.FortTrackerGMOContents) - - if len(decodedGmo.MapCell) == 0 { - return "Skipping GetMapObjectsOutProto: No map cells found" - } - for _, mapCell := range decodedGmo.MapCell { - // initialize cell forts tracking for every map cell (so empty fort lists are seen as "no forts") - cellForts[mapCell.S2CellId] = &decoder.FortTrackerGMOContents{ - Pokestops: make([]string, 0), - Gyms: make([]string, 0), - Timestamp: mapCell.AsOfTimeMs, - } - // always mark this mapCell to be checked for removed forts. Previously only cells with forts were - // added which meant an empty fort list (all forts removed) was never passed to the tracker. - cellsToBeCleaned = append(cellsToBeCleaned, mapCell.S2CellId) - - if isCellNotEmpty(mapCell) { - newMapCells = append(newMapCells, mapCell.S2CellId) - } - - for _, fort := range mapCell.Fort { - newForts = append(newForts, decoder.RawFortData{Cell: mapCell.S2CellId, Data: fort, Timestamp: mapCell.AsOfTimeMs}) - - // track fort by type for memory-based cleanup (only if tracker enabled) - if cf, ok := cellForts[mapCell.S2CellId]; ok { - switch fort.FortType { - case pogo.FortType_GYM: - cf.Gyms = append(cf.Gyms, fort.FortId) - case pogo.FortType_CHECKPOINT: - cf.Pokestops = append(cf.Pokestops, fort.FortId) - } - } - - if fort.ActivePokemon != nil { - newMapPokemon = append(newMapPokemon, decoder.RawMapPokemonData{Cell: mapCell.S2CellId, Data: fort.ActivePokemon, Timestamp: mapCell.AsOfTimeMs}) - } - } - for _, mon := range mapCell.WildPokemon { - newWildPokemon = append(newWildPokemon, decoder.RawWildPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) - } - for _, mon := range mapCell.NearbyPokemon { - newNearbyPokemon = append(newNearbyPokemon, decoder.RawNearbyPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) - } - for _, station := range mapCell.Stations { - newStations = append(newStations, decoder.RawStationData{Cell: mapCell.S2CellId, Data: station}) - } - } - - if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { - decoder.UpdateFortBatch(ctx, dbDetails, scanParameters, newForts) - } - var weatherUpdates []decoder.WeatherUpdate - if scanParameters.ProcessWeather { - weatherUpdates = decoder.UpdateClientWeatherBatch(ctx, dbDetails, decodedGmo.ClientWeather, decodedGmo.MapCell[0].AsOfTimeMs, protoData.Account) - } - if scanParameters.ProcessPokemon { - decoder.UpdatePokemonBatch(ctx, dbDetails, scanParameters, newWildPokemon, newNearbyPokemon, newMapPokemon, decodedGmo.ClientWeather, protoData.Account) - if scanParameters.ProcessWeather && scanParameters.ProactiveIVSwitching { - for _, weatherUpdate := range weatherUpdates { - go func(weatherUpdate decoder.WeatherUpdate) { - decoder.ProactiveIVSwitchSem <- true - defer func() { <-decoder.ProactiveIVSwitchSem }() - decoder.ProactiveIVSwitch(ctx, dbDetails, weatherUpdate, scanParameters.ProactiveIVSwitchingToDB, decodedGmo.MapCell[0].AsOfTimeMs/1000) - }(weatherUpdate) - } - } - } - if scanParameters.ProcessStations { - decoder.UpdateStationBatch(ctx, dbDetails, scanParameters, newStations) - } - - if scanParameters.ProcessCells { - decoder.UpdateClientMapS2CellBatch(ctx, dbDetails, newMapCells) - } - - if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - decoder.CheckRemovedForts(ctx, dbDetails, cellsToBeCleaned, cellForts) - }() - } - - newFortsLen := len(newForts) - newStationsLen := len(newStations) - newWildPokemonLen := len(newWildPokemon) - newNearbyPokemonLen := len(newNearbyPokemon) - newMapPokemonLen := len(newMapPokemon) - newClientWeatherLen := len(decodedGmo.ClientWeather) - newMapCellsLen := len(newMapCells) - - statsCollector.IncDecodeGMO("ok", "") - statsCollector.AddDecodeGMOType("fort", float64(newFortsLen)) - statsCollector.AddDecodeGMOType("station", float64(newStationsLen)) - statsCollector.AddDecodeGMOType("wild_pokemon", float64(newWildPokemonLen)) - statsCollector.AddDecodeGMOType("nearby_pokemon", float64(newNearbyPokemonLen)) - statsCollector.AddDecodeGMOType("map_pokemon", float64(newMapPokemonLen)) - statsCollector.AddDecodeGMOType("weather", float64(newClientWeatherLen)) - statsCollector.AddDecodeGMOType("cell", float64(newMapCellsLen)) - - return fmt.Sprintf("%d cells containing %d forts %d stations %d mon %d nearby", newMapCellsLen, newFortsLen, newStationsLen, newWildPokemonLen, newNearbyPokemonLen) -} - -func isCellNotEmpty(mapCell *pogo.ClientMapCellProto) bool { - return len(mapCell.Stations) > 0 || len(mapCell.Fort) > 0 || len(mapCell.WildPokemon) > 0 || len(mapCell.NearbyPokemon) > 0 || len(mapCell.CatchablePokemon) > 0 -} - -func cellContainsForts(mapCell *pogo.ClientMapCellProto) bool { - return len(mapCell.Fort) > 0 -} - -func decodeGetContestData(ctx context.Context, request []byte, data []byte) string { - var decodedContestData pogo.GetContestDataOutProto - if err := proto.Unmarshal(data, &decodedContestData); err != nil { - log.Errorf("Failed to parse GetContestDataOutProto %s", err) - return fmt.Sprintf("Failed to parse GetContestDataOutProto %s", err) - } - - var decodedContestDataRequest pogo.GetContestDataProto - if request != nil { - if err := proto.Unmarshal(request, &decodedContestDataRequest); err != nil { - log.Errorf("Failed to parse GetContestDataProto %s", err) - return fmt.Sprintf("Failed to parse GetContestDataProto %s", err) - } - } - return decoder.UpdatePokestopWithContestData(ctx, dbDetails, &decodedContestDataRequest, &decodedContestData) -} - -func decodeGetPokemonSizeContestEntry(ctx context.Context, request []byte, data []byte) string { - var decodedPokemonSizeContestEntry pogo.GetPokemonSizeLeaderboardEntryOutProto - if err := proto.Unmarshal(data, &decodedPokemonSizeContestEntry); err != nil { - log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - } - - if decodedPokemonSizeContestEntry.Status != pogo.GetPokemonSizeLeaderboardEntryOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetPokemonSizeLeaderboardEntryOutProto non-success status %s", decodedPokemonSizeContestEntry.Status) - } - - var decodedPokemonSizeContestEntryRequest pogo.GetPokemonSizeLeaderboardEntryProto - if request != nil { - if err := proto.Unmarshal(request, &decodedPokemonSizeContestEntryRequest); err != nil { - log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - } - } - - return decoder.UpdatePokestopWithPokemonSizeContestEntry(ctx, dbDetails, &decodedPokemonSizeContestEntryRequest, &decodedPokemonSizeContestEntry) -} - -func decodeGetStationDetails(ctx context.Context, request []byte, data []byte) string { - var decodedGetStationDetails pogo.GetStationedPokemonDetailsOutProto - if err := proto.Unmarshal(data, &decodedGetStationDetails); err != nil { - log.Errorf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) - return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) - } - - var decodedGetStationDetailsRequest pogo.GetStationedPokemonDetailsProto - if request != nil { - if err := proto.Unmarshal(request, &decodedGetStationDetailsRequest); err != nil { - log.Errorf("Failed to parse GetStationedPokemonDetailsProto %s", err) - return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsProto %s", err) - } - } - - if decodedGetStationDetails.Result == pogo.GetStationedPokemonDetailsOutProto_STATION_NOT_FOUND { - // station without stationed pokemon found, therefore we need to reset the columns - return decoder.ResetStationedPokemonWithStationDetailsNotFound(ctx, dbDetails, &decodedGetStationDetailsRequest) - } else if decodedGetStationDetails.Result != pogo.GetStationedPokemonDetailsOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetStationedPokemonDetailsOutProto non-success status %s", decodedGetStationDetails.Result) - } - - return decoder.UpdateStationWithStationDetails(ctx, dbDetails, &decodedGetStationDetailsRequest, &decodedGetStationDetails) -} - -func decodeTappable(ctx context.Context, request, data []byte, username string, timestampMs int64) string { - var tappable pogo.ProcessTappableOutProto - if err := proto.Unmarshal(data, &tappable); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse ProcessTappableOutProto %s", err) - } - - var tappableRequest pogo.ProcessTappableProto - if request != nil { - if err := proto.Unmarshal(request, &tappableRequest); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse ProcessTappableProto %s", err) - } - } - - if tappable.Status != pogo.ProcessTappableOutProto_SUCCESS { - return fmt.Sprintf("Ignored ProcessTappableOutProto non-success status %s", tappable.Status) - } - var result string - if encounter := tappable.GetEncounter(); encounter != nil { - result = decoder.UpdatePokemonRecordWithTappableEncounter(ctx, dbDetails, &tappableRequest, encounter, username, timestampMs) - } - return result + " " + decoder.UpdateTappable(ctx, dbDetails, &tappableRequest, &tappable, timestampMs) -} - -func decodeGetEventRsvp(ctx context.Context, request []byte, data []byte) string { - var rsvp pogo.GetEventRsvpsOutProto - if err := proto.Unmarshal(data, &rsvp); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpsOutProto %s", err) - } - - var rsvpRequest pogo.GetEventRsvpsProto - if request != nil { - if err := proto.Unmarshal(request, &rsvpRequest); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpsProto %s", err) - } - } - - if rsvp.Status != pogo.GetEventRsvpsOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetEventRsvpsOutProto non-success status %s", rsvp.Status) - } - - switch op := rsvpRequest.EventDetails.(type) { - case *pogo.GetEventRsvpsProto_Raid: - return decoder.UpdateGymRecordWithRsvpProto(ctx, dbDetails, op.Raid, &rsvp) - case *pogo.GetEventRsvpsProto_GmaxBattle: - return "Unsupported GmaxBattle Rsvp received" - } - - return "Failed to parse GetEventRsvpsProto - unknown event type" -} - -func decodeGetEventRsvpCount(ctx context.Context, data []byte) string { - var rsvp pogo.GetEventRsvpCountOutProto - if err := proto.Unmarshal(data, &rsvp); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpCountOutProto %s", err) - } - - if rsvp.Status != pogo.GetEventRsvpCountOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetEventRsvpCountOutProto non-success status %s", rsvp.Status) - } - - var clearLocations []string - for _, rsvpDetails := range rsvp.RsvpDetails { - if rsvpDetails.MaybeCount == 0 && rsvpDetails.GoingCount == 0 { - clearLocations = append(clearLocations, rsvpDetails.LocationId) - decoder.ClearGymRsvp(ctx, dbDetails, rsvpDetails.LocationId) - } - } - - return "Cleared RSVP @ " + strings.Join(clearLocations, ", ") -} diff --git a/routes.go b/routes.go index ff1fda73..0cf7efff 100644 --- a/routes.go +++ b/routes.go @@ -501,14 +501,22 @@ func GetPokestop(c *gin.Context) { return } - c.JSON(http.StatusAccepted, pokestop) + if pokestop == nil { + c.Status(http.StatusNotFound) + return + } + result := decoder.BuildPokestopResult(pokestop) + c.JSON(http.StatusAccepted, result) } func GetGym(c *gin.Context) { gymId := c.Param("gym_id") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - gym, err := decoder.GetGymRecord(ctx, dbDetails, gymId) + gym, unlock, err := decoder.GetGymRecordReadOnly(ctx, dbDetails, gymId) + if unlock != nil { + defer unlock() + } cancel() if err != nil { log.Warnf("GET /api/gym/id/:gym_id/ Error during post retrieve %v", err) @@ -516,7 +524,12 @@ func GetGym(c *gin.Context) { return } - c.JSON(http.StatusAccepted, gym) + if gym == nil { + c.Status(http.StatusNotFound) + return + } + result := decoder.BuildGymResult(gym) + c.JSON(http.StatusAccepted, result) } // POST /api/gym/query @@ -561,23 +574,29 @@ func GetGyms(c *gin.Context) { } if len(ids) == 0 { - c.JSON(http.StatusOK, []decoder.Gym{}) + c.JSON(http.StatusOK, []decoder.ApiGymResult{}) return } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - out := make([]*decoder.Gym, 0, len(ids)) + out := make([]decoder.ApiGymResult, 0, len(ids)) for _, id := range ids { - g, err := decoder.GetGymRecord(ctx, dbDetails, id) + g, unlock, err := decoder.GetGymRecordReadOnly(ctx, dbDetails, id) if err != nil { + if unlock != nil { + unlock() + } log.Warnf("error retrieving gym %s: %v", id, err) c.Status(http.StatusInternalServerError) return } if g != nil { - out = append(out, g) + out = append(out, decoder.BuildGymResult(g)) + } + if unlock != nil { + unlock() } if ctx.Err() != nil { c.Status(http.StatusInternalServerError) @@ -691,13 +710,16 @@ func SearchGyms(c *gin.Context) { return } - out := make([]*decoder.Gym, 0, len(ids)) + out := make([]decoder.ApiGymResult, 0, len(ids)) for _, id := range ids { if id == "" { continue } - g, err := decoder.GetGymRecord(ctx, dbDetails, id) + g, unlock, err := decoder.GetGymRecordReadOnly(ctx, dbDetails, id) if err != nil { + if unlock != nil { + unlock() + } if errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded) { log.Warnf("timed out while fetching %s: %v", id, err) c.Status(http.StatusGatewayTimeout) @@ -708,7 +730,10 @@ func SearchGyms(c *gin.Context) { return } if g != nil { - out = append(out, g) + out = append(out, decoder.BuildGymResult(g)) + } + if unlock != nil { + unlock() } if ctx.Err() != nil { c.Status(http.StatusInternalServerError) @@ -727,16 +752,21 @@ func GetTappable(c *gin.Context) { c.Status(http.StatusBadRequest) return } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - tappable, err := decoder.GetTappableRecord(ctx, dbDetails, tappableId) - cancel() + tappable, unlock, err := decoder.PeekTappableRecord(tappableId) + if unlock != nil { + defer unlock() + } if err != nil { log.Warnf("GET /api/tappable/id/:tappable_id/ Error during post retrieve %v", err) c.Status(http.StatusInternalServerError) return } - - c.JSON(http.StatusAccepted, tappable) + if tappable == nil { + c.Status(http.StatusNotFound) + return + } + result := decoder.BuildTappableResult(tappable) + c.JSON(http.StatusAccepted, result) } func GetDevices(c *gin.Context) { diff --git a/stats_collector/noop.go b/stats_collector/noop.go index e50163ea..1a1ff0fa 100644 --- a/stats_collector/noop.go +++ b/stats_collector/noop.go @@ -1,7 +1,7 @@ package stats_collector import ( - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" "golbat/geo" ) diff --git a/stats_collector/prometheus.go b/stats_collector/prometheus.go index e98ea0ae..93547984 100644 --- a/stats_collector/prometheus.go +++ b/stats_collector/prometheus.go @@ -2,13 +2,14 @@ package stats_collector import ( "database/sql" - "golbat/util" "strconv" "sync" "time" + "golbat/util" + + "github.com/guregu/null/v6" "github.com/prometheus/client_golang/prometheus" - "gopkg.in/guregu/null.v4" "golbat/geo" ) diff --git a/stats_collector/stats_collector.go b/stats_collector/stats_collector.go index de14c3f1..9481c41a 100644 --- a/stats_collector/stats_collector.go +++ b/stats_collector/stats_collector.go @@ -6,8 +6,8 @@ import ( "github.com/Depado/ginprom" "github.com/gin-gonic/gin" + "github.com/guregu/null/v6" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) type StatsCollector interface { From 58c22dbd3a5835b0e2410acb498126aa138ad243 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 31 Jan 2026 14:28:14 +0000 Subject: [PATCH 27/78] Remove json fields, these should not be directly serialised --- decoder/gym.go | 100 +++++++++++++++++++++---------------------- decoder/pokemon.go | 90 +++++++++++++++++++------------------- decoder/pokestop.go | 102 ++++++++++++++++++++++---------------------- decoder/tappable.go | 36 ++++++++-------- 4 files changed, 164 insertions(+), 164 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index c0004247..e966b316 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -9,56 +9,56 @@ import ( // Gym struct. // REMINDER! Keep hasChangesGym updated after making changes type Gym struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - - Id string `db:"id" json:"id"` - Lat float64 `db:"lat" json:"lat"` - Lon float64 `db:"lon" json:"lon"` - Name null.String `db:"name" json:"name"` - Url null.String `db:"url" json:"url"` - LastModifiedTimestamp null.Int `db:"last_modified_timestamp" json:"last_modified_timestamp"` - RaidEndTimestamp null.Int `db:"raid_end_timestamp" json:"raid_end_timestamp"` - RaidSpawnTimestamp null.Int `db:"raid_spawn_timestamp" json:"raid_spawn_timestamp"` - RaidBattleTimestamp null.Int `db:"raid_battle_timestamp" json:"raid_battle_timestamp"` - Updated int64 `db:"updated" json:"updated"` - RaidPokemonId null.Int `db:"raid_pokemon_id" json:"raid_pokemon_id"` - GuardingPokemonId null.Int `db:"guarding_pokemon_id" json:"guarding_pokemon_id"` - GuardingPokemonDisplay null.String `db:"guarding_pokemon_display" json:"guarding_pokemon_display"` - AvailableSlots null.Int `db:"available_slots" json:"available_slots"` - TeamId null.Int `db:"team_id" json:"team_id"` - RaidLevel null.Int `db:"raid_level" json:"raid_level"` - Enabled null.Int `db:"enabled" json:"enabled"` - ExRaidEligible null.Int `db:"ex_raid_eligible" json:"ex_raid_eligible"` - InBattle null.Int `db:"in_battle" json:"in_battle"` - RaidPokemonMove1 null.Int `db:"raid_pokemon_move_1" json:"raid_pokemon_move_1"` - RaidPokemonMove2 null.Int `db:"raid_pokemon_move_2" json:"raid_pokemon_move_2"` - RaidPokemonForm null.Int `db:"raid_pokemon_form" json:"raid_pokemon_form"` - RaidPokemonAlignment null.Int `db:"raid_pokemon_alignment" json:"raid_pokemon_alignment"` - RaidPokemonCp null.Int `db:"raid_pokemon_cp" json:"raid_pokemon_cp"` - RaidIsExclusive null.Int `db:"raid_is_exclusive" json:"raid_is_exclusive"` - CellId null.Int `db:"cell_id" json:"cell_id"` - Deleted bool `db:"deleted" json:"deleted"` - TotalCp null.Int `db:"total_cp" json:"total_cp"` - FirstSeenTimestamp int64 `db:"first_seen_timestamp" json:"first_seen_timestamp"` - RaidPokemonGender null.Int `db:"raid_pokemon_gender" json:"raid_pokemon_gender"` - SponsorId null.Int `db:"sponsor_id" json:"sponsor_id"` - PartnerId null.String `db:"partner_id" json:"partner_id"` - RaidPokemonCostume null.Int `db:"raid_pokemon_costume" json:"raid_pokemon_costume"` - RaidPokemonEvolution null.Int `db:"raid_pokemon_evolution" json:"raid_pokemon_evolution"` - ArScanEligible null.Int `db:"ar_scan_eligible" json:"ar_scan_eligible"` - PowerUpLevel null.Int `db:"power_up_level" json:"power_up_level"` - PowerUpPoints null.Int `db:"power_up_points" json:"power_up_points"` - PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp" json:"power_up_end_timestamp"` - Description null.String `db:"description" json:"description"` - Defenders null.String `db:"defenders" json:"defenders"` - Rsvps null.String `db:"rsvps" json:"rsvps"` - - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving (to db) - internalDirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving (in memory only) - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record - changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) - - oldValues GymOldValues `db:"-" json:"-"` // Old values for webhook comparison + mu sync.Mutex `db:"-"` // Object-level mutex + + Id string `db:"id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + Name null.String `db:"name"` + Url null.String `db:"url"` + LastModifiedTimestamp null.Int `db:"last_modified_timestamp"` + RaidEndTimestamp null.Int `db:"raid_end_timestamp"` + RaidSpawnTimestamp null.Int `db:"raid_spawn_timestamp"` + RaidBattleTimestamp null.Int `db:"raid_battle_timestamp"` + Updated int64 `db:"updated"` + RaidPokemonId null.Int `db:"raid_pokemon_id"` + GuardingPokemonId null.Int `db:"guarding_pokemon_id"` + GuardingPokemonDisplay null.String `db:"guarding_pokemon_display"` + AvailableSlots null.Int `db:"available_slots"` + TeamId null.Int `db:"team_id"` + RaidLevel null.Int `db:"raid_level"` + Enabled null.Int `db:"enabled"` + ExRaidEligible null.Int `db:"ex_raid_eligible"` + InBattle null.Int `db:"in_battle"` + RaidPokemonMove1 null.Int `db:"raid_pokemon_move_1"` + RaidPokemonMove2 null.Int `db:"raid_pokemon_move_2"` + RaidPokemonForm null.Int `db:"raid_pokemon_form"` + RaidPokemonAlignment null.Int `db:"raid_pokemon_alignment"` + RaidPokemonCp null.Int `db:"raid_pokemon_cp"` + RaidIsExclusive null.Int `db:"raid_is_exclusive"` + CellId null.Int `db:"cell_id"` + Deleted bool `db:"deleted"` + TotalCp null.Int `db:"total_cp"` + FirstSeenTimestamp int64 `db:"first_seen_timestamp"` + RaidPokemonGender null.Int `db:"raid_pokemon_gender"` + SponsorId null.Int `db:"sponsor_id"` + PartnerId null.String `db:"partner_id"` + RaidPokemonCostume null.Int `db:"raid_pokemon_costume"` + RaidPokemonEvolution null.Int `db:"raid_pokemon_evolution"` + ArScanEligible null.Int `db:"ar_scan_eligible"` + PowerUpLevel null.Int `db:"power_up_level"` + PowerUpPoints null.Int `db:"power_up_points"` + PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp"` + Description null.String `db:"description"` + Defenders null.String `db:"defenders"` + Rsvps null.String `db:"rsvps"` + + dirty bool `db:"-"` // Not persisted - tracks if object needs saving (to db) + internalDirty bool `db:"-"` // Not persisted - tracks if object needs saving (in memory only) + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) + + oldValues GymOldValues `db:"-"` // Old values for webhook comparison } // GymOldValues holds old field values for webhook comparison (populated when loading from cache/DB) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index cc69bdb6..b4b9973c 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -17,55 +17,55 @@ import ( // // FirstSeenTimestamp: This field is used in IsNewRecord. It should only be set in savePokemonRecord. type Pokemon struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - - Id uint64 `db:"id" json:"id,string"` - PokestopId null.String `db:"pokestop_id" json:"pokestop_id"` - SpawnId null.Int `db:"spawn_id" json:"spawn_id"` - Lat float64 `db:"lat" json:"lat"` - Lon float64 `db:"lon" json:"lon"` - Weight null.Float `db:"weight" json:"weight"` - Size null.Int `db:"size" json:"size"` - Height null.Float `db:"height" json:"height"` - ExpireTimestamp null.Int `db:"expire_timestamp" json:"expire_timestamp"` - Updated null.Int `db:"updated" json:"updated"` - PokemonId int16 `db:"pokemon_id" json:"pokemon_id"` - Move1 null.Int `db:"move_1" json:"move_1"` - Move2 null.Int `db:"move_2" json:"move_2"` - Gender null.Int `db:"gender" json:"gender"` - Cp null.Int `db:"cp" json:"cp"` - AtkIv null.Int `db:"atk_iv" json:"atk_iv"` - DefIv null.Int `db:"def_iv" json:"def_iv"` - StaIv null.Int `db:"sta_iv" json:"sta_iv"` - GolbatInternal []byte `db:"golbat_internal" json:"golbat_internal"` - Iv null.Float `db:"iv" json:"iv"` - Form null.Int `db:"form" json:"form"` - Level null.Int `db:"level" json:"level"` - IsStrong null.Bool `db:"strong" json:"strong"` - Weather null.Int `db:"weather" json:"weather"` - Costume null.Int `db:"costume" json:"costume"` - FirstSeenTimestamp int64 `db:"first_seen_timestamp" json:"first_seen_timestamp"` - Changed int64 `db:"changed" json:"changed"` - CellId null.Int `db:"cell_id" json:"cell_id"` - ExpireTimestampVerified bool `db:"expire_timestamp_verified" json:"expire_timestamp_verified"` - DisplayPokemonId null.Int `db:"display_pokemon_id" json:"display_pokemon_id"` - IsDitto bool `db:"is_ditto" json:"is_ditto"` - SeenType null.String `db:"seen_type" json:"seen_type"` - Shiny null.Bool `db:"shiny" json:"shiny"` - Username null.String `db:"username" json:"username"` - Capture1 null.Float `db:"capture_1" json:"capture_1"` - Capture2 null.Float `db:"capture_2" json:"capture_2"` - Capture3 null.Float `db:"capture_3" json:"capture_3"` - Pvp null.String `db:"pvp" json:"pvp"` - IsEvent int8 `db:"is_event" json:"is_event"` + mu sync.Mutex `db:"-"` // Object-level mutex + + Id uint64 `db:"id"` + PokestopId null.String `db:"pokestop_id"` + SpawnId null.Int `db:"spawn_id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + Weight null.Float `db:"weight"` + Size null.Int `db:"size"` + Height null.Float `db:"height"` + ExpireTimestamp null.Int `db:"expire_timestamp"` + Updated null.Int `db:"updated"` + PokemonId int16 `db:"pokemon_id"` + Move1 null.Int `db:"move_1"` + Move2 null.Int `db:"move_2"` + Gender null.Int `db:"gender"` + Cp null.Int `db:"cp"` + AtkIv null.Int `db:"atk_iv"` + DefIv null.Int `db:"def_iv"` + StaIv null.Int `db:"sta_iv"` + GolbatInternal []byte `db:"golbat_internal"` + Iv null.Float `db:"iv"` + Form null.Int `db:"form"` + Level null.Int `db:"level"` + IsStrong null.Bool `db:"strong"` + Weather null.Int `db:"weather"` + Costume null.Int `db:"costume"` + FirstSeenTimestamp int64 `db:"first_seen_timestamp"` + Changed int64 `db:"changed"` + CellId null.Int `db:"cell_id"` + ExpireTimestampVerified bool `db:"expire_timestamp_verified"` + DisplayPokemonId null.Int `db:"display_pokemon_id"` + IsDitto bool `db:"is_ditto"` + SeenType null.String `db:"seen_type"` + Shiny null.Bool `db:"shiny"` + Username null.String `db:"username"` + Capture1 null.Float `db:"capture_1"` + Capture2 null.Float `db:"capture_2"` + Capture3 null.Float `db:"capture_3"` + Pvp null.String `db:"pvp"` + IsEvent int8 `db:"is_event"` internal grpc.PokemonInternal - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` - changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) - oldValues PokemonOldValues `db:"-" json:"-"` // Old values for webhook comparison and stats + oldValues PokemonOldValues `db:"-"` // Old values for webhook comparison and stats } // PokemonOldValues holds old field values for webhook comparison, stats, and R-tree updates diff --git a/decoder/pokestop.go b/decoder/pokestop.go index cad7e262..829ab153 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -8,57 +8,57 @@ import ( // Pokestop struct. type Pokestop struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - - Id string `db:"id" json:"id"` - Lat float64 `db:"lat" json:"lat"` - Lon float64 `db:"lon" json:"lon"` - Name null.String `db:"name" json:"name"` - Url null.String `db:"url" json:"url"` - LureExpireTimestamp null.Int `db:"lure_expire_timestamp" json:"lure_expire_timestamp"` - LastModifiedTimestamp null.Int `db:"last_modified_timestamp" json:"last_modified_timestamp"` - Updated int64 `db:"updated" json:"updated"` - Enabled null.Bool `db:"enabled" json:"enabled"` - QuestType null.Int `db:"quest_type" json:"quest_type"` - QuestTimestamp null.Int `db:"quest_timestamp" json:"quest_timestamp"` - QuestTarget null.Int `db:"quest_target" json:"quest_target"` - QuestConditions null.String `db:"quest_conditions" json:"quest_conditions"` - QuestRewards null.String `db:"quest_rewards" json:"quest_rewards"` - QuestTemplate null.String `db:"quest_template" json:"quest_template"` - QuestTitle null.String `db:"quest_title" json:"quest_title"` - QuestExpiry null.Int `db:"quest_expiry" json:"quest_expiry"` - CellId null.Int `db:"cell_id" json:"cell_id"` - Deleted bool `db:"deleted" json:"deleted"` - LureId int16 `db:"lure_id" json:"lure_id"` - FirstSeenTimestamp int16 `db:"first_seen_timestamp" json:"first_seen_timestamp"` - SponsorId null.Int `db:"sponsor_id" json:"sponsor_id"` - PartnerId null.String `db:"partner_id" json:"partner_id"` - ArScanEligible null.Int `db:"ar_scan_eligible" json:"ar_scan_eligible"` // is an 8 - PowerUpLevel null.Int `db:"power_up_level" json:"power_up_level"` - PowerUpPoints null.Int `db:"power_up_points" json:"power_up_points"` - PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp" json:"power_up_end_timestamp"` - AlternativeQuestType null.Int `db:"alternative_quest_type" json:"alternative_quest_type"` - AlternativeQuestTimestamp null.Int `db:"alternative_quest_timestamp" json:"alternative_quest_timestamp"` - AlternativeQuestTarget null.Int `db:"alternative_quest_target" json:"alternative_quest_target"` - AlternativeQuestConditions null.String `db:"alternative_quest_conditions" json:"alternative_quest_conditions"` - AlternativeQuestRewards null.String `db:"alternative_quest_rewards" json:"alternative_quest_rewards"` - AlternativeQuestTemplate null.String `db:"alternative_quest_template" json:"alternative_quest_template"` - AlternativeQuestTitle null.String `db:"alternative_quest_title" json:"alternative_quest_title"` - AlternativeQuestExpiry null.Int `db:"alternative_quest_expiry" json:"alternative_quest_expiry"` - Description null.String `db:"description" json:"description"` - ShowcaseFocus null.String `db:"showcase_focus" json:"showcase_focus"` - ShowcasePokemon null.Int `db:"showcase_pokemon_id" json:"showcase_pokemon_id"` - ShowcasePokemonForm null.Int `db:"showcase_pokemon_form_id" json:"showcase_pokemon_form_id"` - ShowcasePokemonType null.Int `db:"showcase_pokemon_type_id" json:"showcase_pokemon_type_id"` - ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard" json:"showcase_ranking_standard"` - ShowcaseExpiry null.Int `db:"showcase_expiry" json:"showcase_expiry"` - ShowcaseRankings null.String `db:"showcase_rankings" json:"showcase_rankings"` - - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record - changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) - - oldValues PokestopOldValues `db:"-" json:"-"` // Old values for webhook comparison + mu sync.Mutex `db:"-"` // Object-level mutex + + Id string `db:"id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + Name null.String `db:"name"` + Url null.String `db:"url"` + LureExpireTimestamp null.Int `db:"lure_expire_timestamp"` + LastModifiedTimestamp null.Int `db:"last_modified_timestamp"` + Updated int64 `db:"updated"` + Enabled null.Bool `db:"enabled"` + QuestType null.Int `db:"quest_type"` + QuestTimestamp null.Int `db:"quest_timestamp"` + QuestTarget null.Int `db:"quest_target"` + QuestConditions null.String `db:"quest_conditions"` + QuestRewards null.String `db:"quest_rewards"` + QuestTemplate null.String `db:"quest_template"` + QuestTitle null.String `db:"quest_title"` + QuestExpiry null.Int `db:"quest_expiry"` + CellId null.Int `db:"cell_id"` + Deleted bool `db:"deleted"` + LureId int16 `db:"lure_id"` + FirstSeenTimestamp int16 `db:"first_seen_timestamp"` + SponsorId null.Int `db:"sponsor_id"` + PartnerId null.String `db:"partner_id"` + ArScanEligible null.Int `db:"ar_scan_eligible"` // is an 8 + PowerUpLevel null.Int `db:"power_up_level"` + PowerUpPoints null.Int `db:"power_up_points"` + PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp"` + AlternativeQuestType null.Int `db:"alternative_quest_type"` + AlternativeQuestTimestamp null.Int `db:"alternative_quest_timestamp"` + AlternativeQuestTarget null.Int `db:"alternative_quest_target"` + AlternativeQuestConditions null.String `db:"alternative_quest_conditions"` + AlternativeQuestRewards null.String `db:"alternative_quest_rewards"` + AlternativeQuestTemplate null.String `db:"alternative_quest_template"` + AlternativeQuestTitle null.String `db:"alternative_quest_title"` + AlternativeQuestExpiry null.Int `db:"alternative_quest_expiry"` + Description null.String `db:"description"` + ShowcaseFocus null.String `db:"showcase_focus"` + ShowcasePokemon null.Int `db:"showcase_pokemon_id"` + ShowcasePokemonForm null.Int `db:"showcase_pokemon_form_id"` + ShowcasePokemonType null.Int `db:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `db:"showcase_expiry"` + ShowcaseRankings null.String `db:"showcase_rankings"` + + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) + + oldValues PokestopOldValues `db:"-"` // Old values for webhook comparison } // PokestopOldValues holds old field values for webhook comparison (populated when loading from cache/DB) diff --git a/decoder/tappable.go b/decoder/tappable.go index 57884bfd..3ab8d218 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -9,24 +9,24 @@ import ( // Tappable struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Tappable struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - - Id uint64 `db:"id" json:"id"` - Lat float64 `db:"lat" json:"lat"` - Lon float64 `db:"lon" json:"lon"` - FortId null.String `db:"fort_id" json:"fort_id"` // either fortId or spawnpointId are given - SpawnId null.Int `db:"spawn_id" json:"spawn_id"` - Type string `db:"type" json:"type"` - Encounter null.Int `db:"pokemon_id" json:"pokemon_id"` - ItemId null.Int `db:"item_id" json:"item_id"` - Count null.Int `db:"count" json:"count"` - ExpireTimestamp null.Int `db:"expire_timestamp" json:"expire_timestamp"` - ExpireTimestampVerified bool `db:"expire_timestamp_verified" json:"expire_timestamp_verified"` - Updated int64 `db:"updated" json:"updated"` - - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record - changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) + mu sync.Mutex `db:"-"` // Object-level mutex + + Id uint64 `db:"id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + FortId null.String `db:"fort_id"` // either fortId or spawnpointId are given + SpawnId null.Int `db:"spawn_id"` + Type string `db:"type"` + Encounter null.Int `db:"pokemon_id"` + ItemId null.Int `db:"item_id"` + Count null.Int `db:"count"` + ExpireTimestamp null.Int `db:"expire_timestamp"` + ExpireTimestampVerified bool `db:"expire_timestamp_verified"` + Updated int64 `db:"updated"` + + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) } // IsDirty returns true if any field has been modified From b69b5acd813ac74b65ec7acbf68ec018f249bb32 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 31 Jan 2026 14:48:20 +0000 Subject: [PATCH 28/78] Claude review items --- decoder/incident.go | 10 +++++----- decoder/pending_pokemon.go | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/decoder/incident.go b/decoder/incident.go index 7149a978..049edf7e 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -3,13 +3,13 @@ package decoder import ( "sync" - null "github.com/guregu/null/v6" + "github.com/guregu/null/v6" ) // Incident struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Incident struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + mu sync.Mutex `db:"-"` // Object-level mutex Id string `db:"id"` PokestopId string `db:"pokestop_id"` @@ -27,10 +27,10 @@ type Incident struct { Slot3PokemonId null.Int `db:"slot_3_pokemon_id"` Slot3Form null.Int `db:"slot_3_form"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record - oldValues IncidentOldValues `db:"-" json:"-"` // Old values for webhook comparison + oldValues IncidentOldValues `db:"-"` // Old values for webhook comparison } // IncidentOldValues holds old field values for webhook comparison and stats diff --git a/decoder/pending_pokemon.go b/decoder/pending_pokemon.go index 60c1153f..39042db3 100644 --- a/decoder/pending_pokemon.go +++ b/decoder/pending_pokemon.go @@ -124,6 +124,12 @@ func (q *PokemonPendingQueue) StartSweeper(ctx context.Context, interval time.Du // processExpired handles pokemon that didn't receive an encounter within the timeout func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.DbDetails, expired []*PendingPokemon) { for _, p := range expired { + // Check for shutdown signal between iterations + if ctx.Err() != nil { + log.Debug("Context cancelled, stopping expired pokemon processing") + return + } + processCtx, cancel := context.WithTimeout(ctx, 3*time.Second) pokemon, unlock, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) From 1452c1fd6403d9538b4799b27d95f4f8b223861c Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 17:00:30 +0000 Subject: [PATCH 29/78] Move config options into tuning --- config/config.go | 50 ++++++++++++++++++++++++------------------------ config/reader.go | 8 ++++---- decoder/main.go | 4 ++-- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/config/config.go b/config/config.go index fb66aa37..ab2423d4 100644 --- a/config/config.go +++ b/config/config.go @@ -7,27 +7,25 @@ import ( ) type configDefinition struct { - Port int `koanf:"port"` - GrpcPort int `koanf:"grpc_port"` - Webhooks []Webhook `koanf:"webhooks"` - Database database `koanf:"database"` - Logging logging `koanf:"logging"` - Sentry sentry `koanf:"sentry"` - Pyroscope pyroscope `koanf:"pyroscope"` - Prometheus Prometheus `koanf:"prometheus"` - PokemonMemoryOnly bool `koanf:"pokemon_memory_only"` - PokemonInternalToDb bool `koanf:"pokemon_internal_to_db"` - TestFortInMemory bool `koanf:"test_fort_in_memory"` - Cleanup cleanup `koanf:"cleanup"` - RawBearer string `koanf:"raw_bearer"` - ApiSecret string `koanf:"api_secret"` - Pvp pvp `koanf:"pvp"` - Koji koji `koanf:"koji"` - Tuning tuning `koanf:"tuning"` - Weather weather `koanf:"weather"` - ScanRules []scanRule `koanf:"scan_rules"` - MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` - ReduceUpdates bool `koanf:"reduce_updates"` + Port int `koanf:"port"` + GrpcPort int `koanf:"grpc_port"` + Webhooks []Webhook `koanf:"webhooks"` + Database database `koanf:"database"` + Logging logging `koanf:"logging"` + Sentry sentry `koanf:"sentry"` + Pyroscope pyroscope `koanf:"pyroscope"` + Prometheus Prometheus `koanf:"prometheus"` + PokemonMemoryOnly bool `koanf:"pokemon_memory_only"` + PokemonInternalToDb bool `koanf:"pokemon_internal_to_db"` + TestFortInMemory bool `koanf:"test_fort_in_memory"` + Cleanup cleanup `koanf:"cleanup"` + RawBearer string `koanf:"raw_bearer"` + ApiSecret string `koanf:"api_secret"` + Pvp pvp `koanf:"pvp"` + Koji koji `koanf:"koji"` + Tuning tuning `koanf:"tuning"` + Weather weather `koanf:"weather"` + ScanRules []scanRule `koanf:"scan_rules"` } func (configDefinition configDefinition) GetWebhookInterval() time.Duration { @@ -121,10 +119,12 @@ type database struct { } type tuning struct { - ExtendedTimeout bool `koanf:"extended_timeout"` - MaxPokemonResults int `koanf:"max_pokemon_results"` - MaxPokemonDistance float64 `koanf:"max_pokemon_distance"` - ProfileRoutes bool `koanf:"profile_routes"` + ExtendedTimeout bool `koanf:"extended_timeout"` + MaxPokemonResults int `koanf:"max_pokemon_results"` + MaxPokemonDistance float64 `koanf:"max_pokemon_distance"` + ProfileRoutes bool `koanf:"profile_routes"` + MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` + ReduceUpdates bool `koanf:"reduce_updates"` } type scanRule struct { diff --git a/config/reader.go b/config/reader.go index e0daa36c..57935895 100644 --- a/config/reader.go +++ b/config/reader.go @@ -51,8 +51,10 @@ func ReadConfig() (configDefinition, error) { MaxPool: 100, }, Tuning: tuning{ - MaxPokemonResults: 3000, - MaxPokemonDistance: 100, + MaxPokemonResults: 3000, + MaxPokemonDistance: 100, + MaxConcurrentProactiveIVSwitch: 6, + ReduceUpdates: false, }, Weather: weather{ ProactiveIVSwitching: true, @@ -61,8 +63,6 @@ func ReadConfig() (configDefinition, error) { Pvp: pvp{ LevelCaps: []int{50, 51}, }, - MaxConcurrentProactiveIVSwitch: 6, - ReduceUpdates: false, }, "koanf"), nil) if defaultErr != nil { fmt.Println(fmt.Errorf("failed to load default config: %w", defaultErr)) diff --git a/decoder/main.go b/decoder/main.go index 51dc4e56..b1e2b589 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -77,7 +77,7 @@ func init() { } func InitProactiveIVSwitchSem() { - ProactiveIVSwitchSem = make(chan bool, config.Config.MaxConcurrentProactiveIVSwitch) + ProactiveIVSwitchSem = make(chan bool, config.Config.Tuning.MaxConcurrentProactiveIVSwitch) } type gohbemLogger struct{} @@ -258,7 +258,7 @@ func SetStatsCollector(collector stats_collector.StatsCollector) { // debounce/last-seen threshold. Pass the default seconds for normal operation // If ReduceUpdates is enabled in the loaded config.Config, this returns 43200 (12 hours). func GetUpdateThreshold(defaultSeconds int64) int64 { - if config.Config.ReduceUpdates { + if config.Config.Tuning.ReduceUpdates { return 43200 // 12 hours } return defaultSeconds From edf6d76e6a5834e02b3f9384d05b32419441e07f Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 21:34:45 +0000 Subject: [PATCH 30/78] Some fields weren't being updated through SetXX --- decoder/gym.go | 10 ++++++++++ decoder/gym_state.go | 2 +- decoder/incident.go | 7 +++++++ decoder/incident_state.go | 2 +- decoder/player.go | 9 ++++++++- decoder/pokemon.go | 20 ++++++++++++++++++++ decoder/pokemon_state.go | 4 ++-- decoder/pokestop.go | 10 ++++++++++ decoder/pokestop_state.go | 2 +- decoder/routes.go | 10 ++++++++++ decoder/routes_state.go | 2 +- decoder/spawnpoint.go | 20 +++++++++++++++++--- decoder/station.go | 10 ++++++++++ decoder/station_state.go | 2 +- decoder/tappable.go | 10 ++++++++++ decoder/tappable_state.go | 2 +- 16 files changed, 110 insertions(+), 12 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index e966b316..b841b730 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -561,3 +561,13 @@ func (gym *Gym) SetRsvps(v null.String) { } } } + +func (gym *Gym) SetUpdated(v int64) { + if gym.Updated != v { + gym.Updated = v + gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Updated") + } + } +} diff --git a/decoder/gym_state.go b/decoder/gym_state.go index ce235085..e646ac1a 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -333,7 +333,7 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { return } } - gym.Updated = now + gym.SetUpdated(now) if gym.IsDirty() { if gym.IsNewRecord() { diff --git a/decoder/incident.go b/decoder/incident.go index 049edf7e..707ed5ee 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -213,3 +213,10 @@ func (incident *Incident) SetSlot3Form(v null.Int) { incident.dirty = true } } + +func (incident *Incident) SetUpdated(v int64) { + if incident.Updated != v { + incident.Updated = v + incident.dirty = true + } +} diff --git a/decoder/incident_state.go b/decoder/incident_state.go index d3b8d80f..c9da5ba7 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -111,7 +111,7 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident return } - incident.Updated = time.Now().Unix() + incident.SetUpdated(time.Now().Unix()) if incident.IsNewRecord() { res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ diff --git a/decoder/player.go b/decoder/player.go index 01dea73f..c24054e3 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -633,6 +633,13 @@ func (p *Player) SetCaughtFairy(v null.Int) { } } +func (p *Player) SetLastSeen(v int64) { + if p.LastSeen != v { + p.LastSeen = v + p.dirty = true + } +} + var badgeTypeToPlayerKey = map[pogo.HoloBadgeType]string{ //pogo.HoloBadgeType_BADGE_TRAVEL_KM: "KmWalked", pogo.HoloBadgeType_BADGE_POKEDEX_ENTRIES: "DexGen1", @@ -782,7 +789,7 @@ func savePlayerRecord(db db.DbDetails, player *Player) { return } - player.LastSeen = time.Now().Unix() + player.SetLastSeen(time.Now().Unix()) if player.IsNewRecord() { _, err := db.GeneralDb.NamedExec( diff --git a/decoder/pokemon.go b/decoder/pokemon.go index b4b9973c..81e0f0f8 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -462,3 +462,23 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { } } } + +func (pokemon *Pokemon) SetUpdated(v null.Int) { + if pokemon.Updated != v { + pokemon.Updated = v + pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Updated") + } + } +} + +func (pokemon *Pokemon) SetChanged(v int64) { + if pokemon.Changed != v { + pokemon.Changed = v + pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Changed") + } + } +} diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index e135dbbe..286dcf44 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -190,9 +190,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po pokemon.FirstSeenTimestamp = now } - pokemon.Updated = null.IntFrom(now) + pokemon.SetUpdated(null.IntFrom(now)) if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId || pokemon.oldValues.Cp != pokemon.Cp { - pokemon.Changed = now + pokemon.SetChanged(now) } changePvpField := false diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 829ab153..3289cfe4 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -576,3 +576,13 @@ func (p *Pokestop) SetShowcaseRankings(v null.String) { } } } + +func (p *Pokestop) SetUpdated(v int64) { + if p.Updated != v { + p.Updated = v + p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Updated") + } + } +} diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index f2116179..9fced113 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -276,7 +276,7 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop return } } - pokestop.Updated = now + pokestop.SetUpdated(now) if pokestop.IsNewRecord() { if dbDebugEnabled { diff --git a/decoder/routes.go b/decoder/routes.go index 214d7812..2a6aaf75 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -280,3 +280,13 @@ func (r *Route) SetWaypoints(v string) { } } } + +func (r *Route) SetUpdated(v int64) { + if r.Updated != v { + r.Updated = v + r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Updated") + } + } +} diff --git a/decoder/routes_state.go b/decoder/routes_state.go index 8538cc86..38a0d918 100644 --- a/decoder/routes_state.go +++ b/decoder/routes_state.go @@ -112,7 +112,7 @@ func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { } } - route.Updated = time.Now().Unix() + route.SetUpdated(time.Now().Unix()) if route.IsNewRecord() { if dbDebugEnabled { diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index f858cc85..9a3a4a8a 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -119,6 +119,20 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { } } +func (s *Spawnpoint) SetUpdated(v int64) { + if s.Updated != v { + s.Updated = v + s.dirty = true + } +} + +func (s *Spawnpoint) SetLastSeen(v int64) { + if s.LastSeen != v { + s.LastSeen = v + s.dirty = true + } +} + func loadSpawnpointFromDatabase(ctx context.Context, db db.DbDetails, spawnpointId int64, spawnpoint *Spawnpoint) error { err := db.GeneralDb.GetContext(ctx, spawnpoint, "SELECT id, lat, lon, updated, last_seen, despawn_sec FROM spawnpoint WHERE id = ?", spawnpointId) @@ -249,8 +263,8 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi return } - spawnpoint.Updated = time.Now().Unix() // ensure future updates are set correctly - spawnpoint.LastSeen = time.Now().Unix() // ensure future updates are set correctly + spawnpoint.SetUpdated(time.Now().Unix()) // ensure future updates are set correctly + spawnpoint.SetLastSeen(time.Now().Unix()) // ensure future updates are set correctly _, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO spawnpoint (id, lat, lon, updated, last_seen, despawn_sec)"+ "VALUES (:id, :lat, :lon, :updated, :last_seen, :despawn_sec)"+ @@ -281,7 +295,7 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoint // update at least every 6 hours (21600s). If reduce_updates is enabled, use 12 hours. if now-spawnpoint.LastSeen > GetUpdateThreshold(21600) { - spawnpoint.LastSeen = now + spawnpoint.SetLastSeen(now) _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ "SET last_seen=? "+ diff --git a/decoder/station.go b/decoder/station.go index 9feea5c1..d46bd2c2 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -359,3 +359,13 @@ func (station *Station) SetStationedPokemon(v null.String) { } } } + +func (station *Station) SetUpdated(v int64) { + if station.Updated != v { + station.Updated = v + station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Updated") + } + } +} diff --git a/decoder/station_state.go b/decoder/station_state.go index d4e4e06a..b038936d 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -140,7 +140,7 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { } } - station.Updated = now + station.SetUpdated(now) if station.IsNewRecord() { if dbDebugEnabled { diff --git a/decoder/tappable.go b/decoder/tappable.go index 3ab8d218..94ab1332 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -155,3 +155,13 @@ func (ta *Tappable) SetExpireTimestampVerified(v bool) { } } } + +func (ta *Tappable) SetUpdated(v int64) { + if ta.Updated != v { + ta.Updated = v + ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Updated") + } + } +} diff --git a/decoder/tappable_state.go b/decoder/tappable_state.go index ff30c42f..517c22ab 100644 --- a/decoder/tappable_state.go +++ b/decoder/tappable_state.go @@ -101,7 +101,7 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } now := time.Now().Unix() - tappable.Updated = now + tappable.SetUpdated(now) if tappable.IsNewRecord() { if dbDebugEnabled { From 140e36af73b93152c5c24a40e2c355018c27ad93 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 22:00:24 +0000 Subject: [PATCH 31/78] Improved dbupdate logging to show before/after --- decoder/gym.go | 235 ++++++++++++++------------- decoder/incident.go | 51 +++++- decoder/incident_state.go | 6 + decoder/player.go | 332 +++++++++++++++++++++++++++++++++++++- decoder/pokemon.go | 187 ++++++++++----------- decoder/pokemon_state.go | 4 + decoder/pokestop.go | 259 ++++++++++++++--------------- decoder/routes.go | 127 +++++++-------- decoder/s2cell.go | 2 +- decoder/spawnpoint.go | 32 +++- decoder/station.go | 163 +++++++++---------- decoder/tappable.go | 67 ++++---- 12 files changed, 945 insertions(+), 520 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index b841b730..d906a0c0 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -178,176 +179,179 @@ func (gym *Gym) Unlock() { func (gym *Gym) SetId(v string) { if gym.Id != v { - gym.Id = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Id") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Id:%s->%s", gym.Id, v)) } + gym.Id = v + gym.dirty = true } } func (gym *Gym) SetLat(v float64) { if !floatAlmostEqual(gym.Lat, v, floatTolerance) { - gym.Lat = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Lat") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Lat:%f->%f", gym.Lat, v)) } + gym.Lat = v + gym.dirty = true } } func (gym *Gym) SetLon(v float64) { if !floatAlmostEqual(gym.Lon, v, floatTolerance) { - gym.Lon = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Lon") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Lon:%f->%f", gym.Lon, v)) } + gym.Lon = v + gym.dirty = true } } func (gym *Gym) SetName(v null.String) { if gym.Name != v { - gym.Name = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Name") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Name:%v->%v", gym.Name, v)) } + gym.Name = v + gym.dirty = true } } func (gym *Gym) SetUrl(v null.String) { if gym.Url != v { - gym.Url = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Url") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Url:%v->%v", gym.Url, v)) } + gym.Url = v + gym.dirty = true } } func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { if gym.LastModifiedTimestamp != v { - gym.LastModifiedTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "LastModifiedTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("LastModifiedTimestamp:%v->%v", gym.LastModifiedTimestamp, v)) } + gym.LastModifiedTimestamp = v + gym.dirty = true } } func (gym *Gym) SetRaidEndTimestamp(v null.Int) { if gym.RaidEndTimestamp != v { - gym.RaidEndTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidEndTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidEndTimestamp:%v->%v", gym.RaidEndTimestamp, v)) } + gym.RaidEndTimestamp = v + gym.dirty = true } } func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { if gym.RaidSpawnTimestamp != v { - gym.RaidSpawnTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidSpawnTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidSpawnTimestamp:%v->%v", gym.RaidSpawnTimestamp, v)) } + gym.RaidSpawnTimestamp = v + gym.dirty = true } } func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { if gym.RaidBattleTimestamp != v { - gym.RaidBattleTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidBattleTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidBattleTimestamp:%v->%v", gym.RaidBattleTimestamp, v)) } + gym.RaidBattleTimestamp = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonId(v null.Int) { if gym.RaidPokemonId != v { - gym.RaidPokemonId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonId:%v->%v", gym.RaidPokemonId, v)) } + gym.RaidPokemonId = v + gym.dirty = true } } func (gym *Gym) SetGuardingPokemonId(v null.Int) { if gym.GuardingPokemonId != v { - gym.GuardingPokemonId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "GuardingPokemonId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonId:%v->%v", gym.GuardingPokemonId, v)) } + gym.GuardingPokemonId = v + gym.dirty = true } } func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { if gym.GuardingPokemonDisplay != v { - gym.GuardingPokemonDisplay = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "GuardingPokemonDisplay") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonDisplay:%v->%v", gym.GuardingPokemonDisplay, v)) } + gym.GuardingPokemonDisplay = v + gym.dirty = true } } func (gym *Gym) SetAvailableSlots(v null.Int) { if gym.AvailableSlots != v { - gym.AvailableSlots = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "AvailableSlots") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("AvailableSlots:%v->%v", gym.AvailableSlots, v)) } + gym.AvailableSlots = v + gym.dirty = true } } func (gym *Gym) SetTeamId(v null.Int) { if gym.TeamId != v { - gym.TeamId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "TeamId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("TeamId:%v->%v", gym.TeamId, v)) } + gym.TeamId = v + gym.dirty = true } } func (gym *Gym) SetRaidLevel(v null.Int) { if gym.RaidLevel != v { - gym.RaidLevel = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidLevel") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidLevel:%v->%v", gym.RaidLevel, v)) } + gym.RaidLevel = v + gym.dirty = true } } func (gym *Gym) SetEnabled(v null.Int) { if gym.Enabled != v { - gym.Enabled = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Enabled") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Enabled:%v->%v", gym.Enabled, v)) } + gym.Enabled = v + gym.dirty = true } } func (gym *Gym) SetExRaidEligible(v null.Int) { if gym.ExRaidEligible != v { - gym.ExRaidEligible = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "ExRaidEligible") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("ExRaidEligible:%v->%v", gym.ExRaidEligible, v)) } + gym.ExRaidEligible = v + gym.dirty = true } } func (gym *Gym) SetInBattle(v null.Int) { if gym.InBattle != v { + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, fmt.Sprintf("InBattle:%v->%v", gym.InBattle, v)) + } gym.InBattle = v //Do not set to dirty, as don't trigger an update gym.internalDirty = true @@ -356,196 +360,199 @@ func (gym *Gym) SetInBattle(v null.Int) { func (gym *Gym) SetRaidPokemonMove1(v null.Int) { if gym.RaidPokemonMove1 != v { - gym.RaidPokemonMove1 = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonMove1") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove1:%v->%v", gym.RaidPokemonMove1, v)) } + gym.RaidPokemonMove1 = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonMove2(v null.Int) { if gym.RaidPokemonMove2 != v { - gym.RaidPokemonMove2 = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonMove2") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove2:%v->%v", gym.RaidPokemonMove2, v)) } + gym.RaidPokemonMove2 = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonForm(v null.Int) { if gym.RaidPokemonForm != v { - gym.RaidPokemonForm = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonForm") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonForm:%v->%v", gym.RaidPokemonForm, v)) } + gym.RaidPokemonForm = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { if gym.RaidPokemonAlignment != v { - gym.RaidPokemonAlignment = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonAlignment") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonAlignment:%v->%v", gym.RaidPokemonAlignment, v)) } + gym.RaidPokemonAlignment = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonCp(v null.Int) { if gym.RaidPokemonCp != v { - gym.RaidPokemonCp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonCp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCp:%v->%v", gym.RaidPokemonCp, v)) } + gym.RaidPokemonCp = v + gym.dirty = true } } func (gym *Gym) SetRaidIsExclusive(v null.Int) { if gym.RaidIsExclusive != v { - gym.RaidIsExclusive = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidIsExclusive") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidIsExclusive:%v->%v", gym.RaidIsExclusive, v)) } + gym.RaidIsExclusive = v + gym.dirty = true } } func (gym *Gym) SetCellId(v null.Int) { if gym.CellId != v { - gym.CellId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "CellId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("CellId:%v->%v", gym.CellId, v)) } + gym.CellId = v + gym.dirty = true } } func (gym *Gym) SetDeleted(v bool) { if gym.Deleted != v { - gym.Deleted = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Deleted") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Deleted:%t->%t", gym.Deleted, v)) } + gym.Deleted = v + gym.dirty = true } } func (gym *Gym) SetTotalCp(v null.Int) { if gym.TotalCp != v { - gym.TotalCp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "TotalCp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("TotalCp:%v->%v", gym.TotalCp, v)) } + gym.TotalCp = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonGender(v null.Int) { if gym.RaidPokemonGender != v { - gym.RaidPokemonGender = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonGender") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonGender:%v->%v", gym.RaidPokemonGender, v)) } + gym.RaidPokemonGender = v + gym.dirty = true } } func (gym *Gym) SetSponsorId(v null.Int) { if gym.SponsorId != v { - gym.SponsorId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "SponsorId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("SponsorId:%v->%v", gym.SponsorId, v)) } + gym.SponsorId = v + gym.dirty = true } } func (gym *Gym) SetPartnerId(v null.String) { if gym.PartnerId != v { - gym.PartnerId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "PartnerId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PartnerId:%v->%v", gym.PartnerId, v)) } + gym.PartnerId = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonCostume(v null.Int) { if gym.RaidPokemonCostume != v { - gym.RaidPokemonCostume = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonCostume") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCostume:%v->%v", gym.RaidPokemonCostume, v)) } + gym.RaidPokemonCostume = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { if gym.RaidPokemonEvolution != v { - gym.RaidPokemonEvolution = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonEvolution") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonEvolution:%v->%v", gym.RaidPokemonEvolution, v)) } + gym.RaidPokemonEvolution = v + gym.dirty = true } } func (gym *Gym) SetArScanEligible(v null.Int) { if gym.ArScanEligible != v { - gym.ArScanEligible = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "ArScanEligible") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("ArScanEligible:%v->%v", gym.ArScanEligible, v)) } + gym.ArScanEligible = v + gym.dirty = true } } func (gym *Gym) SetPowerUpLevel(v null.Int) { if gym.PowerUpLevel != v { - gym.PowerUpLevel = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "PowerUpLevel") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpLevel:%v->%v", gym.PowerUpLevel, v)) } + gym.PowerUpLevel = v + gym.dirty = true } } func (gym *Gym) SetPowerUpPoints(v null.Int) { if gym.PowerUpPoints != v { - gym.PowerUpPoints = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "PowerUpPoints") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpPoints:%v->%v", gym.PowerUpPoints, v)) } + gym.PowerUpPoints = v + gym.dirty = true } } func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { if gym.PowerUpEndTimestamp != v { - gym.PowerUpEndTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "PowerUpEndTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%v->%v", gym.PowerUpEndTimestamp, v)) } + gym.PowerUpEndTimestamp = v + gym.dirty = true } } func (gym *Gym) SetDescription(v null.String) { if gym.Description != v { - gym.Description = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Description") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Description:%v->%v", gym.Description, v)) } + gym.Description = v + gym.dirty = true } } func (gym *Gym) SetDefenders(v null.String) { if gym.Defenders != v { + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Defenders:%v->%v", gym.Defenders, v)) + } gym.Defenders = v //Do not set to dirty, as don't trigger an update gym.internalDirty = true @@ -554,20 +561,20 @@ func (gym *Gym) SetDefenders(v null.String) { func (gym *Gym) SetRsvps(v null.String) { if gym.Rsvps != v { - gym.Rsvps = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Rsvps") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Rsvps:%v->%v", gym.Rsvps, v)) } + gym.Rsvps = v + gym.dirty = true } } func (gym *Gym) SetUpdated(v int64) { if gym.Updated != v { - gym.Updated = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Updated") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Updated:%d->%d", gym.Updated, v)) } + gym.Updated = v + gym.dirty = true } } diff --git a/decoder/incident.go b/decoder/incident.go index 707ed5ee..dd85d474 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -27,8 +28,9 @@ type Incident struct { Slot3PokemonId null.Int `db:"slot_3_pokemon_id"` Slot3Form null.Int `db:"slot_3_form"` - dirty bool `db:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues IncidentOldValues `db:"-"` // Old values for webhook comparison } @@ -118,6 +120,9 @@ func (incident *Incident) snapshotOldValues() { func (incident *Incident) SetId(v string) { if incident.Id != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Id:%s->%s", incident.Id, v)) + } incident.Id = v incident.dirty = true } @@ -125,6 +130,9 @@ func (incident *Incident) SetId(v string) { func (incident *Incident) SetPokestopId(v string) { if incident.PokestopId != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("PokestopId:%s->%s", incident.PokestopId, v)) + } incident.PokestopId = v incident.dirty = true } @@ -132,6 +140,9 @@ func (incident *Incident) SetPokestopId(v string) { func (incident *Incident) SetStartTime(v int64) { if incident.StartTime != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("StartTime:%d->%d", incident.StartTime, v)) + } incident.StartTime = v incident.dirty = true } @@ -139,6 +150,9 @@ func (incident *Incident) SetStartTime(v int64) { func (incident *Incident) SetExpirationTime(v int64) { if incident.ExpirationTime != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("ExpirationTime:%d->%d", incident.ExpirationTime, v)) + } incident.ExpirationTime = v incident.dirty = true } @@ -146,6 +160,9 @@ func (incident *Incident) SetExpirationTime(v int64) { func (incident *Incident) SetDisplayType(v int16) { if incident.DisplayType != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("DisplayType:%d->%d", incident.DisplayType, v)) + } incident.DisplayType = v incident.dirty = true } @@ -153,6 +170,9 @@ func (incident *Incident) SetDisplayType(v int16) { func (incident *Incident) SetStyle(v int16) { if incident.Style != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Style:%d->%d", incident.Style, v)) + } incident.Style = v incident.dirty = true } @@ -160,6 +180,9 @@ func (incident *Incident) SetStyle(v int16) { func (incident *Incident) SetCharacter(v int16) { if incident.Character != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Character:%d->%d", incident.Character, v)) + } incident.Character = v incident.dirty = true } @@ -167,6 +190,9 @@ func (incident *Incident) SetCharacter(v int16) { func (incident *Incident) SetConfirmed(v bool) { if incident.Confirmed != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Confirmed:%t->%t", incident.Confirmed, v)) + } incident.Confirmed = v incident.dirty = true } @@ -174,6 +200,9 @@ func (incident *Incident) SetConfirmed(v bool) { func (incident *Incident) SetSlot1PokemonId(v null.Int) { if incident.Slot1PokemonId != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1PokemonId:%v->%v", incident.Slot1PokemonId, v)) + } incident.Slot1PokemonId = v incident.dirty = true } @@ -181,6 +210,9 @@ func (incident *Incident) SetSlot1PokemonId(v null.Int) { func (incident *Incident) SetSlot1Form(v null.Int) { if incident.Slot1Form != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1Form:%v->%v", incident.Slot1Form, v)) + } incident.Slot1Form = v incident.dirty = true } @@ -188,6 +220,9 @@ func (incident *Incident) SetSlot1Form(v null.Int) { func (incident *Incident) SetSlot2PokemonId(v null.Int) { if incident.Slot2PokemonId != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2PokemonId:%v->%v", incident.Slot2PokemonId, v)) + } incident.Slot2PokemonId = v incident.dirty = true } @@ -195,6 +230,9 @@ func (incident *Incident) SetSlot2PokemonId(v null.Int) { func (incident *Incident) SetSlot2Form(v null.Int) { if incident.Slot2Form != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2Form:%v->%v", incident.Slot2Form, v)) + } incident.Slot2Form = v incident.dirty = true } @@ -202,6 +240,9 @@ func (incident *Incident) SetSlot2Form(v null.Int) { func (incident *Incident) SetSlot3PokemonId(v null.Int) { if incident.Slot3PokemonId != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3PokemonId:%v->%v", incident.Slot3PokemonId, v)) + } incident.Slot3PokemonId = v incident.dirty = true } @@ -209,6 +250,9 @@ func (incident *Incident) SetSlot3PokemonId(v null.Int) { func (incident *Incident) SetSlot3Form(v null.Int) { if incident.Slot3Form != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3Form:%v->%v", incident.Slot3Form, v)) + } incident.Slot3Form = v incident.dirty = true } @@ -216,6 +260,9 @@ func (incident *Incident) SetSlot3Form(v null.Int) { func (incident *Incident) SetUpdated(v int64) { if incident.Updated != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Updated:%d->%d", incident.Updated, v)) + } incident.Updated = v incident.dirty = true } diff --git a/decoder/incident_state.go b/decoder/incident_state.go index c9da5ba7..641bc06b 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -114,6 +114,9 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident incident.SetUpdated(time.Now().Unix()) if incident.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Incident", incident.Id, incident.changedFields) + } res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) @@ -124,6 +127,9 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident statsCollector.IncDbQuery("insert incident", err) _, _ = res, err } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Incident", incident.Id, incident.changedFields) + } res, err := db.GeneralDb.NamedExec("UPDATE incident SET "+ "start = :start, "+ "expiration = :expiration, "+ diff --git a/decoder/player.go b/decoder/player.go index c24054e3..433885bb 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -2,6 +2,7 @@ package decoder import ( "database/sql" + "fmt" "reflect" "strconv" "time" @@ -103,8 +104,9 @@ type Player struct { CaughtDark null.Int `db:"caught_dark"` CaughtFairy null.Int `db:"caught_fairy"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) } // IsDirty returns true if any field has been modified @@ -131,6 +133,9 @@ func (p *Player) setFieldDirty() { func (p *Player) SetFriendshipId(v null.String) { if p.FriendshipId != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("FriendshipId:%v->%v", p.FriendshipId, v)) + } p.FriendshipId = v p.dirty = true } @@ -138,6 +143,9 @@ func (p *Player) SetFriendshipId(v null.String) { func (p *Player) SetFriendCode(v null.String) { if p.FriendCode != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("FriendCode:%v->%v", p.FriendCode, v)) + } p.FriendCode = v p.dirty = true } @@ -145,6 +153,9 @@ func (p *Player) SetFriendCode(v null.String) { func (p *Player) SetTeam(v null.Int) { if p.Team != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Team:%v->%v", p.Team, v)) + } p.Team = v p.dirty = true } @@ -152,6 +163,9 @@ func (p *Player) SetTeam(v null.Int) { func (p *Player) SetLevel(v null.Int) { if p.Level != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Level:%v->%v", p.Level, v)) + } p.Level = v p.dirty = true } @@ -159,6 +173,9 @@ func (p *Player) SetLevel(v null.Int) { func (p *Player) SetXp(v null.Int) { if p.Xp != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Xp:%v->%v", p.Xp, v)) + } p.Xp = v p.dirty = true } @@ -166,6 +183,9 @@ func (p *Player) SetXp(v null.Int) { func (p *Player) SetBattlesWon(v null.Int) { if p.BattlesWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("BattlesWon:%v->%v", p.BattlesWon, v)) + } p.BattlesWon = v p.dirty = true } @@ -173,6 +193,9 @@ func (p *Player) SetBattlesWon(v null.Int) { func (p *Player) SetKmWalked(v null.Float) { if !nullFloatAlmostEqual(p.KmWalked, v, 0.001) { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("KmWalked:%v->%v", p.KmWalked, v)) + } p.KmWalked = v p.dirty = true } @@ -180,6 +203,9 @@ func (p *Player) SetKmWalked(v null.Float) { func (p *Player) SetCaughtPokemon(v null.Int) { if p.CaughtPokemon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPokemon:%v->%v", p.CaughtPokemon, v)) + } p.CaughtPokemon = v p.dirty = true } @@ -187,6 +213,9 @@ func (p *Player) SetCaughtPokemon(v null.Int) { func (p *Player) SetGblRank(v null.Int) { if p.GblRank != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GblRank:%v->%v", p.GblRank, v)) + } p.GblRank = v p.dirty = true } @@ -194,6 +223,9 @@ func (p *Player) SetGblRank(v null.Int) { func (p *Player) SetGblRating(v null.Int) { if p.GblRating != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GblRating:%v->%v", p.GblRating, v)) + } p.GblRating = v p.dirty = true } @@ -201,6 +233,9 @@ func (p *Player) SetGblRating(v null.Int) { func (p *Player) SetEventBadges(v null.String) { if p.EventBadges != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("EventBadges:%v->%v", p.EventBadges, v)) + } p.EventBadges = v p.dirty = true } @@ -208,426 +243,708 @@ func (p *Player) SetEventBadges(v null.String) { func (p *Player) SetStopsSpun(v null.Int) { if p.StopsSpun != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("StopsSpun:%v->%v", p.StopsSpun, v)) + } p.StopsSpun = v p.dirty = true } } + func (p *Player) SetEvolved(v null.Int) { if p.Evolved != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Evolved:%v->%v", p.Evolved, v)) + } p.Evolved = v p.dirty = true } } + func (p *Player) SetHatched(v null.Int) { if p.Hatched != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Hatched:%v->%v", p.Hatched, v)) + } p.Hatched = v p.dirty = true } } + func (p *Player) SetQuests(v null.Int) { if p.Quests != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Quests:%v->%v", p.Quests, v)) + } p.Quests = v p.dirty = true } } + func (p *Player) SetTrades(v null.Int) { if p.Trades != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Trades:%v->%v", p.Trades, v)) + } p.Trades = v p.dirty = true } } + func (p *Player) SetPhotobombs(v null.Int) { if p.Photobombs != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Photobombs:%v->%v", p.Photobombs, v)) + } p.Photobombs = v p.dirty = true } } + func (p *Player) SetPurified(v null.Int) { if p.Purified != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Purified:%v->%v", p.Purified, v)) + } p.Purified = v p.dirty = true } } + func (p *Player) SetGruntsDefeated(v null.Int) { if p.GruntsDefeated != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GruntsDefeated:%v->%v", p.GruntsDefeated, v)) + } p.GruntsDefeated = v p.dirty = true } } + func (p *Player) SetGymBattlesWon(v null.Int) { if p.GymBattlesWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GymBattlesWon:%v->%v", p.GymBattlesWon, v)) + } p.GymBattlesWon = v p.dirty = true } } + func (p *Player) SetNormalRaidsWon(v null.Int) { if p.NormalRaidsWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("NormalRaidsWon:%v->%v", p.NormalRaidsWon, v)) + } p.NormalRaidsWon = v p.dirty = true } } + func (p *Player) SetLegendaryRaidsWon(v null.Int) { if p.LegendaryRaidsWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LegendaryRaidsWon:%v->%v", p.LegendaryRaidsWon, v)) + } p.LegendaryRaidsWon = v p.dirty = true } } + func (p *Player) SetTrainingsWon(v null.Int) { if p.TrainingsWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TrainingsWon:%v->%v", p.TrainingsWon, v)) + } p.TrainingsWon = v p.dirty = true } } + func (p *Player) SetBerriesFed(v null.Int) { if p.BerriesFed != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("BerriesFed:%v->%v", p.BerriesFed, v)) + } p.BerriesFed = v p.dirty = true } } + func (p *Player) SetHoursDefended(v null.Int) { if p.HoursDefended != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("HoursDefended:%v->%v", p.HoursDefended, v)) + } p.HoursDefended = v p.dirty = true } } + func (p *Player) SetBestFriends(v null.Int) { if p.BestFriends != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("BestFriends:%v->%v", p.BestFriends, v)) + } p.BestFriends = v p.dirty = true } } + func (p *Player) SetBestBuddies(v null.Int) { if p.BestBuddies != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("BestBuddies:%v->%v", p.BestBuddies, v)) + } p.BestBuddies = v p.dirty = true } } + func (p *Player) SetGiovanniDefeated(v null.Int) { if p.GiovanniDefeated != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GiovanniDefeated:%v->%v", p.GiovanniDefeated, v)) + } p.GiovanniDefeated = v p.dirty = true } } + func (p *Player) SetMegaEvos(v null.Int) { if p.MegaEvos != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("MegaEvos:%v->%v", p.MegaEvos, v)) + } p.MegaEvos = v p.dirty = true } } + func (p *Player) SetCollectionsDone(v null.Int) { if p.CollectionsDone != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CollectionsDone:%v->%v", p.CollectionsDone, v)) + } p.CollectionsDone = v p.dirty = true } } + func (p *Player) SetUniqueStopsSpun(v null.Int) { if p.UniqueStopsSpun != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueStopsSpun:%v->%v", p.UniqueStopsSpun, v)) + } p.UniqueStopsSpun = v p.dirty = true } } + func (p *Player) SetUniqueMegaEvos(v null.Int) { if p.UniqueMegaEvos != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueMegaEvos:%v->%v", p.UniqueMegaEvos, v)) + } p.UniqueMegaEvos = v p.dirty = true } } + func (p *Player) SetUniqueRaidBosses(v null.Int) { if p.UniqueRaidBosses != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueRaidBosses:%v->%v", p.UniqueRaidBosses, v)) + } p.UniqueRaidBosses = v p.dirty = true } } + func (p *Player) SetUniqueUnown(v null.Int) { if p.UniqueUnown != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueUnown:%v->%v", p.UniqueUnown, v)) + } p.UniqueUnown = v p.dirty = true } } + func (p *Player) SetSevenDayStreaks(v null.Int) { if p.SevenDayStreaks != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("SevenDayStreaks:%v->%v", p.SevenDayStreaks, v)) + } p.SevenDayStreaks = v p.dirty = true } } + func (p *Player) SetTradeKm(v null.Int) { if p.TradeKm != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TradeKm:%v->%v", p.TradeKm, v)) + } p.TradeKm = v p.dirty = true } } + func (p *Player) SetRaidsWithFriends(v null.Int) { if p.RaidsWithFriends != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("RaidsWithFriends:%v->%v", p.RaidsWithFriends, v)) + } p.RaidsWithFriends = v p.dirty = true } } + func (p *Player) SetCaughtAtLure(v null.Int) { if p.CaughtAtLure != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtAtLure:%v->%v", p.CaughtAtLure, v)) + } p.CaughtAtLure = v p.dirty = true } } + func (p *Player) SetWayfarerAgreements(v null.Int) { if p.WayfarerAgreements != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("WayfarerAgreements:%v->%v", p.WayfarerAgreements, v)) + } p.WayfarerAgreements = v p.dirty = true } } + func (p *Player) SetTrainersReferred(v null.Int) { if p.TrainersReferred != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TrainersReferred:%v->%v", p.TrainersReferred, v)) + } p.TrainersReferred = v p.dirty = true } } + func (p *Player) SetRaidAchievements(v null.Int) { if p.RaidAchievements != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("RaidAchievements:%v->%v", p.RaidAchievements, v)) + } p.RaidAchievements = v p.dirty = true } } + func (p *Player) SetXlKarps(v null.Int) { if p.XlKarps != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("XlKarps:%v->%v", p.XlKarps, v)) + } p.XlKarps = v p.dirty = true } } + func (p *Player) SetXsRats(v null.Int) { if p.XsRats != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("XsRats:%v->%v", p.XsRats, v)) + } p.XsRats = v p.dirty = true } } + func (p *Player) SetPikachuCaught(v null.Int) { if p.PikachuCaught != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("PikachuCaught:%v->%v", p.PikachuCaught, v)) + } p.PikachuCaught = v p.dirty = true } } + func (p *Player) SetLeagueGreatWon(v null.Int) { if p.LeagueGreatWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueGreatWon:%v->%v", p.LeagueGreatWon, v)) + } p.LeagueGreatWon = v p.dirty = true } } + func (p *Player) SetLeagueUltraWon(v null.Int) { if p.LeagueUltraWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueUltraWon:%v->%v", p.LeagueUltraWon, v)) + } p.LeagueUltraWon = v p.dirty = true } } + func (p *Player) SetLeagueMasterWon(v null.Int) { if p.LeagueMasterWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueMasterWon:%v->%v", p.LeagueMasterWon, v)) + } p.LeagueMasterWon = v p.dirty = true } } + func (p *Player) SetTinyPokemonCaught(v null.Int) { if p.TinyPokemonCaught != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TinyPokemonCaught:%v->%v", p.TinyPokemonCaught, v)) + } p.TinyPokemonCaught = v p.dirty = true } } + func (p *Player) SetJumboPokemonCaught(v null.Int) { if p.JumboPokemonCaught != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("JumboPokemonCaught:%v->%v", p.JumboPokemonCaught, v)) + } p.JumboPokemonCaught = v p.dirty = true } } + func (p *Player) SetVivillon(v null.Int) { if p.Vivillon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Vivillon:%v->%v", p.Vivillon, v)) + } p.Vivillon = v p.dirty = true } } + func (p *Player) SetMaxSizeFirstPlace(v null.Int) { if p.MaxSizeFirstPlace != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("MaxSizeFirstPlace:%v->%v", p.MaxSizeFirstPlace, v)) + } p.MaxSizeFirstPlace = v p.dirty = true } } + func (p *Player) SetTotalRoutePlay(v null.Int) { if p.TotalRoutePlay != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TotalRoutePlay:%v->%v", p.TotalRoutePlay, v)) + } p.TotalRoutePlay = v p.dirty = true } } + func (p *Player) SetPartiesCompleted(v null.Int) { if p.PartiesCompleted != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("PartiesCompleted:%v->%v", p.PartiesCompleted, v)) + } p.PartiesCompleted = v p.dirty = true } } + func (p *Player) SetEventCheckIns(v null.Int) { if p.EventCheckIns != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("EventCheckIns:%v->%v", p.EventCheckIns, v)) + } p.EventCheckIns = v p.dirty = true } } func (p *Player) SetDexGen1(v null.Int) { if p.DexGen1 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen1:%v->%v", p.DexGen1, v)) + } p.DexGen1 = v p.dirty = true } } + func (p *Player) SetDexGen2(v null.Int) { if p.DexGen2 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen2:%v->%v", p.DexGen2, v)) + } p.DexGen2 = v p.dirty = true } } + func (p *Player) SetDexGen3(v null.Int) { if p.DexGen3 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen3:%v->%v", p.DexGen3, v)) + } p.DexGen3 = v p.dirty = true } } + func (p *Player) SetDexGen4(v null.Int) { if p.DexGen4 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen4:%v->%v", p.DexGen4, v)) + } p.DexGen4 = v p.dirty = true } } + func (p *Player) SetDexGen5(v null.Int) { if p.DexGen5 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen5:%v->%v", p.DexGen5, v)) + } p.DexGen5 = v p.dirty = true } } + func (p *Player) SetDexGen6(v null.Int) { if p.DexGen6 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen6:%v->%v", p.DexGen6, v)) + } p.DexGen6 = v p.dirty = true } } + func (p *Player) SetDexGen7(v null.Int) { if p.DexGen7 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen7:%v->%v", p.DexGen7, v)) + } p.DexGen7 = v p.dirty = true } } + func (p *Player) SetDexGen8(v null.Int) { if p.DexGen8 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8:%v->%v", p.DexGen8, v)) + } p.DexGen8 = v p.dirty = true } } + func (p *Player) SetDexGen8A(v null.Int) { if p.DexGen8A != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8A:%v->%v", p.DexGen8A, v)) + } p.DexGen8A = v p.dirty = true } } + func (p *Player) SetDexGen9(v null.Int) { if p.DexGen9 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen9:%v->%v", p.DexGen9, v)) + } p.DexGen9 = v p.dirty = true } } + func (p *Player) SetCaughtNormal(v null.Int) { if p.CaughtNormal != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtNormal:%v->%v", p.CaughtNormal, v)) + } p.CaughtNormal = v p.dirty = true } } + func (p *Player) SetCaughtFighting(v null.Int) { if p.CaughtFighting != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFighting:%v->%v", p.CaughtFighting, v)) + } p.CaughtFighting = v p.dirty = true } } + func (p *Player) SetCaughtFlying(v null.Int) { if p.CaughtFlying != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFlying:%v->%v", p.CaughtFlying, v)) + } p.CaughtFlying = v p.dirty = true } } + func (p *Player) SetCaughtPoison(v null.Int) { if p.CaughtPoison != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPoison:%v->%v", p.CaughtPoison, v)) + } p.CaughtPoison = v p.dirty = true } } + func (p *Player) SetCaughtGround(v null.Int) { if p.CaughtGround != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGround:%v->%v", p.CaughtGround, v)) + } p.CaughtGround = v p.dirty = true } } + func (p *Player) SetCaughtRock(v null.Int) { if p.CaughtRock != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtRock:%v->%v", p.CaughtRock, v)) + } p.CaughtRock = v p.dirty = true } } + func (p *Player) SetCaughtBug(v null.Int) { if p.CaughtBug != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtBug:%v->%v", p.CaughtBug, v)) + } p.CaughtBug = v p.dirty = true } } + func (p *Player) SetCaughtGhost(v null.Int) { if p.CaughtGhost != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGhost:%v->%v", p.CaughtGhost, v)) + } p.CaughtGhost = v p.dirty = true } } + func (p *Player) SetCaughtSteel(v null.Int) { if p.CaughtSteel != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtSteel:%v->%v", p.CaughtSteel, v)) + } p.CaughtSteel = v p.dirty = true } } + func (p *Player) SetCaughtFire(v null.Int) { if p.CaughtFire != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFire:%v->%v", p.CaughtFire, v)) + } p.CaughtFire = v p.dirty = true } } + func (p *Player) SetCaughtWater(v null.Int) { if p.CaughtWater != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtWater:%v->%v", p.CaughtWater, v)) + } p.CaughtWater = v p.dirty = true } } + func (p *Player) SetCaughtGrass(v null.Int) { if p.CaughtGrass != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGrass:%v->%v", p.CaughtGrass, v)) + } p.CaughtGrass = v p.dirty = true } } + func (p *Player) SetCaughtElectric(v null.Int) { if p.CaughtElectric != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtElectric:%v->%v", p.CaughtElectric, v)) + } p.CaughtElectric = v p.dirty = true } } + func (p *Player) SetCaughtPsychic(v null.Int) { if p.CaughtPsychic != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPsychic:%v->%v", p.CaughtPsychic, v)) + } p.CaughtPsychic = v p.dirty = true } } + func (p *Player) SetCaughtIce(v null.Int) { if p.CaughtIce != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtIce:%v->%v", p.CaughtIce, v)) + } p.CaughtIce = v p.dirty = true } } + func (p *Player) SetCaughtDragon(v null.Int) { if p.CaughtDragon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDragon:%v->%v", p.CaughtDragon, v)) + } p.CaughtDragon = v p.dirty = true } } + func (p *Player) SetCaughtDark(v null.Int) { if p.CaughtDark != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDark:%v->%v", p.CaughtDark, v)) + } p.CaughtDark = v p.dirty = true } } + func (p *Player) SetCaughtFairy(v null.Int) { if p.CaughtFairy != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFairy:%v->%v", p.CaughtFairy, v)) + } p.CaughtFairy = v p.dirty = true } @@ -635,6 +952,9 @@ func (p *Player) SetCaughtFairy(v null.Int) { func (p *Player) SetLastSeen(v int64) { if p.LastSeen != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LastSeen:%d->%d", p.LastSeen, v)) + } p.LastSeen = v p.dirty = true } @@ -791,6 +1111,14 @@ func savePlayerRecord(db db.DbDetails, player *Player) { player.SetLastSeen(time.Now().Unix()) + if dbDebugEnabled { + if player.IsNewRecord() { + dbDebugLog("INSERT", "Player", player.Name, player.changedFields) + } else { + dbDebugLog("UPDATE", "Player", player.Name, player.changedFields) + } + } + if player.IsNewRecord() { _, err := db.GeneralDb.NamedExec( ` diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 81e0f0f8..d20ba93b 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "golbat/grpc" @@ -168,131 +169,131 @@ func (pokemon *Pokemon) Unlock() { func (pokemon *Pokemon) SetPokestopId(v null.String) { if pokemon.PokestopId != v { - pokemon.PokestopId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "PokestopId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("PokestopId:%v->%v", pokemon.PokestopId, v)) } + pokemon.PokestopId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetSpawnId(v null.Int) { if pokemon.SpawnId != v { - pokemon.SpawnId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "SpawnId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SpawnId:%v->%v", pokemon.SpawnId, v)) } + pokemon.SpawnId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetLat(v float64) { if !floatAlmostEqual(pokemon.Lat, v, floatTolerance) { - pokemon.Lat = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Lat") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Lat:%f->%f", pokemon.Lat, v)) } + pokemon.Lat = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetLon(v float64) { if !floatAlmostEqual(pokemon.Lon, v, floatTolerance) { - pokemon.Lon = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Lon") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Lon:%f->%f", pokemon.Lon, v)) } + pokemon.Lon = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetPokemonId(v int16) { if pokemon.PokemonId != v { - pokemon.PokemonId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "PokemonId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("PokemonId:%d->%d", pokemon.PokemonId, v)) } + pokemon.PokemonId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetForm(v null.Int) { if pokemon.Form != v { - pokemon.Form = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Form") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Form:%v->%v", pokemon.Form, v)) } + pokemon.Form = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCostume(v null.Int) { if pokemon.Costume != v { - pokemon.Costume = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Costume") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Costume:%v->%v", pokemon.Costume, v)) } + pokemon.Costume = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetGender(v null.Int) { if pokemon.Gender != v { - pokemon.Gender = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Gender") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Gender:%v->%v", pokemon.Gender, v)) } + pokemon.Gender = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetWeather(v null.Int) { if pokemon.Weather != v { - pokemon.Weather = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Weather") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weather:%v->%v", pokemon.Weather, v)) } + pokemon.Weather = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetIsStrong(v null.Bool) { if pokemon.IsStrong != v { - pokemon.IsStrong = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "IsStrong") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsStrong:%v->%v", pokemon.IsStrong, v)) } + pokemon.IsStrong = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetExpireTimestamp(v null.Int) { if pokemon.ExpireTimestamp != v { - pokemon.ExpireTimestamp = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "ExpireTimestamp") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("ExpireTimestamp:%v->%v", pokemon.ExpireTimestamp, v)) } + pokemon.ExpireTimestamp = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetExpireTimestampVerified(v bool) { if pokemon.ExpireTimestampVerified != v { - pokemon.ExpireTimestampVerified = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "ExpireTimestampVerified") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("ExpireTimestampVerified:%t->%t", pokemon.ExpireTimestampVerified, v)) } + pokemon.ExpireTimestampVerified = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetSeenType(v null.String) { if pokemon.SeenType != v { - pokemon.SeenType = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "SeenType") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SeenType:%v->%v", pokemon.SeenType, v)) } + pokemon.SeenType = v + pokemon.dirty = true } } @@ -305,180 +306,180 @@ func (pokemon *Pokemon) SetUsername(v null.String) { func (pokemon *Pokemon) SetCellId(v null.Int) { if pokemon.CellId != v { - pokemon.CellId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "CellId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("CellId:%v->%v", pokemon.CellId, v)) } + pokemon.CellId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetIsEvent(v int8) { if pokemon.IsEvent != v { - pokemon.IsEvent = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "IsEvent") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsEvent:%d->%d", pokemon.IsEvent, v)) } + pokemon.IsEvent = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetShiny(v null.Bool) { if pokemon.Shiny != v { - pokemon.Shiny = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Shiny") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Shiny:%v->%v", pokemon.Shiny, v)) } + pokemon.Shiny = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCp(v null.Int) { if pokemon.Cp != v { - pokemon.Cp = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Cp") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Cp:%v->%v", pokemon.Cp, v)) } + pokemon.Cp = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetLevel(v null.Int) { if pokemon.Level != v { - pokemon.Level = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Level") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Level:%v->%v", pokemon.Level, v)) } + pokemon.Level = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetMove1(v null.Int) { if pokemon.Move1 != v { - pokemon.Move1 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Move1") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move1:%v->%v", pokemon.Move1, v)) } + pokemon.Move1 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetMove2(v null.Int) { if pokemon.Move2 != v { - pokemon.Move2 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Move2") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move2:%v->%v", pokemon.Move2, v)) } + pokemon.Move2 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetHeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Height, v, floatTolerance) { - pokemon.Height = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Height") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Height:%v->%v", pokemon.Height, v)) } + pokemon.Height = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetWeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Weight, v, floatTolerance) { - pokemon.Weight = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Weight") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weight:%v->%v", pokemon.Weight, v)) } + pokemon.Weight = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetSize(v null.Int) { if pokemon.Size != v { - pokemon.Size = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Size") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Size:%v->%v", pokemon.Size, v)) } + pokemon.Size = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetIsDitto(v bool) { if pokemon.IsDitto != v { - pokemon.IsDitto = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "IsDitto") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsDitto:%t->%t", pokemon.IsDitto, v)) } + pokemon.IsDitto = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { if pokemon.DisplayPokemonId != v { - pokemon.DisplayPokemonId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "DisplayPokemonId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("DisplayPokemonId:%v->%v", pokemon.DisplayPokemonId, v)) } + pokemon.DisplayPokemonId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetPvp(v null.String) { if pokemon.Pvp != v { - pokemon.Pvp = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Pvp") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Pvp:%v->%v", pokemon.Pvp, v)) } + pokemon.Pvp = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCapture1(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture1, v, floatTolerance) { - pokemon.Capture1 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Capture1") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture1:%v->%v", pokemon.Capture1, v)) } + pokemon.Capture1 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCapture2(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture2, v, floatTolerance) { - pokemon.Capture2 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Capture2") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture2:%v->%v", pokemon.Capture2, v)) } + pokemon.Capture2 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCapture3(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture3, v, floatTolerance) { - pokemon.Capture3 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Capture3") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture3:%v->%v", pokemon.Capture3, v)) } + pokemon.Capture3 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetUpdated(v null.Int) { if pokemon.Updated != v { - pokemon.Updated = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Updated") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Updated:%v->%v", pokemon.Updated, v)) } + pokemon.Updated = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetChanged(v int64) { if pokemon.Changed != v { - pokemon.Changed = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Changed") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Changed:%d->%d", pokemon.Changed, v)) } + pokemon.Changed = v + pokemon.dirty = true } } diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index 286dcf44..dcfb2840 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -339,6 +339,10 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po rows, rowsErr := res.RowsAffected() log.Debugf("Updating pokemon [%d] after update res = %d %v", pokemon.Id, rows, rowsErr) } + } else { + if dbDebugEnabled { + dbDebugLog("MEMORY", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } } // Update pokemon rtree diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 3289cfe4..8d97c8a1 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -159,430 +160,430 @@ func (p *Pokestop) Unlock() { func (p *Pokestop) SetId(v string) { if p.Id != v { - p.Id = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Id") + p.changedFields = append(p.changedFields, fmt.Sprintf("Id:%s->%s", p.Id, v)) } + p.Id = v + p.dirty = true } } func (p *Pokestop) SetLat(v float64) { if !floatAlmostEqual(p.Lat, v, floatTolerance) { - p.Lat = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Lat") + p.changedFields = append(p.changedFields, fmt.Sprintf("Lat:%f->%f", p.Lat, v)) } + p.Lat = v + p.dirty = true } } func (p *Pokestop) SetLon(v float64) { if !floatAlmostEqual(p.Lon, v, floatTolerance) { - p.Lon = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Lon") + p.changedFields = append(p.changedFields, fmt.Sprintf("Lon:%f->%f", p.Lon, v)) } + p.Lon = v + p.dirty = true } } func (p *Pokestop) SetName(v null.String) { if p.Name != v { - p.Name = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Name") + p.changedFields = append(p.changedFields, fmt.Sprintf("Name:%v->%v", p.Name, v)) } + p.Name = v + p.dirty = true } } func (p *Pokestop) SetUrl(v null.String) { if p.Url != v { - p.Url = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Url") + p.changedFields = append(p.changedFields, fmt.Sprintf("Url:%v->%v", p.Url, v)) } + p.Url = v + p.dirty = true } } func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { if p.LureExpireTimestamp != v { - p.LureExpireTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "LureExpireTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("LureExpireTimestamp:%v->%v", p.LureExpireTimestamp, v)) } + p.LureExpireTimestamp = v + p.dirty = true } } func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { if p.LastModifiedTimestamp != v { - p.LastModifiedTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "LastModifiedTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("LastModifiedTimestamp:%v->%v", p.LastModifiedTimestamp, v)) } + p.LastModifiedTimestamp = v + p.dirty = true } } func (p *Pokestop) SetEnabled(v null.Bool) { if p.Enabled != v { - p.Enabled = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Enabled") + p.changedFields = append(p.changedFields, fmt.Sprintf("Enabled:%v->%v", p.Enabled, v)) } + p.Enabled = v + p.dirty = true } } func (p *Pokestop) SetQuestType(v null.Int) { if p.QuestType != v { - p.QuestType = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestType") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestType:%v->%v", p.QuestType, v)) } + p.QuestType = v + p.dirty = true } } func (p *Pokestop) SetQuestTimestamp(v null.Int) { if p.QuestTimestamp != v { - p.QuestTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTimestamp:%v->%v", p.QuestTimestamp, v)) } + p.QuestTimestamp = v + p.dirty = true } } func (p *Pokestop) SetQuestTarget(v null.Int) { if p.QuestTarget != v { - p.QuestTarget = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestTarget") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTarget:%v->%v", p.QuestTarget, v)) } + p.QuestTarget = v + p.dirty = true } } func (p *Pokestop) SetQuestConditions(v null.String) { if p.QuestConditions != v { - p.QuestConditions = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestConditions") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestConditions:%v->%v", p.QuestConditions, v)) } + p.QuestConditions = v + p.dirty = true } } func (p *Pokestop) SetQuestRewards(v null.String) { if p.QuestRewards != v { - p.QuestRewards = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestRewards") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestRewards:%v->%v", p.QuestRewards, v)) } + p.QuestRewards = v + p.dirty = true } } func (p *Pokestop) SetQuestTemplate(v null.String) { if p.QuestTemplate != v { - p.QuestTemplate = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestTemplate") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTemplate:%v->%v", p.QuestTemplate, v)) } + p.QuestTemplate = v + p.dirty = true } } func (p *Pokestop) SetQuestTitle(v null.String) { if p.QuestTitle != v { - p.QuestTitle = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestTitle") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTitle:%v->%v", p.QuestTitle, v)) } + p.QuestTitle = v + p.dirty = true } } func (p *Pokestop) SetQuestExpiry(v null.Int) { if p.QuestExpiry != v { - p.QuestExpiry = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestExpiry") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestExpiry:%v->%v", p.QuestExpiry, v)) } + p.QuestExpiry = v + p.dirty = true } } func (p *Pokestop) SetCellId(v null.Int) { if p.CellId != v { - p.CellId = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "CellId") + p.changedFields = append(p.changedFields, fmt.Sprintf("CellId:%v->%v", p.CellId, v)) } + p.CellId = v + p.dirty = true } } func (p *Pokestop) SetDeleted(v bool) { if p.Deleted != v { - p.Deleted = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Deleted") + p.changedFields = append(p.changedFields, fmt.Sprintf("Deleted:%t->%t", p.Deleted, v)) } + p.Deleted = v + p.dirty = true } } func (p *Pokestop) SetLureId(v int16) { if p.LureId != v { - p.LureId = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "LureId") + p.changedFields = append(p.changedFields, fmt.Sprintf("LureId:%d->%d", p.LureId, v)) } + p.LureId = v + p.dirty = true } } func (p *Pokestop) SetFirstSeenTimestamp(v int16) { if p.FirstSeenTimestamp != v { - p.FirstSeenTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "FirstSeenTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("FirstSeenTimestamp:%d->%d", p.FirstSeenTimestamp, v)) } + p.FirstSeenTimestamp = v + p.dirty = true } } func (p *Pokestop) SetSponsorId(v null.Int) { if p.SponsorId != v { - p.SponsorId = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "SponsorId") + p.changedFields = append(p.changedFields, fmt.Sprintf("SponsorId:%v->%v", p.SponsorId, v)) } + p.SponsorId = v + p.dirty = true } } func (p *Pokestop) SetPartnerId(v null.String) { if p.PartnerId != v { - p.PartnerId = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "PartnerId") + p.changedFields = append(p.changedFields, fmt.Sprintf("PartnerId:%v->%v", p.PartnerId, v)) } + p.PartnerId = v + p.dirty = true } } func (p *Pokestop) SetArScanEligible(v null.Int) { if p.ArScanEligible != v { - p.ArScanEligible = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ArScanEligible") + p.changedFields = append(p.changedFields, fmt.Sprintf("ArScanEligible:%v->%v", p.ArScanEligible, v)) } + p.ArScanEligible = v + p.dirty = true } } func (p *Pokestop) SetPowerUpLevel(v null.Int) { if p.PowerUpLevel != v { - p.PowerUpLevel = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "PowerUpLevel") + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpLevel:%v->%v", p.PowerUpLevel, v)) } + p.PowerUpLevel = v + p.dirty = true } } func (p *Pokestop) SetPowerUpPoints(v null.Int) { if p.PowerUpPoints != v { - p.PowerUpPoints = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "PowerUpPoints") + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpPoints:%v->%v", p.PowerUpPoints, v)) } + p.PowerUpPoints = v + p.dirty = true } } func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { if p.PowerUpEndTimestamp != v { - p.PowerUpEndTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "PowerUpEndTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%v->%v", p.PowerUpEndTimestamp, v)) } + p.PowerUpEndTimestamp = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestType(v null.Int) { if p.AlternativeQuestType != v { - p.AlternativeQuestType = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestType") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestType:%v->%v", p.AlternativeQuestType, v)) } + p.AlternativeQuestType = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { if p.AlternativeQuestTimestamp != v { - p.AlternativeQuestTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTimestamp:%v->%v", p.AlternativeQuestTimestamp, v)) } + p.AlternativeQuestTimestamp = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { if p.AlternativeQuestTarget != v { - p.AlternativeQuestTarget = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestTarget") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTarget:%v->%v", p.AlternativeQuestTarget, v)) } + p.AlternativeQuestTarget = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { if p.AlternativeQuestConditions != v { - p.AlternativeQuestConditions = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestConditions") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestConditions:%v->%v", p.AlternativeQuestConditions, v)) } + p.AlternativeQuestConditions = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { if p.AlternativeQuestRewards != v { - p.AlternativeQuestRewards = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestRewards") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestRewards:%v->%v", p.AlternativeQuestRewards, v)) } + p.AlternativeQuestRewards = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { if p.AlternativeQuestTemplate != v { - p.AlternativeQuestTemplate = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestTemplate") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTemplate:%v->%v", p.AlternativeQuestTemplate, v)) } + p.AlternativeQuestTemplate = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { if p.AlternativeQuestTitle != v { - p.AlternativeQuestTitle = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestTitle") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTitle:%v->%v", p.AlternativeQuestTitle, v)) } + p.AlternativeQuestTitle = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { if p.AlternativeQuestExpiry != v { - p.AlternativeQuestExpiry = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestExpiry") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestExpiry:%v->%v", p.AlternativeQuestExpiry, v)) } + p.AlternativeQuestExpiry = v + p.dirty = true } } func (p *Pokestop) SetDescription(v null.String) { if p.Description != v { - p.Description = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Description") + p.changedFields = append(p.changedFields, fmt.Sprintf("Description:%v->%v", p.Description, v)) } + p.Description = v + p.dirty = true } } func (p *Pokestop) SetShowcaseFocus(v null.String) { if p.ShowcaseFocus != v { - p.ShowcaseFocus = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcaseFocus") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseFocus:%v->%v", p.ShowcaseFocus, v)) } + p.ShowcaseFocus = v + p.dirty = true } } func (p *Pokestop) SetShowcasePokemon(v null.Int) { if p.ShowcasePokemon != v { - p.ShowcasePokemon = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcasePokemon") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemon:%v->%v", p.ShowcasePokemon, v)) } + p.ShowcasePokemon = v + p.dirty = true } } func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { if p.ShowcasePokemonForm != v { - p.ShowcasePokemonForm = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcasePokemonForm") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonForm:%v->%v", p.ShowcasePokemonForm, v)) } + p.ShowcasePokemonForm = v + p.dirty = true } } func (p *Pokestop) SetShowcasePokemonType(v null.Int) { if p.ShowcasePokemonType != v { - p.ShowcasePokemonType = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcasePokemonType") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonType:%v->%v", p.ShowcasePokemonType, v)) } + p.ShowcasePokemonType = v + p.dirty = true } } func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { if p.ShowcaseRankingStandard != v { - p.ShowcaseRankingStandard = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcaseRankingStandard") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankingStandard:%v->%v", p.ShowcaseRankingStandard, v)) } + p.ShowcaseRankingStandard = v + p.dirty = true } } func (p *Pokestop) SetShowcaseExpiry(v null.Int) { if p.ShowcaseExpiry != v { - p.ShowcaseExpiry = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcaseExpiry") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseExpiry:%v->%v", p.ShowcaseExpiry, v)) } + p.ShowcaseExpiry = v + p.dirty = true } } func (p *Pokestop) SetShowcaseRankings(v null.String) { if p.ShowcaseRankings != v { - p.ShowcaseRankings = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcaseRankings") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankings:%v->%v", p.ShowcaseRankings, v)) } + p.ShowcaseRankings = v + p.dirty = true } } func (p *Pokestop) SetUpdated(v int64) { if p.Updated != v { - p.Updated = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Updated") + p.changedFields = append(p.changedFields, fmt.Sprintf("Updated:%d->%d", p.Updated, v)) } + p.Updated = v + p.dirty = true } } diff --git a/decoder/routes.go b/decoder/routes.go index 2a6aaf75..0fa7ee6a 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -83,210 +84,210 @@ func (r *Route) snapshotOldValues() { func (r *Route) SetName(v string) { if r.Name != v { - r.Name = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Name") + r.changedFields = append(r.changedFields, fmt.Sprintf("Name:%s->%s", r.Name, v)) } + r.Name = v + r.dirty = true } } func (r *Route) SetShortcode(v string) { if r.Shortcode != v { - r.Shortcode = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Shortcode") + r.changedFields = append(r.changedFields, fmt.Sprintf("Shortcode:%s->%s", r.Shortcode, v)) } + r.Shortcode = v + r.dirty = true } } func (r *Route) SetDescription(v string) { if r.Description != v { - r.Description = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Description") + r.changedFields = append(r.changedFields, fmt.Sprintf("Description:%s->%s", r.Description, v)) } + r.Description = v + r.dirty = true } } func (r *Route) SetDistanceMeters(v int64) { if r.DistanceMeters != v { - r.DistanceMeters = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "DistanceMeters") + r.changedFields = append(r.changedFields, fmt.Sprintf("DistanceMeters:%d->%d", r.DistanceMeters, v)) } + r.DistanceMeters = v + r.dirty = true } } func (r *Route) SetDurationSeconds(v int64) { if r.DurationSeconds != v { - r.DurationSeconds = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "DurationSeconds") + r.changedFields = append(r.changedFields, fmt.Sprintf("DurationSeconds:%d->%d", r.DurationSeconds, v)) } + r.DurationSeconds = v + r.dirty = true } } func (r *Route) SetEndFortId(v string) { if r.EndFortId != v { - r.EndFortId = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "EndFortId") + r.changedFields = append(r.changedFields, fmt.Sprintf("EndFortId:%s->%s", r.EndFortId, v)) } + r.EndFortId = v + r.dirty = true } } func (r *Route) SetEndImage(v string) { if r.EndImage != v { - r.EndImage = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "EndImage") + r.changedFields = append(r.changedFields, fmt.Sprintf("EndImage:%s->%s", r.EndImage, v)) } + r.EndImage = v + r.dirty = true } } func (r *Route) SetEndLat(v float64) { if !floatAlmostEqual(r.EndLat, v, floatTolerance) { - r.EndLat = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "EndLat") + r.changedFields = append(r.changedFields, fmt.Sprintf("EndLat:%f->%f", r.EndLat, v)) } + r.EndLat = v + r.dirty = true } } func (r *Route) SetEndLon(v float64) { if !floatAlmostEqual(r.EndLon, v, floatTolerance) { - r.EndLon = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "EndLon") + r.changedFields = append(r.changedFields, fmt.Sprintf("EndLon:%f->%f", r.EndLon, v)) } + r.EndLon = v + r.dirty = true } } func (r *Route) SetImage(v string) { if r.Image != v { - r.Image = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Image") + r.changedFields = append(r.changedFields, fmt.Sprintf("Image:%s->%s", r.Image, v)) } + r.Image = v + r.dirty = true } } func (r *Route) SetImageBorderColor(v string) { if r.ImageBorderColor != v { - r.ImageBorderColor = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "ImageBorderColor") + r.changedFields = append(r.changedFields, fmt.Sprintf("ImageBorderColor:%s->%s", r.ImageBorderColor, v)) } + r.ImageBorderColor = v + r.dirty = true } } func (r *Route) SetReversible(v bool) { if r.Reversible != v { - r.Reversible = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Reversible") + r.changedFields = append(r.changedFields, fmt.Sprintf("Reversible:%t->%t", r.Reversible, v)) } + r.Reversible = v + r.dirty = true } } func (r *Route) SetStartFortId(v string) { if r.StartFortId != v { - r.StartFortId = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "StartFortId") + r.changedFields = append(r.changedFields, fmt.Sprintf("StartFortId:%s->%s", r.StartFortId, v)) } + r.StartFortId = v + r.dirty = true } } func (r *Route) SetStartImage(v string) { if r.StartImage != v { - r.StartImage = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "StartImage") + r.changedFields = append(r.changedFields, fmt.Sprintf("StartImage:%s->%s", r.StartImage, v)) } + r.StartImage = v + r.dirty = true } } func (r *Route) SetStartLat(v float64) { if !floatAlmostEqual(r.StartLat, v, floatTolerance) { - r.StartLat = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "StartLat") + r.changedFields = append(r.changedFields, fmt.Sprintf("StartLat:%f->%f", r.StartLat, v)) } + r.StartLat = v + r.dirty = true } } func (r *Route) SetStartLon(v float64) { if !floatAlmostEqual(r.StartLon, v, floatTolerance) { - r.StartLon = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "StartLon") + r.changedFields = append(r.changedFields, fmt.Sprintf("StartLon:%f->%f", r.StartLon, v)) } + r.StartLon = v + r.dirty = true } } func (r *Route) SetTags(v null.String) { if r.Tags != v { - r.Tags = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Tags") + r.changedFields = append(r.changedFields, fmt.Sprintf("Tags:%v->%v", r.Tags, v)) } + r.Tags = v + r.dirty = true } } func (r *Route) SetType(v int8) { if r.Type != v { - r.Type = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Type") + r.changedFields = append(r.changedFields, fmt.Sprintf("Type:%d->%d", r.Type, v)) } + r.Type = v + r.dirty = true } } func (r *Route) SetVersion(v int64) { if r.Version != v { - r.Version = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Version") + r.changedFields = append(r.changedFields, fmt.Sprintf("Version:%d->%d", r.Version, v)) } + r.Version = v + r.dirty = true } } func (r *Route) SetWaypoints(v string) { if r.Waypoints != v { - r.Waypoints = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Waypoints") + r.changedFields = append(r.changedFields, fmt.Sprintf("Waypoints:%s->%s", r.Waypoints, v)) } + r.Waypoints = v + r.dirty = true } } func (r *Route) SetUpdated(v int64) { if r.Updated != v { - r.Updated = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Updated") + r.changedFields = append(r.changedFields, fmt.Sprintf("Updated:%d->%d", r.Updated, v)) } + r.Updated = v + r.dirty = true } } diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 736b302f..60513492 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -69,7 +69,7 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { for _, s2cell := range outputCellIds { updatedCells = append(updatedCells, strconv.FormatUint(s2cell.Id, 10)) } - log.Debugf("[DB_S2CELL] Updated cells: %s", strings.Join(updatedCells, ",")) + log.Debugf("[DB_UPDATE] S2Cell Updated cells: %s", strings.Join(updatedCells, ",")) } // run bulk query diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 9a3a4a8a..9171a852 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "strconv" "sync" "time" @@ -28,8 +29,9 @@ type Spawnpoint struct { LastSeen int64 `db:"last_seen"` DespawnSec null.Int `db:"despawn_sec"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) } //CREATE TABLE `spawnpoint` ( @@ -74,6 +76,9 @@ func (s *Spawnpoint) Unlock() { func (s *Spawnpoint) SetLat(v float64) { if !floatAlmostEqual(s.Lat, v, floatTolerance) { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("Lat:%f->%f", s.Lat, v)) + } s.Lat = v s.dirty = true } @@ -81,6 +86,9 @@ func (s *Spawnpoint) SetLat(v float64) { func (s *Spawnpoint) SetLon(v float64) { if !floatAlmostEqual(s.Lon, v, floatTolerance) { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("Lon:%f->%f", s.Lon, v)) + } s.Lon = v s.dirty = true } @@ -90,6 +98,9 @@ func (s *Spawnpoint) SetLon(v float64) { func (s *Spawnpoint) SetDespawnSec(v null.Int) { // Handle validity changes if (s.DespawnSec.Valid && !v.Valid) || (!s.DespawnSec.Valid && v.Valid) { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%v->%v", s.DespawnSec, v)) + } s.DespawnSec = v s.dirty = true return @@ -114,6 +125,9 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { // Allow 2-second tolerance for despawn time if Abs(oldVal-newVal) > 2 { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%v->%v", s.DespawnSec, v)) + } s.DespawnSec = v s.dirty = true } @@ -121,6 +135,9 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { func (s *Spawnpoint) SetUpdated(v int64) { if s.Updated != v { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("Updated:%d->%d", s.Updated, v)) + } s.Updated = v s.dirty = true } @@ -128,6 +145,9 @@ func (s *Spawnpoint) SetUpdated(v int64) { func (s *Spawnpoint) SetLastSeen(v int64) { if s.LastSeen != v { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("LastSeen:%d->%d", s.LastSeen, v)) + } s.LastSeen = v s.dirty = true } @@ -266,6 +286,14 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi spawnpoint.SetUpdated(time.Now().Unix()) // ensure future updates are set correctly spawnpoint.SetLastSeen(time.Now().Unix()) // ensure future updates are set correctly + if dbDebugEnabled { + if spawnpoint.IsNewRecord() { + dbDebugLog("INSERT", "Spawnpoint", strconv.FormatInt(spawnpoint.Id, 10), spawnpoint.changedFields) + } else { + dbDebugLog("UPDATE", "Spawnpoint", strconv.FormatInt(spawnpoint.Id, 10), spawnpoint.changedFields) + } + } + _, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO spawnpoint (id, lat, lon, updated, last_seen, despawn_sec)"+ "VALUES (:id, :lat, :lon, :updated, :last_seen, :despawn_sec)"+ "ON DUPLICATE KEY UPDATE "+ diff --git a/decoder/station.go b/decoder/station.go index d46bd2c2..8eb1e307 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -102,270 +103,270 @@ func (station *Station) snapshotOldValues() { func (station *Station) SetId(v string) { if station.Id != v { - station.Id = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Id") + station.changedFields = append(station.changedFields, fmt.Sprintf("Id:%s->%s", station.Id, v)) } + station.Id = v + station.dirty = true } } func (station *Station) SetLat(v float64) { if !floatAlmostEqual(station.Lat, v, floatTolerance) { - station.Lat = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Lat") + station.changedFields = append(station.changedFields, fmt.Sprintf("Lat:%f->%f", station.Lat, v)) } + station.Lat = v + station.dirty = true } } func (station *Station) SetLon(v float64) { if !floatAlmostEqual(station.Lon, v, floatTolerance) { - station.Lon = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Lon") + station.changedFields = append(station.changedFields, fmt.Sprintf("Lon:%f->%f", station.Lon, v)) } + station.Lon = v + station.dirty = true } } func (station *Station) SetName(v string) { if station.Name != v { - station.Name = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Name") + station.changedFields = append(station.changedFields, fmt.Sprintf("Name:%s->%s", station.Name, v)) } + station.Name = v + station.dirty = true } } func (station *Station) SetCellId(v int64) { if station.CellId != v { - station.CellId = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "CellId") + station.changedFields = append(station.changedFields, fmt.Sprintf("CellId:%d->%d", station.CellId, v)) } + station.CellId = v + station.dirty = true } } func (station *Station) SetStartTime(v int64) { if station.StartTime != v { - station.StartTime = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "StartTime") + station.changedFields = append(station.changedFields, fmt.Sprintf("StartTime:%d->%d", station.StartTime, v)) } + station.StartTime = v + station.dirty = true } } func (station *Station) SetEndTime(v int64) { if station.EndTime != v { - station.EndTime = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "EndTime") + station.changedFields = append(station.changedFields, fmt.Sprintf("EndTime:%d->%d", station.EndTime, v)) } + station.EndTime = v + station.dirty = true } } func (station *Station) SetCooldownComplete(v int64) { if station.CooldownComplete != v { - station.CooldownComplete = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "CooldownComplete") + station.changedFields = append(station.changedFields, fmt.Sprintf("CooldownComplete:%d->%d", station.CooldownComplete, v)) } + station.CooldownComplete = v + station.dirty = true } } func (station *Station) SetIsBattleAvailable(v bool) { if station.IsBattleAvailable != v { - station.IsBattleAvailable = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "IsBattleAvailable") + station.changedFields = append(station.changedFields, fmt.Sprintf("IsBattleAvailable:%t->%t", station.IsBattleAvailable, v)) } + station.IsBattleAvailable = v + station.dirty = true } } func (station *Station) SetIsInactive(v bool) { if station.IsInactive != v { - station.IsInactive = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "IsInactive") + station.changedFields = append(station.changedFields, fmt.Sprintf("IsInactive:%t->%t", station.IsInactive, v)) } + station.IsInactive = v + station.dirty = true } } func (station *Station) SetBattleLevel(v null.Int) { if station.BattleLevel != v { - station.BattleLevel = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattleLevel") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleLevel:%v->%v", station.BattleLevel, v)) } + station.BattleLevel = v + station.dirty = true } } func (station *Station) SetBattleStart(v null.Int) { if station.BattleStart != v { - station.BattleStart = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattleStart") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleStart:%v->%v", station.BattleStart, v)) } + station.BattleStart = v + station.dirty = true } } func (station *Station) SetBattleEnd(v null.Int) { if station.BattleEnd != v { - station.BattleEnd = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattleEnd") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleEnd:%v->%v", station.BattleEnd, v)) } + station.BattleEnd = v + station.dirty = true } } func (station *Station) SetBattlePokemonId(v null.Int) { if station.BattlePokemonId != v { - station.BattlePokemonId = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonId") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonId:%v->%v", station.BattlePokemonId, v)) } + station.BattlePokemonId = v + station.dirty = true } } func (station *Station) SetBattlePokemonForm(v null.Int) { if station.BattlePokemonForm != v { - station.BattlePokemonForm = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonForm") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonForm:%v->%v", station.BattlePokemonForm, v)) } + station.BattlePokemonForm = v + station.dirty = true } } func (station *Station) SetBattlePokemonCostume(v null.Int) { if station.BattlePokemonCostume != v { - station.BattlePokemonCostume = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonCostume") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCostume:%v->%v", station.BattlePokemonCostume, v)) } + station.BattlePokemonCostume = v + station.dirty = true } } func (station *Station) SetBattlePokemonGender(v null.Int) { if station.BattlePokemonGender != v { - station.BattlePokemonGender = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonGender") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonGender:%v->%v", station.BattlePokemonGender, v)) } + station.BattlePokemonGender = v + station.dirty = true } } func (station *Station) SetBattlePokemonAlignment(v null.Int) { if station.BattlePokemonAlignment != v { - station.BattlePokemonAlignment = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonAlignment") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonAlignment:%v->%v", station.BattlePokemonAlignment, v)) } + station.BattlePokemonAlignment = v + station.dirty = true } } func (station *Station) SetBattlePokemonBreadMode(v null.Int) { if station.BattlePokemonBreadMode != v { - station.BattlePokemonBreadMode = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonBreadMode") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonBreadMode:%v->%v", station.BattlePokemonBreadMode, v)) } + station.BattlePokemonBreadMode = v + station.dirty = true } } func (station *Station) SetBattlePokemonMove1(v null.Int) { if station.BattlePokemonMove1 != v { - station.BattlePokemonMove1 = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonMove1") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove1:%v->%v", station.BattlePokemonMove1, v)) } + station.BattlePokemonMove1 = v + station.dirty = true } } func (station *Station) SetBattlePokemonMove2(v null.Int) { if station.BattlePokemonMove2 != v { - station.BattlePokemonMove2 = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonMove2") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove2:%v->%v", station.BattlePokemonMove2, v)) } + station.BattlePokemonMove2 = v + station.dirty = true } } func (station *Station) SetBattlePokemonStamina(v null.Int) { if station.BattlePokemonStamina != v { - station.BattlePokemonStamina = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonStamina") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonStamina:%v->%v", station.BattlePokemonStamina, v)) } + station.BattlePokemonStamina = v + station.dirty = true } } func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { if !nullFloatAlmostEqual(station.BattlePokemonCpMultiplier, v, floatTolerance) { - station.BattlePokemonCpMultiplier = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonCpMultiplier") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCpMultiplier:%v->%v", station.BattlePokemonCpMultiplier, v)) } + station.BattlePokemonCpMultiplier = v + station.dirty = true } } func (station *Station) SetTotalStationedPokemon(v null.Int) { if station.TotalStationedPokemon != v { - station.TotalStationedPokemon = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "TotalStationedPokemon") + station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedPokemon:%v->%v", station.TotalStationedPokemon, v)) } + station.TotalStationedPokemon = v + station.dirty = true } } func (station *Station) SetTotalStationedGmax(v null.Int) { if station.TotalStationedGmax != v { - station.TotalStationedGmax = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "TotalStationedGmax") + station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedGmax:%v->%v", station.TotalStationedGmax, v)) } + station.TotalStationedGmax = v + station.dirty = true } } func (station *Station) SetStationedPokemon(v null.String) { if station.StationedPokemon != v { - station.StationedPokemon = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "StationedPokemon") + station.changedFields = append(station.changedFields, fmt.Sprintf("StationedPokemon:%v->%v", station.StationedPokemon, v)) } + station.StationedPokemon = v + station.dirty = true } } func (station *Station) SetUpdated(v int64) { if station.Updated != v { - station.Updated = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Updated") + station.changedFields = append(station.changedFields, fmt.Sprintf("Updated:%d->%d", station.Updated, v)) } + station.Updated = v + station.dirty = true } } diff --git a/decoder/tappable.go b/decoder/tappable.go index 94ab1332..98573571 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -58,110 +59,110 @@ func (ta *Tappable) Unlock() { func (ta *Tappable) SetLat(v float64) { if !floatAlmostEqual(ta.Lat, v, floatTolerance) { - ta.Lat = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Lat") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Lat:%f->%f", ta.Lat, v)) } + ta.Lat = v + ta.dirty = true } } func (ta *Tappable) SetLon(v float64) { if !floatAlmostEqual(ta.Lon, v, floatTolerance) { - ta.Lon = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Lon") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Lon:%f->%f", ta.Lon, v)) } + ta.Lon = v + ta.dirty = true } } func (ta *Tappable) SetFortId(v null.String) { if ta.FortId != v { - ta.FortId = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "FortId") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("FortId:%v->%v", ta.FortId, v)) } + ta.FortId = v + ta.dirty = true } } func (ta *Tappable) SetSpawnId(v null.Int) { if ta.SpawnId != v { - ta.SpawnId = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "SpawnId") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("SpawnId:%v->%v", ta.SpawnId, v)) } + ta.SpawnId = v + ta.dirty = true } } func (ta *Tappable) SetType(v string) { if ta.Type != v { - ta.Type = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Type") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Type:%s->%s", ta.Type, v)) } + ta.Type = v + ta.dirty = true } } func (ta *Tappable) SetEncounter(v null.Int) { if ta.Encounter != v { - ta.Encounter = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Encounter") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Encounter:%v->%v", ta.Encounter, v)) } + ta.Encounter = v + ta.dirty = true } } func (ta *Tappable) SetItemId(v null.Int) { if ta.ItemId != v { - ta.ItemId = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "ItemId") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ItemId:%v->%v", ta.ItemId, v)) } + ta.ItemId = v + ta.dirty = true } } func (ta *Tappable) SetCount(v null.Int) { if ta.Count != v { - ta.Count = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Count") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Count:%v->%v", ta.Count, v)) } + ta.Count = v + ta.dirty = true } } func (ta *Tappable) SetExpireTimestamp(v null.Int) { if ta.ExpireTimestamp != v { - ta.ExpireTimestamp = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "ExpireTimestamp") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ExpireTimestamp:%v->%v", ta.ExpireTimestamp, v)) } + ta.ExpireTimestamp = v + ta.dirty = true } } func (ta *Tappable) SetExpireTimestampVerified(v bool) { if ta.ExpireTimestampVerified != v { - ta.ExpireTimestampVerified = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "ExpireTimestampVerified") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ExpireTimestampVerified:%t->%t", ta.ExpireTimestampVerified, v)) } + ta.ExpireTimestampVerified = v + ta.dirty = true } } func (ta *Tappable) SetUpdated(v int64) { if ta.Updated != v { - ta.Updated = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Updated") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Updated:%d->%d", ta.Updated, v)) } + ta.Updated = v + ta.dirty = true } } From ba42ef9a82c21481864cb2b3296ae0961def1241 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 22:20:07 +0000 Subject: [PATCH 32/78] Ability to turn off nearby cell pokemon --- config/config.go | 1 + decoder/gmo_decode.go | 24 +++++++++++++----------- decoder/scanarea.go | 16 +++++++++++++++- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/config/config.go b/config/config.go index ab2423d4..6eacd7da 100644 --- a/config/config.go +++ b/config/config.go @@ -134,6 +134,7 @@ type scanRule struct { ProcessPokemon *bool `koanf:"pokemon"` ProcessWilds *bool `koanf:"wild_pokemon"` ProcessNearby *bool `koanf:"nearby_pokemon"` + ProcessNearbyCell *bool `koanf:"nearby_cell_pokemon"` ProcessWeather *bool `koanf:"weather"` ProcessCells *bool `koanf:"cells"` ProcessPokestops *bool `koanf:"pokestops"` diff --git a/decoder/gmo_decode.go b/decoder/gmo_decode.go index 95784faf..8f2a7f54 100644 --- a/decoder/gmo_decode.go +++ b/decoder/gmo_decode.go @@ -145,19 +145,21 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca for _, nearby := range nearbyPokemonList { encounterId := nearby.Data.EncounterId - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Printf("getOrCreatePokemonRecord: %s", err) - continue - } + if nearby.Data.FortId != "" || scanParameters.ProcessNearbyCell { + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Printf("getOrCreatePokemonRecord: %s", err) + continue + } - updateTime := nearby.Timestamp / 1000 - if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { - pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) - } + updateTime := nearby.Timestamp / 1000 + if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { + pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) + } - unlock() + unlock() + } } } diff --git a/decoder/scanarea.go b/decoder/scanarea.go index 9e7831ae..7b52636f 100644 --- a/decoder/scanarea.go +++ b/decoder/scanarea.go @@ -1,15 +1,17 @@ package decoder import ( + "strings" + "golbat/config" "golbat/geo" - "strings" ) type ScanParameters struct { ProcessPokemon bool ProcessWild bool ProcessNearby bool + ProcessNearbyCell bool ProcessWeather bool ProcessPokestops bool ProcessGyms bool @@ -56,6 +58,16 @@ func FindScanConfiguration(scanContext string, lat, lon float64) ScanParameters return *value } + defaultTrueFirst := func(value *bool, value2 *bool) bool { + if value != nil { + return *value + } + if value2 != nil { + return *value2 + } + return true + } + defaultFromWeatherConfig := func(value *bool, weatherDefault bool) bool { if value == nil { return weatherDefault @@ -67,6 +79,7 @@ func FindScanConfiguration(scanContext string, lat, lon float64) ScanParameters ProcessPokemon: defaultTrue(rule.ProcessPokemon), ProcessWild: defaultTrue(rule.ProcessWilds), ProcessNearby: defaultTrue(rule.ProcessNearby), + ProcessNearbyCell: defaultTrueFirst(rule.ProcessNearbyCell, rule.ProcessNearby), ProcessCells: defaultTrue(rule.ProcessCells), ProcessWeather: defaultTrue(rule.ProcessWeather), ProcessPokestops: defaultTrue(rule.ProcessPokestops), @@ -82,6 +95,7 @@ func FindScanConfiguration(scanContext string, lat, lon float64) ScanParameters ProcessPokemon: true, ProcessWild: true, ProcessNearby: true, + ProcessNearbyCell: true, ProcessCells: true, ProcessWeather: true, ProcessGyms: true, From dba646a78462cf01d9637925d1ac2d11cdf25fa0 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 22:37:47 +0000 Subject: [PATCH 33/78] API documentation --- api.md | 760 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 760 insertions(+) create mode 100644 api.md diff --git a/api.md b/api.md new file mode 100644 index 00000000..953f71c5 --- /dev/null +++ b/api.md @@ -0,0 +1,760 @@ +# Golbat API Documentation + +Golbat provides both HTTP REST and gRPC APIs for querying Pokemon GO data. + +## Table of Contents + +- [Authentication](#authentication) +- [Health Check](#health-check) +- [Raw Data Ingestion](#raw-data-ingestion) +- [Pokemon Endpoints](#pokemon-endpoints) +- [Pokestop Endpoints](#pokestop-endpoints) +- [Gym Endpoints](#gym-endpoints) +- [Quest Endpoints](#quest-endpoints) +- [Tappable Endpoints](#tappable-endpoints) +- [Device Endpoints](#device-endpoints) +- [Debug Endpoints](#debug-endpoints) +- [gRPC API](#grpc-api) +- [Data Structures](#data-structures) + +--- + +## Authentication + +### API Authentication + +All `/api/*` endpoints require authentication via the `X-Golbat-Secret` header. + +``` +X-Golbat-Secret: your_api_secret +``` + +The secret is configured via `api_secret` in the configuration file. + +### Raw Endpoint Authentication + +The `/raw` endpoint optionally supports Bearer token authentication: + +``` +Authorization: Bearer your_raw_bearer_token +``` + +This is only enforced if `raw_bearer` is configured. + +--- + +## Health Check + +### GET /health + +Unrestricted health check endpoint for monitoring. + +**Authentication:** Not required + +**Response:** +```json +{ + "status": "ok" +} +``` + +### GET /api/health + +Authenticated health check endpoint. + +**Authentication:** Required + +**Response:** +```json +{ + "status": "ok" +} +``` + +--- + +## Raw Data Ingestion + +### POST /raw + +Accept raw protobuf data from scanning clients. + +**Authentication:** Bearer token (optional, if configured) + +**Request Body:** +```json +{ + "uuid": "device_uuid", + "username": "account_name", + "trainerlvl": 30, + "scan_context": "context_string", + "lat_target": 40.7128, + "lon_target": -74.0060, + "timestamp_ms": 1234567890, + "have_ar": true, + "contents": [ + { + "payload": "base64_encoded_proto", + "type": 1, + "request": "optional_request_proto" + } + ] +} +``` + +**Response:** HTTP 201 Created (async processing) + +**Notes:** +- Multiple provider formats supported (Pogodroid, standard format) +- Processing timeout: 5s normal, 30s if `extended_timeout` enabled +- Content can use `data` or `payload` for the proto data +- Content can use `method` or `type` for the method number + +--- + +## Pokemon Endpoints + +### GET /api/pokemon/id/:pokemon_id + +Retrieve a single pokemon by encounter ID. + +**Authentication:** Required + +**Parameters:** +| Name | Type | Location | Description | +|------|------|----------|-------------| +| pokemon_id | uint64 | path | Pokemon encounter ID | + +**Response:** [ApiPokemonResult](#apipokemonresult) + +**Status Codes:** +- 200: Pokemon found +- 404: Pokemon not found + +--- + +### GET /api/pokemon/available + +List all available pokemon species with counts. + +**Authentication:** Required + +**Response:** +```json +[ + { + "id": 1, + "form": 0, + "count": 42 + } +] +``` + +--- + +### POST /api/pokemon/scan + +Query pokemon in a geographic area with filters (v1 - legacy). + +**Authentication:** Required + +**Request Body:** +```json +{ + "min": {"lat": 40.7, "lon": -74.0}, + "max": {"lat": 40.8, "lon": -73.9}, + "center": {"lat": 40.75, "lon": -73.95}, + "limit": 500, + "global": { + "iv": [0, 100], + "atk_iv": [0, 15], + "def_iv": [0, 15], + "sta_iv": [0, 15], + "level": [1, 50], + "cp": [0, 3000], + "gender": 1, + "additional": { + "include_everything": false, + "include_hundoiv": true, + "include_zeroiv": false, + "include_xxs": true, + "include_xxl": false + }, + "pvp": { + "little": [1, 100], + "great": [1, 100], + "ultra": [1, 100] + } + }, + "filters": { + "1-0": {} + } +} +``` + +**Response:** Array of [ApiPokemonResult](#apipokemonresult) + +--- + +### POST /api/pokemon/v2/scan + +Query pokemon with DNF (Disjunctive Normal Form) filters - more efficient filtering. + +**Authentication:** Required + +**Request Body:** +```json +{ + "min": {"lat": 40.7, "lon": -74.0}, + "max": {"lat": 40.8, "lon": -73.9}, + "limit": 500, + "filters": [ + { + "pokemon": [{"id": 1, "form": 0}], + "iv": {"min": 90, "max": 100}, + "atk_iv": {"min": 10, "max": 15}, + "def_iv": {"min": 10, "max": 15}, + "sta_iv": {"min": 10, "max": 15}, + "level": {"min": 30, "max": 50}, + "cp": {"min": 2000, "max": 3000}, + "gender": {"min": 0, "max": 2}, + "size": {"min": 0, "max": 5}, + "pvp_little": {"min": 1, "max": 100}, + "pvp_great": {"min": 1, "max": 100}, + "pvp_ultra": {"min": 1, "max": 100} + } + ] +} +``` + +**Response:** Array of [ApiPokemonResult](#apipokemonresult) + +--- + +### POST /api/pokemon/v3/scan + +Query pokemon with advanced DNF filters, returns metadata about scan. + +**Authentication:** Required + +**Request Body:** Same as v2, with gender as array + +**Response:** +```json +{ + "pokemon": [], + "examined": 1000, + "skipped": 50, + "total": 1050 +} +``` + +--- + +### POST /api/pokemon/search + +Advanced search using center point and distance. + +**Authentication:** Required + +**Request Body:** +```json +{ + "min": {"lat": 40.7, "lon": -74.0}, + "max": {"lat": 40.8, "lon": -73.9}, + "center": {"lat": 40.75, "lon": -73.95}, + "limit": 500, + "searchIds": [1, 4, 7] +} +``` + +**Response:** Array of [ApiPokemonResult](#apipokemonresult) + +**Status Codes:** +- 200: Success +- 400: Bad Request (validation failed) + +--- + +## Pokestop Endpoints + +### GET /api/pokestop/id/:fort_id + +Retrieve a single pokestop by fort ID. + +**Authentication:** Required + +**Parameters:** +| Name | Type | Location | Description | +|------|------|----------|-------------| +| fort_id | string | path | Pokestop fort ID | + +**Response:** [ApiPokestopResult](#apipokestopresult) + +**Status Codes:** +- 200: Pokestop found +- 404: Pokestop not found + +--- + +### POST /api/pokestop-positions + +Get coordinates of all pokestops within a geofence. + +**Authentication:** Required + +**Request Body:** GeoJSON Feature, Geometry, or Golbat Geofence format +```json +{ + "fence": [ + {"lat": 40.7, "lon": -74.0}, + {"lat": 40.8, "lon": -74.0}, + {"lat": 40.8, "lon": -73.9}, + {"lat": 40.7, "lon": -73.9} + ] +} +``` + +**Response:** +```json +[ + { + "id": "fort_id", + "latitude": 40.7128, + "longitude": -74.0060 + } +] +``` + +--- + +## Gym Endpoints + +### GET /api/gym/id/:gym_id + +Retrieve a single gym by gym ID. + +**Authentication:** Required + +**Parameters:** +| Name | Type | Location | Description | +|------|------|----------|-------------| +| gym_id | string | path | Gym ID | + +**Response:** [ApiGymResult](#apigymresult) + +**Status Codes:** +- 200: Gym found +- 404: Gym not found + +--- + +### POST /api/gym/query + +Get multiple gyms by IDs. + +**Authentication:** Required + +**Request Body:** +```json +{ + "ids": ["gym_id1", "gym_id2"] +} +``` +Or as an array: +```json +["gym_id1", "gym_id2"] +``` + +**Response:** Array of [ApiGymResult](#apigymresult) + +**Limits:** +- Maximum 500 IDs per request +- Duplicates are filtered + +**Status Codes:** +- 200: Success +- 413: Request Entity Too Large (exceeds 500 IDs) + +--- + +### POST /api/gym/search + +Advanced gym search with filters. + +**Authentication:** Required + +**Request Body:** +```json +{ + "filters": [ + { + "name": "central park", + "description": "playground", + "location_distance": { + "location": {"lat": 40.7829, "lon": -73.9654}, + "distance": 500 + }, + "bbox": { + "min_lon": -74.0, + "min_lat": 40.7, + "max_lon": -73.9, + "max_lat": 40.8 + } + } + ], + "limit": 100 +} +``` + +**Response:** Array of [ApiGymResult](#apigymresult) + +**Limits:** +- Default limit: 500 +- Max limit: 10,000 +- Max distance: 500,000 meters + +**Status Codes:** +- 200: Success +- 400: Bad Request (invalid filters) +- 504: Gateway Timeout + +--- + +## Quest Endpoints + +### POST /api/quest-status + +Get quest statistics for a geofence area. + +**Authentication:** Required + +**Request Body:** GeoJSON Feature, Geometry, or Golbat Geofence format + +**Response:** +```json +{ + "ar_quests": 50, + "no_ar_quests": 100, + "total": 200 +} +``` + +--- + +### POST /api/clear-quests +### DELETE /api/clear-quests + +Clear all quests within a geofence area. + +**Authentication:** Required + +**Request Body:** GeoJSON Feature, Geometry, or Golbat Geofence format + +**Response:** +```json +{ + "status": "ok" +} +``` + +--- + +### POST /api/reload-geojson +### GET /api/reload-geojson + +Reload geofence boundaries and clear stats. + +**Authentication:** Required + +**Response:** +```json +{ + "status": "ok" +} +``` + +--- + +## Tappable Endpoints + +### GET /api/tappable/id/:tappable_id + +Retrieve a tappable (invasions, research, etc.). + +**Authentication:** Required + +**Parameters:** +| Name | Type | Location | Description | +|------|------|----------|-------------| +| tappable_id | uint64 | path | Tappable ID | + +**Response:** [ApiTappableResult](#apitappableresult) + +**Status Codes:** +- 200: Tappable found +- 400: Invalid ID +- 404: Tappable not found + +--- + +## Device Endpoints + +### GET /api/devices/all + +Get information about all connected/known devices. + +**Authentication:** Required + +**Response:** +```json +{ + "devices": [ + { + "uuid": "device_uuid", + "lat": 40.7128, + "lon": -74.0060, + "last_scan": 1234567890 + } + ] +} +``` + +--- + +## Debug Endpoints + +These endpoints are only available if `tuning.profile_routes` is enabled in configuration. + +**Authentication:** Required + +| Endpoint | Description | +|----------|-------------| +| GET /debug/pprof/cmdline | Command line arguments | +| GET /debug/pprof/heap | Heap memory profile | +| GET /debug/pprof/block | Block profile | +| GET /debug/pprof/mutex | Mutex profile | +| GET /debug/pprof/trace | Execution trace | +| GET /debug/pprof/profile | CPU profile | +| GET /debug/pprof/symbol | Symbol lookup | + +--- + +## gRPC API + +Golbat also provides a gRPC API running on a separate port (configured via `grpc_port`). + +### Authentication + +Use the `authorization` metadata header with the API secret. + +### Pokemon Service + +```protobuf +service Pokemon { + rpc Search(PokemonScanRequest) returns (PokemonScanResponse); + rpc SearchV3(PokemonScanRequestV3) returns (PokemonScanResponseV3); +} +``` + +The gRPC endpoints mirror the HTTP v2/v3 scan endpoints. + +--- + +## Data Structures + +### Location + +```json +{ + "lat": 40.7128, + "lon": -74.0060 +} +``` + +### Bounding Box (Bbox) + +```json +{ + "min_lon": -74.0, + "min_lat": 40.7, + "max_lon": -73.9, + "max_lat": 40.8 +} +``` + +### ApiPokemonResult + +```json +{ + "id": "encounter_id", + "pokestop_id": "fort_id_or_null", + "spawn_id": 123456789, + "lat": 40.7128, + "lon": -74.0060, + "weight": 5.5, + "size": 2, + "height": 0.8, + "expire_timestamp": 1234567890, + "updated": 1234567800, + "pokemon_id": 1, + "move_1": 100, + "move_2": 200, + "gender": 1, + "cp": 500, + "atk_iv": 15, + "def_iv": 15, + "sta_iv": 15, + "iv": 100.0, + "form": 0, + "level": 30, + "weather": 1, + "costume": 0, + "first_seen_timestamp": 1234567000, + "changed": 1234567800, + "cell_id": 123456789, + "expire_timestamp_verified": true, + "display_pokemon_id": 1, + "is_ditto": false, + "seen_type": "encounter", + "shiny": false, + "username": "trainer_name", + "capture_1": 0.5, + "capture_2": 0.6, + "capture_3": 0.7, + "pvp": {}, + "is_event": 0 +} +``` + +### ApiPokestopResult + +```json +{ + "id": "fort_id", + "lat": 40.7128, + "lon": -74.0060, + "name": "Pokestop Name", + "url": "image_url", + "lure_expire_timestamp": 1234567890, + "last_modified_timestamp": 1234567800, + "updated": 1234567800, + "enabled": true, + "quest_type": 1, + "quest_timestamp": 1234567800, + "quest_target": 3, + "quest_conditions": "json_conditions", + "quest_rewards": "json_rewards", + "quest_template": "template_string", + "quest_title": "Quest Title", + "quest_expiry": 1234667800, + "cell_id": 123456789, + "deleted": false, + "lure_id": 501, + "first_seen_timestamp": 1234567000, + "sponsor_id": 1, + "partner_id": "partner_code", + "ar_scan_eligible": 1, + "power_up_level": 1, + "power_up_points": 100, + "power_up_end_timestamp": 1234567890, + "alternative_quest_type": null, + "alternative_quest_timestamp": null, + "alternative_quest_target": null, + "alternative_quest_conditions": null, + "alternative_quest_rewards": null, + "alternative_quest_template": null, + "alternative_quest_title": null, + "alternative_quest_expiry": null, + "description": "Pokestop description", + "showcase_focus": "focus_pokemon", + "showcase_pokemon_id": 1, + "showcase_pokemon_form_id": 0, + "showcase_pokemon_type_id": 1, + "showcase_ranking_standard": 1, + "showcase_expiry": 1234567890, + "showcase_rankings": "json_rankings" +} +``` + +### ApiGymResult + +```json +{ + "id": "gym_id", + "lat": 40.7128, + "lon": -74.0060, + "name": "Gym Name", + "url": "image_url", + "last_modified_timestamp": 1234567800, + "raid_end_timestamp": 1234567890, + "raid_spawn_timestamp": 1234567800, + "raid_battle_timestamp": 1234567850, + "updated": 1234567800, + "raid_pokemon_id": 1, + "guarding_pokemon_id": 25, + "guarding_pokemon_display": "display_string", + "available_slots": 3, + "team_id": 1, + "raid_level": 3, + "enabled": 1, + "ex_raid_eligible": 1, + "in_battle": 0, + "raid_pokemon_move_1": 100, + "raid_pokemon_move_2": 200, + "raid_pokemon_form": 0, + "raid_pokemon_alignment": 1, + "raid_pokemon_cp": 30000, + "raid_is_exclusive": 0, + "cell_id": 123456789, + "deleted": false, + "total_cp": 150000, + "first_seen_timestamp": 1234567000, + "raid_pokemon_gender": 1, + "sponsor_id": 1, + "partner_id": "partner_code", + "raid_pokemon_costume": 0, + "raid_pokemon_evolution": 0, + "ar_scan_eligible": 1, + "power_up_level": 1, + "power_up_points": 100, + "power_up_end_timestamp": 1234567890, + "description": "Gym description", + "defenders": "json_defenders", + "rsvps": "json_rsvps" +} +``` + +### ApiTappableResult + +```json +{ + "id": 1234567890, + "lat": 40.7128, + "lon": -74.0060, + "fort_id": "gym_or_pokestop_id", + "spawn_id": 987654321, + "type": "invasion", + "pokemon_id": 1, + "item_id": 1, + "count": 1, + "expire_timestamp": 1234567890, + "expire_timestamp_verified": true, + "updated": 1234567800 +} +``` + +--- + +## Configuration Reference + +| Key | Description | +|-----|-------------| +| `api_secret` | API authentication token (header: `X-Golbat-Secret`) | +| `raw_bearer` | Bearer token for raw endpoint (header: `Authorization: Bearer`) | +| `port` | HTTP server port | +| `grpc_port` | gRPC server port | +| `tuning.extended_timeout` | Enable 30s timeout for raw processing | +| `tuning.profile_routes` | Enable pprof debug endpoints | +| `tuning.max_pokemon_results` | Max pokemon returned per query | +| `tuning.max_pokemon_distance` | Max distance between min/max points in searches | From c02ade86e81b8025724e44f05ca90d9074f86b6e Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 2 Feb 2026 08:21:41 +0000 Subject: [PATCH 34/78] Change null debug output so it is more readable --- decoder/gym.go | 70 +++++++++--------- decoder/incident.go | 12 ++-- decoder/main.go | 15 ++++ decoder/player.go | 164 +++++++++++++++++++++--------------------- decoder/pokemon.go | 48 ++++++------- decoder/pokestop.go | 72 +++++++++---------- decoder/routes.go | 2 +- decoder/spawnpoint.go | 4 +- decoder/station.go | 32 ++++----- decoder/tappable.go | 12 ++-- 10 files changed, 223 insertions(+), 208 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index d906a0c0..902cb15b 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -210,7 +210,7 @@ func (gym *Gym) SetLon(v float64) { func (gym *Gym) SetName(v null.String) { if gym.Name != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Name:%v->%v", gym.Name, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Name:%s->%s", FormatNull(gym.Name), FormatNull(v))) } gym.Name = v gym.dirty = true @@ -220,7 +220,7 @@ func (gym *Gym) SetName(v null.String) { func (gym *Gym) SetUrl(v null.String) { if gym.Url != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Url:%v->%v", gym.Url, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Url:%s->%s", FormatNull(gym.Url), FormatNull(v))) } gym.Url = v gym.dirty = true @@ -230,7 +230,7 @@ func (gym *Gym) SetUrl(v null.String) { func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { if gym.LastModifiedTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("LastModifiedTimestamp:%v->%v", gym.LastModifiedTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("LastModifiedTimestamp:%s->%s", FormatNull(gym.LastModifiedTimestamp), FormatNull(v))) } gym.LastModifiedTimestamp = v gym.dirty = true @@ -240,7 +240,7 @@ func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { func (gym *Gym) SetRaidEndTimestamp(v null.Int) { if gym.RaidEndTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidEndTimestamp:%v->%v", gym.RaidEndTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidEndTimestamp:%s->%s", FormatNull(gym.RaidEndTimestamp), FormatNull(v))) } gym.RaidEndTimestamp = v gym.dirty = true @@ -250,7 +250,7 @@ func (gym *Gym) SetRaidEndTimestamp(v null.Int) { func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { if gym.RaidSpawnTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidSpawnTimestamp:%v->%v", gym.RaidSpawnTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidSpawnTimestamp:%s->%s", FormatNull(gym.RaidSpawnTimestamp), FormatNull(v))) } gym.RaidSpawnTimestamp = v gym.dirty = true @@ -260,7 +260,7 @@ func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { if gym.RaidBattleTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidBattleTimestamp:%v->%v", gym.RaidBattleTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidBattleTimestamp:%s->%s", FormatNull(gym.RaidBattleTimestamp), FormatNull(v))) } gym.RaidBattleTimestamp = v gym.dirty = true @@ -270,7 +270,7 @@ func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { func (gym *Gym) SetRaidPokemonId(v null.Int) { if gym.RaidPokemonId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonId:%v->%v", gym.RaidPokemonId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonId:%s->%s", FormatNull(gym.RaidPokemonId), FormatNull(v))) } gym.RaidPokemonId = v gym.dirty = true @@ -280,7 +280,7 @@ func (gym *Gym) SetRaidPokemonId(v null.Int) { func (gym *Gym) SetGuardingPokemonId(v null.Int) { if gym.GuardingPokemonId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonId:%v->%v", gym.GuardingPokemonId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonId:%s->%s", FormatNull(gym.GuardingPokemonId), FormatNull(v))) } gym.GuardingPokemonId = v gym.dirty = true @@ -290,7 +290,7 @@ func (gym *Gym) SetGuardingPokemonId(v null.Int) { func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { if gym.GuardingPokemonDisplay != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonDisplay:%v->%v", gym.GuardingPokemonDisplay, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonDisplay:%s->%s", FormatNull(gym.GuardingPokemonDisplay), FormatNull(v))) } gym.GuardingPokemonDisplay = v gym.dirty = true @@ -300,7 +300,7 @@ func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { func (gym *Gym) SetAvailableSlots(v null.Int) { if gym.AvailableSlots != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("AvailableSlots:%v->%v", gym.AvailableSlots, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("AvailableSlots:%s->%s", FormatNull(gym.AvailableSlots), FormatNull(v))) } gym.AvailableSlots = v gym.dirty = true @@ -310,7 +310,7 @@ func (gym *Gym) SetAvailableSlots(v null.Int) { func (gym *Gym) SetTeamId(v null.Int) { if gym.TeamId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("TeamId:%v->%v", gym.TeamId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("TeamId:%s->%s", FormatNull(gym.TeamId), FormatNull(v))) } gym.TeamId = v gym.dirty = true @@ -320,7 +320,7 @@ func (gym *Gym) SetTeamId(v null.Int) { func (gym *Gym) SetRaidLevel(v null.Int) { if gym.RaidLevel != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidLevel:%v->%v", gym.RaidLevel, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidLevel:%s->%s", FormatNull(gym.RaidLevel), FormatNull(v))) } gym.RaidLevel = v gym.dirty = true @@ -330,7 +330,7 @@ func (gym *Gym) SetRaidLevel(v null.Int) { func (gym *Gym) SetEnabled(v null.Int) { if gym.Enabled != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Enabled:%v->%v", gym.Enabled, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Enabled:%s->%s", FormatNull(gym.Enabled), FormatNull(v))) } gym.Enabled = v gym.dirty = true @@ -340,7 +340,7 @@ func (gym *Gym) SetEnabled(v null.Int) { func (gym *Gym) SetExRaidEligible(v null.Int) { if gym.ExRaidEligible != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("ExRaidEligible:%v->%v", gym.ExRaidEligible, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("ExRaidEligible:%s->%s", FormatNull(gym.ExRaidEligible), FormatNull(v))) } gym.ExRaidEligible = v gym.dirty = true @@ -350,7 +350,7 @@ func (gym *Gym) SetExRaidEligible(v null.Int) { func (gym *Gym) SetInBattle(v null.Int) { if gym.InBattle != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("InBattle:%v->%v", gym.InBattle, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("InBattle:%s->%s", FormatNull(gym.InBattle), FormatNull(v))) } gym.InBattle = v //Do not set to dirty, as don't trigger an update @@ -361,7 +361,7 @@ func (gym *Gym) SetInBattle(v null.Int) { func (gym *Gym) SetRaidPokemonMove1(v null.Int) { if gym.RaidPokemonMove1 != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove1:%v->%v", gym.RaidPokemonMove1, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove1:%s->%s", FormatNull(gym.RaidPokemonMove1), FormatNull(v))) } gym.RaidPokemonMove1 = v gym.dirty = true @@ -371,7 +371,7 @@ func (gym *Gym) SetRaidPokemonMove1(v null.Int) { func (gym *Gym) SetRaidPokemonMove2(v null.Int) { if gym.RaidPokemonMove2 != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove2:%v->%v", gym.RaidPokemonMove2, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove2:%s->%s", FormatNull(gym.RaidPokemonMove2), FormatNull(v))) } gym.RaidPokemonMove2 = v gym.dirty = true @@ -381,7 +381,7 @@ func (gym *Gym) SetRaidPokemonMove2(v null.Int) { func (gym *Gym) SetRaidPokemonForm(v null.Int) { if gym.RaidPokemonForm != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonForm:%v->%v", gym.RaidPokemonForm, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonForm:%s->%s", FormatNull(gym.RaidPokemonForm), FormatNull(v))) } gym.RaidPokemonForm = v gym.dirty = true @@ -391,7 +391,7 @@ func (gym *Gym) SetRaidPokemonForm(v null.Int) { func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { if gym.RaidPokemonAlignment != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonAlignment:%v->%v", gym.RaidPokemonAlignment, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonAlignment:%s->%s", FormatNull(gym.RaidPokemonAlignment), FormatNull(v))) } gym.RaidPokemonAlignment = v gym.dirty = true @@ -401,7 +401,7 @@ func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { func (gym *Gym) SetRaidPokemonCp(v null.Int) { if gym.RaidPokemonCp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCp:%v->%v", gym.RaidPokemonCp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCp:%s->%s", FormatNull(gym.RaidPokemonCp), FormatNull(v))) } gym.RaidPokemonCp = v gym.dirty = true @@ -411,7 +411,7 @@ func (gym *Gym) SetRaidPokemonCp(v null.Int) { func (gym *Gym) SetRaidIsExclusive(v null.Int) { if gym.RaidIsExclusive != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidIsExclusive:%v->%v", gym.RaidIsExclusive, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidIsExclusive:%s->%s", FormatNull(gym.RaidIsExclusive), FormatNull(v))) } gym.RaidIsExclusive = v gym.dirty = true @@ -421,7 +421,7 @@ func (gym *Gym) SetRaidIsExclusive(v null.Int) { func (gym *Gym) SetCellId(v null.Int) { if gym.CellId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("CellId:%v->%v", gym.CellId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("CellId:%s->%s", FormatNull(gym.CellId), FormatNull(v))) } gym.CellId = v gym.dirty = true @@ -441,7 +441,7 @@ func (gym *Gym) SetDeleted(v bool) { func (gym *Gym) SetTotalCp(v null.Int) { if gym.TotalCp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("TotalCp:%v->%v", gym.TotalCp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("TotalCp:%s->%s", FormatNull(gym.TotalCp), FormatNull(v))) } gym.TotalCp = v gym.dirty = true @@ -451,7 +451,7 @@ func (gym *Gym) SetTotalCp(v null.Int) { func (gym *Gym) SetRaidPokemonGender(v null.Int) { if gym.RaidPokemonGender != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonGender:%v->%v", gym.RaidPokemonGender, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonGender:%s->%s", FormatNull(gym.RaidPokemonGender), FormatNull(v))) } gym.RaidPokemonGender = v gym.dirty = true @@ -461,7 +461,7 @@ func (gym *Gym) SetRaidPokemonGender(v null.Int) { func (gym *Gym) SetSponsorId(v null.Int) { if gym.SponsorId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("SponsorId:%v->%v", gym.SponsorId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("SponsorId:%s->%s", FormatNull(gym.SponsorId), FormatNull(v))) } gym.SponsorId = v gym.dirty = true @@ -471,7 +471,7 @@ func (gym *Gym) SetSponsorId(v null.Int) { func (gym *Gym) SetPartnerId(v null.String) { if gym.PartnerId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("PartnerId:%v->%v", gym.PartnerId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PartnerId:%s->%s", FormatNull(gym.PartnerId), FormatNull(v))) } gym.PartnerId = v gym.dirty = true @@ -481,7 +481,7 @@ func (gym *Gym) SetPartnerId(v null.String) { func (gym *Gym) SetRaidPokemonCostume(v null.Int) { if gym.RaidPokemonCostume != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCostume:%v->%v", gym.RaidPokemonCostume, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCostume:%s->%s", FormatNull(gym.RaidPokemonCostume), FormatNull(v))) } gym.RaidPokemonCostume = v gym.dirty = true @@ -491,7 +491,7 @@ func (gym *Gym) SetRaidPokemonCostume(v null.Int) { func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { if gym.RaidPokemonEvolution != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonEvolution:%v->%v", gym.RaidPokemonEvolution, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonEvolution:%s->%s", FormatNull(gym.RaidPokemonEvolution), FormatNull(v))) } gym.RaidPokemonEvolution = v gym.dirty = true @@ -501,7 +501,7 @@ func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { func (gym *Gym) SetArScanEligible(v null.Int) { if gym.ArScanEligible != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("ArScanEligible:%v->%v", gym.ArScanEligible, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("ArScanEligible:%s->%s", FormatNull(gym.ArScanEligible), FormatNull(v))) } gym.ArScanEligible = v gym.dirty = true @@ -511,7 +511,7 @@ func (gym *Gym) SetArScanEligible(v null.Int) { func (gym *Gym) SetPowerUpLevel(v null.Int) { if gym.PowerUpLevel != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpLevel:%v->%v", gym.PowerUpLevel, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpLevel:%s->%s", FormatNull(gym.PowerUpLevel), FormatNull(v))) } gym.PowerUpLevel = v gym.dirty = true @@ -521,7 +521,7 @@ func (gym *Gym) SetPowerUpLevel(v null.Int) { func (gym *Gym) SetPowerUpPoints(v null.Int) { if gym.PowerUpPoints != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpPoints:%v->%v", gym.PowerUpPoints, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpPoints:%s->%s", FormatNull(gym.PowerUpPoints), FormatNull(v))) } gym.PowerUpPoints = v gym.dirty = true @@ -531,7 +531,7 @@ func (gym *Gym) SetPowerUpPoints(v null.Int) { func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { if gym.PowerUpEndTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%v->%v", gym.PowerUpEndTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%s->%s", FormatNull(gym.PowerUpEndTimestamp), FormatNull(v))) } gym.PowerUpEndTimestamp = v gym.dirty = true @@ -541,7 +541,7 @@ func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { func (gym *Gym) SetDescription(v null.String) { if gym.Description != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Description:%v->%v", gym.Description, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Description:%s->%s", FormatNull(gym.Description), FormatNull(v))) } gym.Description = v gym.dirty = true @@ -551,7 +551,7 @@ func (gym *Gym) SetDescription(v null.String) { func (gym *Gym) SetDefenders(v null.String) { if gym.Defenders != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Defenders:%v->%v", gym.Defenders, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Defenders:%s->%s", FormatNull(gym.Defenders), FormatNull(v))) } gym.Defenders = v //Do not set to dirty, as don't trigger an update @@ -562,7 +562,7 @@ func (gym *Gym) SetDefenders(v null.String) { func (gym *Gym) SetRsvps(v null.String) { if gym.Rsvps != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Rsvps:%v->%v", gym.Rsvps, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Rsvps:%s->%s", FormatNull(gym.Rsvps), FormatNull(v))) } gym.Rsvps = v gym.dirty = true diff --git a/decoder/incident.go b/decoder/incident.go index dd85d474..c30e135d 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -201,7 +201,7 @@ func (incident *Incident) SetConfirmed(v bool) { func (incident *Incident) SetSlot1PokemonId(v null.Int) { if incident.Slot1PokemonId != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1PokemonId:%v->%v", incident.Slot1PokemonId, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1PokemonId:%s->%s", FormatNull(incident.Slot1PokemonId), FormatNull(v))) } incident.Slot1PokemonId = v incident.dirty = true @@ -211,7 +211,7 @@ func (incident *Incident) SetSlot1PokemonId(v null.Int) { func (incident *Incident) SetSlot1Form(v null.Int) { if incident.Slot1Form != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1Form:%v->%v", incident.Slot1Form, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1Form:%s->%s", FormatNull(incident.Slot1Form), FormatNull(v))) } incident.Slot1Form = v incident.dirty = true @@ -221,7 +221,7 @@ func (incident *Incident) SetSlot1Form(v null.Int) { func (incident *Incident) SetSlot2PokemonId(v null.Int) { if incident.Slot2PokemonId != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2PokemonId:%v->%v", incident.Slot2PokemonId, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2PokemonId:%s->%s", FormatNull(incident.Slot2PokemonId), FormatNull(v))) } incident.Slot2PokemonId = v incident.dirty = true @@ -231,7 +231,7 @@ func (incident *Incident) SetSlot2PokemonId(v null.Int) { func (incident *Incident) SetSlot2Form(v null.Int) { if incident.Slot2Form != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2Form:%v->%v", incident.Slot2Form, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2Form:%s->%s", FormatNull(incident.Slot2Form), FormatNull(v))) } incident.Slot2Form = v incident.dirty = true @@ -241,7 +241,7 @@ func (incident *Incident) SetSlot2Form(v null.Int) { func (incident *Incident) SetSlot3PokemonId(v null.Int) { if incident.Slot3PokemonId != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3PokemonId:%v->%v", incident.Slot3PokemonId, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3PokemonId:%s->%s", FormatNull(incident.Slot3PokemonId), FormatNull(v))) } incident.Slot3PokemonId = v incident.dirty = true @@ -251,7 +251,7 @@ func (incident *Incident) SetSlot3PokemonId(v null.Int) { func (incident *Incident) SetSlot3Form(v null.Int) { if incident.Slot3Form != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3Form:%v->%v", incident.Slot3Form, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3Form:%s->%s", FormatNull(incident.Slot3Form), FormatNull(v))) } incident.Slot3Form = v incident.dirty = true diff --git a/decoder/main.go b/decoder/main.go index b1e2b589..3fcb043a 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "math" "runtime" "time" @@ -246,6 +247,20 @@ func nullFloatAlmostEqual(a, b null.Float, tolerance float64) bool { } } +// Ptrable is an interface for any type that has a Ptr() method returning *T +// specifically these are the null objects +type Ptrable[T any] interface { + Ptr() *T +} + +// FormatNull returns "NULL" if the nullable value is not valid, otherwise formats the value +func FormatNull[T any](n Ptrable[T]) string { + if ptr := n.Ptr(); ptr != nil { + return fmt.Sprintf("%v", *ptr) + } + return "NULL" +} + func SetWebhooksSender(whSender webhooksSenderInterface) { webhooksSender = whSender } diff --git a/decoder/player.go b/decoder/player.go index 433885bb..0bd20785 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -134,7 +134,7 @@ func (p *Player) setFieldDirty() { func (p *Player) SetFriendshipId(v null.String) { if p.FriendshipId != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("FriendshipId:%v->%v", p.FriendshipId, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("FriendshipId:%s->%s", FormatNull(p.FriendshipId), FormatNull(v))) } p.FriendshipId = v p.dirty = true @@ -144,7 +144,7 @@ func (p *Player) SetFriendshipId(v null.String) { func (p *Player) SetFriendCode(v null.String) { if p.FriendCode != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("FriendCode:%v->%v", p.FriendCode, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("FriendCode:%s->%s", FormatNull(p.FriendCode), FormatNull(v))) } p.FriendCode = v p.dirty = true @@ -154,7 +154,7 @@ func (p *Player) SetFriendCode(v null.String) { func (p *Player) SetTeam(v null.Int) { if p.Team != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Team:%v->%v", p.Team, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Team:%s->%s", FormatNull(p.Team), FormatNull(v))) } p.Team = v p.dirty = true @@ -164,7 +164,7 @@ func (p *Player) SetTeam(v null.Int) { func (p *Player) SetLevel(v null.Int) { if p.Level != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Level:%v->%v", p.Level, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Level:%s->%s", FormatNull(p.Level), FormatNull(v))) } p.Level = v p.dirty = true @@ -174,7 +174,7 @@ func (p *Player) SetLevel(v null.Int) { func (p *Player) SetXp(v null.Int) { if p.Xp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Xp:%v->%v", p.Xp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Xp:%s->%s", FormatNull(p.Xp), FormatNull(v))) } p.Xp = v p.dirty = true @@ -184,7 +184,7 @@ func (p *Player) SetXp(v null.Int) { func (p *Player) SetBattlesWon(v null.Int) { if p.BattlesWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("BattlesWon:%v->%v", p.BattlesWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("BattlesWon:%s->%s", FormatNull(p.BattlesWon), FormatNull(v))) } p.BattlesWon = v p.dirty = true @@ -194,7 +194,7 @@ func (p *Player) SetBattlesWon(v null.Int) { func (p *Player) SetKmWalked(v null.Float) { if !nullFloatAlmostEqual(p.KmWalked, v, 0.001) { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("KmWalked:%v->%v", p.KmWalked, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("KmWalked:%s->%s", FormatNull(p.KmWalked), FormatNull(v))) } p.KmWalked = v p.dirty = true @@ -204,7 +204,7 @@ func (p *Player) SetKmWalked(v null.Float) { func (p *Player) SetCaughtPokemon(v null.Int) { if p.CaughtPokemon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPokemon:%v->%v", p.CaughtPokemon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPokemon:%s->%s", FormatNull(p.CaughtPokemon), FormatNull(v))) } p.CaughtPokemon = v p.dirty = true @@ -214,7 +214,7 @@ func (p *Player) SetCaughtPokemon(v null.Int) { func (p *Player) SetGblRank(v null.Int) { if p.GblRank != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GblRank:%v->%v", p.GblRank, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GblRank:%s->%s", FormatNull(p.GblRank), FormatNull(v))) } p.GblRank = v p.dirty = true @@ -224,7 +224,7 @@ func (p *Player) SetGblRank(v null.Int) { func (p *Player) SetGblRating(v null.Int) { if p.GblRating != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GblRating:%v->%v", p.GblRating, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GblRating:%s->%s", FormatNull(p.GblRating), FormatNull(v))) } p.GblRating = v p.dirty = true @@ -234,7 +234,7 @@ func (p *Player) SetGblRating(v null.Int) { func (p *Player) SetEventBadges(v null.String) { if p.EventBadges != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("EventBadges:%v->%v", p.EventBadges, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("EventBadges:%s->%s", FormatNull(p.EventBadges), FormatNull(v))) } p.EventBadges = v p.dirty = true @@ -244,7 +244,7 @@ func (p *Player) SetEventBadges(v null.String) { func (p *Player) SetStopsSpun(v null.Int) { if p.StopsSpun != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("StopsSpun:%v->%v", p.StopsSpun, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("StopsSpun:%s->%s", FormatNull(p.StopsSpun), FormatNull(v))) } p.StopsSpun = v p.dirty = true @@ -254,7 +254,7 @@ func (p *Player) SetStopsSpun(v null.Int) { func (p *Player) SetEvolved(v null.Int) { if p.Evolved != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Evolved:%v->%v", p.Evolved, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Evolved:%s->%s", FormatNull(p.Evolved), FormatNull(v))) } p.Evolved = v p.dirty = true @@ -264,7 +264,7 @@ func (p *Player) SetEvolved(v null.Int) { func (p *Player) SetHatched(v null.Int) { if p.Hatched != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Hatched:%v->%v", p.Hatched, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Hatched:%s->%s", FormatNull(p.Hatched), FormatNull(v))) } p.Hatched = v p.dirty = true @@ -274,7 +274,7 @@ func (p *Player) SetHatched(v null.Int) { func (p *Player) SetQuests(v null.Int) { if p.Quests != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Quests:%v->%v", p.Quests, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Quests:%s->%s", FormatNull(p.Quests), FormatNull(v))) } p.Quests = v p.dirty = true @@ -284,7 +284,7 @@ func (p *Player) SetQuests(v null.Int) { func (p *Player) SetTrades(v null.Int) { if p.Trades != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Trades:%v->%v", p.Trades, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Trades:%s->%s", FormatNull(p.Trades), FormatNull(v))) } p.Trades = v p.dirty = true @@ -294,7 +294,7 @@ func (p *Player) SetTrades(v null.Int) { func (p *Player) SetPhotobombs(v null.Int) { if p.Photobombs != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Photobombs:%v->%v", p.Photobombs, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Photobombs:%s->%s", FormatNull(p.Photobombs), FormatNull(v))) } p.Photobombs = v p.dirty = true @@ -304,7 +304,7 @@ func (p *Player) SetPhotobombs(v null.Int) { func (p *Player) SetPurified(v null.Int) { if p.Purified != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Purified:%v->%v", p.Purified, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Purified:%s->%s", FormatNull(p.Purified), FormatNull(v))) } p.Purified = v p.dirty = true @@ -314,7 +314,7 @@ func (p *Player) SetPurified(v null.Int) { func (p *Player) SetGruntsDefeated(v null.Int) { if p.GruntsDefeated != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GruntsDefeated:%v->%v", p.GruntsDefeated, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GruntsDefeated:%s->%s", FormatNull(p.GruntsDefeated), FormatNull(v))) } p.GruntsDefeated = v p.dirty = true @@ -324,7 +324,7 @@ func (p *Player) SetGruntsDefeated(v null.Int) { func (p *Player) SetGymBattlesWon(v null.Int) { if p.GymBattlesWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GymBattlesWon:%v->%v", p.GymBattlesWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GymBattlesWon:%s->%s", FormatNull(p.GymBattlesWon), FormatNull(v))) } p.GymBattlesWon = v p.dirty = true @@ -334,7 +334,7 @@ func (p *Player) SetGymBattlesWon(v null.Int) { func (p *Player) SetNormalRaidsWon(v null.Int) { if p.NormalRaidsWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("NormalRaidsWon:%v->%v", p.NormalRaidsWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("NormalRaidsWon:%s->%s", FormatNull(p.NormalRaidsWon), FormatNull(v))) } p.NormalRaidsWon = v p.dirty = true @@ -344,7 +344,7 @@ func (p *Player) SetNormalRaidsWon(v null.Int) { func (p *Player) SetLegendaryRaidsWon(v null.Int) { if p.LegendaryRaidsWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LegendaryRaidsWon:%v->%v", p.LegendaryRaidsWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LegendaryRaidsWon:%s->%s", FormatNull(p.LegendaryRaidsWon), FormatNull(v))) } p.LegendaryRaidsWon = v p.dirty = true @@ -354,7 +354,7 @@ func (p *Player) SetLegendaryRaidsWon(v null.Int) { func (p *Player) SetTrainingsWon(v null.Int) { if p.TrainingsWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TrainingsWon:%v->%v", p.TrainingsWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TrainingsWon:%s->%s", FormatNull(p.TrainingsWon), FormatNull(v))) } p.TrainingsWon = v p.dirty = true @@ -364,7 +364,7 @@ func (p *Player) SetTrainingsWon(v null.Int) { func (p *Player) SetBerriesFed(v null.Int) { if p.BerriesFed != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("BerriesFed:%v->%v", p.BerriesFed, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("BerriesFed:%s->%s", FormatNull(p.BerriesFed), FormatNull(v))) } p.BerriesFed = v p.dirty = true @@ -374,7 +374,7 @@ func (p *Player) SetBerriesFed(v null.Int) { func (p *Player) SetHoursDefended(v null.Int) { if p.HoursDefended != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("HoursDefended:%v->%v", p.HoursDefended, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("HoursDefended:%s->%s", FormatNull(p.HoursDefended), FormatNull(v))) } p.HoursDefended = v p.dirty = true @@ -384,7 +384,7 @@ func (p *Player) SetHoursDefended(v null.Int) { func (p *Player) SetBestFriends(v null.Int) { if p.BestFriends != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("BestFriends:%v->%v", p.BestFriends, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("BestFriends:%s->%s", FormatNull(p.BestFriends), FormatNull(v))) } p.BestFriends = v p.dirty = true @@ -394,7 +394,7 @@ func (p *Player) SetBestFriends(v null.Int) { func (p *Player) SetBestBuddies(v null.Int) { if p.BestBuddies != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("BestBuddies:%v->%v", p.BestBuddies, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("BestBuddies:%s->%s", FormatNull(p.BestBuddies), FormatNull(v))) } p.BestBuddies = v p.dirty = true @@ -404,7 +404,7 @@ func (p *Player) SetBestBuddies(v null.Int) { func (p *Player) SetGiovanniDefeated(v null.Int) { if p.GiovanniDefeated != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GiovanniDefeated:%v->%v", p.GiovanniDefeated, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GiovanniDefeated:%s->%s", FormatNull(p.GiovanniDefeated), FormatNull(v))) } p.GiovanniDefeated = v p.dirty = true @@ -414,7 +414,7 @@ func (p *Player) SetGiovanniDefeated(v null.Int) { func (p *Player) SetMegaEvos(v null.Int) { if p.MegaEvos != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("MegaEvos:%v->%v", p.MegaEvos, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("MegaEvos:%s->%s", FormatNull(p.MegaEvos), FormatNull(v))) } p.MegaEvos = v p.dirty = true @@ -424,7 +424,7 @@ func (p *Player) SetMegaEvos(v null.Int) { func (p *Player) SetCollectionsDone(v null.Int) { if p.CollectionsDone != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CollectionsDone:%v->%v", p.CollectionsDone, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CollectionsDone:%s->%s", FormatNull(p.CollectionsDone), FormatNull(v))) } p.CollectionsDone = v p.dirty = true @@ -434,7 +434,7 @@ func (p *Player) SetCollectionsDone(v null.Int) { func (p *Player) SetUniqueStopsSpun(v null.Int) { if p.UniqueStopsSpun != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueStopsSpun:%v->%v", p.UniqueStopsSpun, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueStopsSpun:%s->%s", FormatNull(p.UniqueStopsSpun), FormatNull(v))) } p.UniqueStopsSpun = v p.dirty = true @@ -444,7 +444,7 @@ func (p *Player) SetUniqueStopsSpun(v null.Int) { func (p *Player) SetUniqueMegaEvos(v null.Int) { if p.UniqueMegaEvos != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueMegaEvos:%v->%v", p.UniqueMegaEvos, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueMegaEvos:%s->%s", FormatNull(p.UniqueMegaEvos), FormatNull(v))) } p.UniqueMegaEvos = v p.dirty = true @@ -454,7 +454,7 @@ func (p *Player) SetUniqueMegaEvos(v null.Int) { func (p *Player) SetUniqueRaidBosses(v null.Int) { if p.UniqueRaidBosses != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueRaidBosses:%v->%v", p.UniqueRaidBosses, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueRaidBosses:%s->%s", FormatNull(p.UniqueRaidBosses), FormatNull(v))) } p.UniqueRaidBosses = v p.dirty = true @@ -464,7 +464,7 @@ func (p *Player) SetUniqueRaidBosses(v null.Int) { func (p *Player) SetUniqueUnown(v null.Int) { if p.UniqueUnown != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueUnown:%v->%v", p.UniqueUnown, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueUnown:%s->%s", FormatNull(p.UniqueUnown), FormatNull(v))) } p.UniqueUnown = v p.dirty = true @@ -474,7 +474,7 @@ func (p *Player) SetUniqueUnown(v null.Int) { func (p *Player) SetSevenDayStreaks(v null.Int) { if p.SevenDayStreaks != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("SevenDayStreaks:%v->%v", p.SevenDayStreaks, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("SevenDayStreaks:%s->%s", FormatNull(p.SevenDayStreaks), FormatNull(v))) } p.SevenDayStreaks = v p.dirty = true @@ -484,7 +484,7 @@ func (p *Player) SetSevenDayStreaks(v null.Int) { func (p *Player) SetTradeKm(v null.Int) { if p.TradeKm != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TradeKm:%v->%v", p.TradeKm, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TradeKm:%s->%s", FormatNull(p.TradeKm), FormatNull(v))) } p.TradeKm = v p.dirty = true @@ -494,7 +494,7 @@ func (p *Player) SetTradeKm(v null.Int) { func (p *Player) SetRaidsWithFriends(v null.Int) { if p.RaidsWithFriends != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("RaidsWithFriends:%v->%v", p.RaidsWithFriends, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("RaidsWithFriends:%s->%s", FormatNull(p.RaidsWithFriends), FormatNull(v))) } p.RaidsWithFriends = v p.dirty = true @@ -504,7 +504,7 @@ func (p *Player) SetRaidsWithFriends(v null.Int) { func (p *Player) SetCaughtAtLure(v null.Int) { if p.CaughtAtLure != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtAtLure:%v->%v", p.CaughtAtLure, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtAtLure:%s->%s", FormatNull(p.CaughtAtLure), FormatNull(v))) } p.CaughtAtLure = v p.dirty = true @@ -514,7 +514,7 @@ func (p *Player) SetCaughtAtLure(v null.Int) { func (p *Player) SetWayfarerAgreements(v null.Int) { if p.WayfarerAgreements != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("WayfarerAgreements:%v->%v", p.WayfarerAgreements, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("WayfarerAgreements:%s->%s", FormatNull(p.WayfarerAgreements), FormatNull(v))) } p.WayfarerAgreements = v p.dirty = true @@ -524,7 +524,7 @@ func (p *Player) SetWayfarerAgreements(v null.Int) { func (p *Player) SetTrainersReferred(v null.Int) { if p.TrainersReferred != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TrainersReferred:%v->%v", p.TrainersReferred, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TrainersReferred:%s->%s", FormatNull(p.TrainersReferred), FormatNull(v))) } p.TrainersReferred = v p.dirty = true @@ -534,7 +534,7 @@ func (p *Player) SetTrainersReferred(v null.Int) { func (p *Player) SetRaidAchievements(v null.Int) { if p.RaidAchievements != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("RaidAchievements:%v->%v", p.RaidAchievements, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("RaidAchievements:%s->%s", FormatNull(p.RaidAchievements), FormatNull(v))) } p.RaidAchievements = v p.dirty = true @@ -544,7 +544,7 @@ func (p *Player) SetRaidAchievements(v null.Int) { func (p *Player) SetXlKarps(v null.Int) { if p.XlKarps != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("XlKarps:%v->%v", p.XlKarps, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("XlKarps:%s->%s", FormatNull(p.XlKarps), FormatNull(v))) } p.XlKarps = v p.dirty = true @@ -554,7 +554,7 @@ func (p *Player) SetXlKarps(v null.Int) { func (p *Player) SetXsRats(v null.Int) { if p.XsRats != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("XsRats:%v->%v", p.XsRats, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("XsRats:%s->%s", FormatNull(p.XsRats), FormatNull(v))) } p.XsRats = v p.dirty = true @@ -564,7 +564,7 @@ func (p *Player) SetXsRats(v null.Int) { func (p *Player) SetPikachuCaught(v null.Int) { if p.PikachuCaught != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PikachuCaught:%v->%v", p.PikachuCaught, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PikachuCaught:%s->%s", FormatNull(p.PikachuCaught), FormatNull(v))) } p.PikachuCaught = v p.dirty = true @@ -574,7 +574,7 @@ func (p *Player) SetPikachuCaught(v null.Int) { func (p *Player) SetLeagueGreatWon(v null.Int) { if p.LeagueGreatWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueGreatWon:%v->%v", p.LeagueGreatWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueGreatWon:%s->%s", FormatNull(p.LeagueGreatWon), FormatNull(v))) } p.LeagueGreatWon = v p.dirty = true @@ -584,7 +584,7 @@ func (p *Player) SetLeagueGreatWon(v null.Int) { func (p *Player) SetLeagueUltraWon(v null.Int) { if p.LeagueUltraWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueUltraWon:%v->%v", p.LeagueUltraWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueUltraWon:%s->%s", FormatNull(p.LeagueUltraWon), FormatNull(v))) } p.LeagueUltraWon = v p.dirty = true @@ -594,7 +594,7 @@ func (p *Player) SetLeagueUltraWon(v null.Int) { func (p *Player) SetLeagueMasterWon(v null.Int) { if p.LeagueMasterWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueMasterWon:%v->%v", p.LeagueMasterWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueMasterWon:%s->%s", FormatNull(p.LeagueMasterWon), FormatNull(v))) } p.LeagueMasterWon = v p.dirty = true @@ -604,7 +604,7 @@ func (p *Player) SetLeagueMasterWon(v null.Int) { func (p *Player) SetTinyPokemonCaught(v null.Int) { if p.TinyPokemonCaught != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TinyPokemonCaught:%v->%v", p.TinyPokemonCaught, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TinyPokemonCaught:%s->%s", FormatNull(p.TinyPokemonCaught), FormatNull(v))) } p.TinyPokemonCaught = v p.dirty = true @@ -614,7 +614,7 @@ func (p *Player) SetTinyPokemonCaught(v null.Int) { func (p *Player) SetJumboPokemonCaught(v null.Int) { if p.JumboPokemonCaught != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("JumboPokemonCaught:%v->%v", p.JumboPokemonCaught, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("JumboPokemonCaught:%s->%s", FormatNull(p.JumboPokemonCaught), FormatNull(v))) } p.JumboPokemonCaught = v p.dirty = true @@ -624,7 +624,7 @@ func (p *Player) SetJumboPokemonCaught(v null.Int) { func (p *Player) SetVivillon(v null.Int) { if p.Vivillon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Vivillon:%v->%v", p.Vivillon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Vivillon:%s->%s", FormatNull(p.Vivillon), FormatNull(v))) } p.Vivillon = v p.dirty = true @@ -634,7 +634,7 @@ func (p *Player) SetVivillon(v null.Int) { func (p *Player) SetMaxSizeFirstPlace(v null.Int) { if p.MaxSizeFirstPlace != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("MaxSizeFirstPlace:%v->%v", p.MaxSizeFirstPlace, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("MaxSizeFirstPlace:%s->%s", FormatNull(p.MaxSizeFirstPlace), FormatNull(v))) } p.MaxSizeFirstPlace = v p.dirty = true @@ -644,7 +644,7 @@ func (p *Player) SetMaxSizeFirstPlace(v null.Int) { func (p *Player) SetTotalRoutePlay(v null.Int) { if p.TotalRoutePlay != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TotalRoutePlay:%v->%v", p.TotalRoutePlay, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TotalRoutePlay:%s->%s", FormatNull(p.TotalRoutePlay), FormatNull(v))) } p.TotalRoutePlay = v p.dirty = true @@ -654,7 +654,7 @@ func (p *Player) SetTotalRoutePlay(v null.Int) { func (p *Player) SetPartiesCompleted(v null.Int) { if p.PartiesCompleted != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PartiesCompleted:%v->%v", p.PartiesCompleted, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PartiesCompleted:%s->%s", FormatNull(p.PartiesCompleted), FormatNull(v))) } p.PartiesCompleted = v p.dirty = true @@ -664,7 +664,7 @@ func (p *Player) SetPartiesCompleted(v null.Int) { func (p *Player) SetEventCheckIns(v null.Int) { if p.EventCheckIns != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("EventCheckIns:%v->%v", p.EventCheckIns, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("EventCheckIns:%s->%s", FormatNull(p.EventCheckIns), FormatNull(v))) } p.EventCheckIns = v p.dirty = true @@ -673,7 +673,7 @@ func (p *Player) SetEventCheckIns(v null.Int) { func (p *Player) SetDexGen1(v null.Int) { if p.DexGen1 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen1:%v->%v", p.DexGen1, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen1:%s->%s", FormatNull(p.DexGen1), FormatNull(v))) } p.DexGen1 = v p.dirty = true @@ -683,7 +683,7 @@ func (p *Player) SetDexGen1(v null.Int) { func (p *Player) SetDexGen2(v null.Int) { if p.DexGen2 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen2:%v->%v", p.DexGen2, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen2:%s->%s", FormatNull(p.DexGen2), FormatNull(v))) } p.DexGen2 = v p.dirty = true @@ -693,7 +693,7 @@ func (p *Player) SetDexGen2(v null.Int) { func (p *Player) SetDexGen3(v null.Int) { if p.DexGen3 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen3:%v->%v", p.DexGen3, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen3:%s->%s", FormatNull(p.DexGen3), FormatNull(v))) } p.DexGen3 = v p.dirty = true @@ -703,7 +703,7 @@ func (p *Player) SetDexGen3(v null.Int) { func (p *Player) SetDexGen4(v null.Int) { if p.DexGen4 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen4:%v->%v", p.DexGen4, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen4:%s->%s", FormatNull(p.DexGen4), FormatNull(v))) } p.DexGen4 = v p.dirty = true @@ -713,7 +713,7 @@ func (p *Player) SetDexGen4(v null.Int) { func (p *Player) SetDexGen5(v null.Int) { if p.DexGen5 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen5:%v->%v", p.DexGen5, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen5:%s->%s", FormatNull(p.DexGen5), FormatNull(v))) } p.DexGen5 = v p.dirty = true @@ -723,7 +723,7 @@ func (p *Player) SetDexGen5(v null.Int) { func (p *Player) SetDexGen6(v null.Int) { if p.DexGen6 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen6:%v->%v", p.DexGen6, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen6:%s->%s", FormatNull(p.DexGen6), FormatNull(v))) } p.DexGen6 = v p.dirty = true @@ -733,7 +733,7 @@ func (p *Player) SetDexGen6(v null.Int) { func (p *Player) SetDexGen7(v null.Int) { if p.DexGen7 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen7:%v->%v", p.DexGen7, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen7:%s->%s", FormatNull(p.DexGen7), FormatNull(v))) } p.DexGen7 = v p.dirty = true @@ -743,7 +743,7 @@ func (p *Player) SetDexGen7(v null.Int) { func (p *Player) SetDexGen8(v null.Int) { if p.DexGen8 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8:%v->%v", p.DexGen8, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8:%s->%s", FormatNull(p.DexGen8), FormatNull(v))) } p.DexGen8 = v p.dirty = true @@ -753,7 +753,7 @@ func (p *Player) SetDexGen8(v null.Int) { func (p *Player) SetDexGen8A(v null.Int) { if p.DexGen8A != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8A:%v->%v", p.DexGen8A, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8A:%s->%s", FormatNull(p.DexGen8A), FormatNull(v))) } p.DexGen8A = v p.dirty = true @@ -763,7 +763,7 @@ func (p *Player) SetDexGen8A(v null.Int) { func (p *Player) SetDexGen9(v null.Int) { if p.DexGen9 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen9:%v->%v", p.DexGen9, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen9:%s->%s", FormatNull(p.DexGen9), FormatNull(v))) } p.DexGen9 = v p.dirty = true @@ -773,7 +773,7 @@ func (p *Player) SetDexGen9(v null.Int) { func (p *Player) SetCaughtNormal(v null.Int) { if p.CaughtNormal != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtNormal:%v->%v", p.CaughtNormal, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtNormal:%s->%s", FormatNull(p.CaughtNormal), FormatNull(v))) } p.CaughtNormal = v p.dirty = true @@ -783,7 +783,7 @@ func (p *Player) SetCaughtNormal(v null.Int) { func (p *Player) SetCaughtFighting(v null.Int) { if p.CaughtFighting != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFighting:%v->%v", p.CaughtFighting, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFighting:%s->%s", FormatNull(p.CaughtFighting), FormatNull(v))) } p.CaughtFighting = v p.dirty = true @@ -793,7 +793,7 @@ func (p *Player) SetCaughtFighting(v null.Int) { func (p *Player) SetCaughtFlying(v null.Int) { if p.CaughtFlying != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFlying:%v->%v", p.CaughtFlying, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFlying:%s->%s", FormatNull(p.CaughtFlying), FormatNull(v))) } p.CaughtFlying = v p.dirty = true @@ -803,7 +803,7 @@ func (p *Player) SetCaughtFlying(v null.Int) { func (p *Player) SetCaughtPoison(v null.Int) { if p.CaughtPoison != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPoison:%v->%v", p.CaughtPoison, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPoison:%s->%s", FormatNull(p.CaughtPoison), FormatNull(v))) } p.CaughtPoison = v p.dirty = true @@ -813,7 +813,7 @@ func (p *Player) SetCaughtPoison(v null.Int) { func (p *Player) SetCaughtGround(v null.Int) { if p.CaughtGround != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGround:%v->%v", p.CaughtGround, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGround:%s->%s", FormatNull(p.CaughtGround), FormatNull(v))) } p.CaughtGround = v p.dirty = true @@ -823,7 +823,7 @@ func (p *Player) SetCaughtGround(v null.Int) { func (p *Player) SetCaughtRock(v null.Int) { if p.CaughtRock != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtRock:%v->%v", p.CaughtRock, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtRock:%s->%s", FormatNull(p.CaughtRock), FormatNull(v))) } p.CaughtRock = v p.dirty = true @@ -833,7 +833,7 @@ func (p *Player) SetCaughtRock(v null.Int) { func (p *Player) SetCaughtBug(v null.Int) { if p.CaughtBug != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtBug:%v->%v", p.CaughtBug, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtBug:%s->%s", FormatNull(p.CaughtBug), FormatNull(v))) } p.CaughtBug = v p.dirty = true @@ -843,7 +843,7 @@ func (p *Player) SetCaughtBug(v null.Int) { func (p *Player) SetCaughtGhost(v null.Int) { if p.CaughtGhost != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGhost:%v->%v", p.CaughtGhost, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGhost:%s->%s", FormatNull(p.CaughtGhost), FormatNull(v))) } p.CaughtGhost = v p.dirty = true @@ -853,7 +853,7 @@ func (p *Player) SetCaughtGhost(v null.Int) { func (p *Player) SetCaughtSteel(v null.Int) { if p.CaughtSteel != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtSteel:%v->%v", p.CaughtSteel, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtSteel:%s->%s", FormatNull(p.CaughtSteel), FormatNull(v))) } p.CaughtSteel = v p.dirty = true @@ -863,7 +863,7 @@ func (p *Player) SetCaughtSteel(v null.Int) { func (p *Player) SetCaughtFire(v null.Int) { if p.CaughtFire != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFire:%v->%v", p.CaughtFire, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFire:%s->%s", FormatNull(p.CaughtFire), FormatNull(v))) } p.CaughtFire = v p.dirty = true @@ -873,7 +873,7 @@ func (p *Player) SetCaughtFire(v null.Int) { func (p *Player) SetCaughtWater(v null.Int) { if p.CaughtWater != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtWater:%v->%v", p.CaughtWater, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtWater:%s->%s", FormatNull(p.CaughtWater), FormatNull(v))) } p.CaughtWater = v p.dirty = true @@ -883,7 +883,7 @@ func (p *Player) SetCaughtWater(v null.Int) { func (p *Player) SetCaughtGrass(v null.Int) { if p.CaughtGrass != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGrass:%v->%v", p.CaughtGrass, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGrass:%s->%s", FormatNull(p.CaughtGrass), FormatNull(v))) } p.CaughtGrass = v p.dirty = true @@ -893,7 +893,7 @@ func (p *Player) SetCaughtGrass(v null.Int) { func (p *Player) SetCaughtElectric(v null.Int) { if p.CaughtElectric != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtElectric:%v->%v", p.CaughtElectric, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtElectric:%s->%s", FormatNull(p.CaughtElectric), FormatNull(v))) } p.CaughtElectric = v p.dirty = true @@ -903,7 +903,7 @@ func (p *Player) SetCaughtElectric(v null.Int) { func (p *Player) SetCaughtPsychic(v null.Int) { if p.CaughtPsychic != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPsychic:%v->%v", p.CaughtPsychic, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPsychic:%s->%s", FormatNull(p.CaughtPsychic), FormatNull(v))) } p.CaughtPsychic = v p.dirty = true @@ -913,7 +913,7 @@ func (p *Player) SetCaughtPsychic(v null.Int) { func (p *Player) SetCaughtIce(v null.Int) { if p.CaughtIce != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtIce:%v->%v", p.CaughtIce, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtIce:%s->%s", FormatNull(p.CaughtIce), FormatNull(v))) } p.CaughtIce = v p.dirty = true @@ -923,7 +923,7 @@ func (p *Player) SetCaughtIce(v null.Int) { func (p *Player) SetCaughtDragon(v null.Int) { if p.CaughtDragon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDragon:%v->%v", p.CaughtDragon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDragon:%s->%s", FormatNull(p.CaughtDragon), FormatNull(v))) } p.CaughtDragon = v p.dirty = true @@ -933,7 +933,7 @@ func (p *Player) SetCaughtDragon(v null.Int) { func (p *Player) SetCaughtDark(v null.Int) { if p.CaughtDark != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDark:%v->%v", p.CaughtDark, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDark:%s->%s", FormatNull(p.CaughtDark), FormatNull(v))) } p.CaughtDark = v p.dirty = true @@ -943,7 +943,7 @@ func (p *Player) SetCaughtDark(v null.Int) { func (p *Player) SetCaughtFairy(v null.Int) { if p.CaughtFairy != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFairy:%v->%v", p.CaughtFairy, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFairy:%s->%s", FormatNull(p.CaughtFairy), FormatNull(v))) } p.CaughtFairy = v p.dirty = true diff --git a/decoder/pokemon.go b/decoder/pokemon.go index d20ba93b..56cf62b1 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -170,7 +170,7 @@ func (pokemon *Pokemon) Unlock() { func (pokemon *Pokemon) SetPokestopId(v null.String) { if pokemon.PokestopId != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("PokestopId:%v->%v", pokemon.PokestopId, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("PokestopId:%s->%s", FormatNull(pokemon.PokestopId), FormatNull(v))) } pokemon.PokestopId = v pokemon.dirty = true @@ -180,7 +180,7 @@ func (pokemon *Pokemon) SetPokestopId(v null.String) { func (pokemon *Pokemon) SetSpawnId(v null.Int) { if pokemon.SpawnId != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SpawnId:%v->%v", pokemon.SpawnId, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SpawnId:%s->%s", FormatNull(pokemon.SpawnId), FormatNull(v))) } pokemon.SpawnId = v pokemon.dirty = true @@ -220,7 +220,7 @@ func (pokemon *Pokemon) SetPokemonId(v int16) { func (pokemon *Pokemon) SetForm(v null.Int) { if pokemon.Form != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Form:%v->%v", pokemon.Form, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Form:%s->%s", FormatNull(pokemon.Form), FormatNull(v))) } pokemon.Form = v pokemon.dirty = true @@ -230,7 +230,7 @@ func (pokemon *Pokemon) SetForm(v null.Int) { func (pokemon *Pokemon) SetCostume(v null.Int) { if pokemon.Costume != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Costume:%v->%v", pokemon.Costume, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Costume:%s->%s", FormatNull(pokemon.Costume), FormatNull(v))) } pokemon.Costume = v pokemon.dirty = true @@ -240,7 +240,7 @@ func (pokemon *Pokemon) SetCostume(v null.Int) { func (pokemon *Pokemon) SetGender(v null.Int) { if pokemon.Gender != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Gender:%v->%v", pokemon.Gender, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Gender:%s->%s", FormatNull(pokemon.Gender), FormatNull(v))) } pokemon.Gender = v pokemon.dirty = true @@ -250,7 +250,7 @@ func (pokemon *Pokemon) SetGender(v null.Int) { func (pokemon *Pokemon) SetWeather(v null.Int) { if pokemon.Weather != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weather:%v->%v", pokemon.Weather, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weather:%s->%s", FormatNull(pokemon.Weather), FormatNull(v))) } pokemon.Weather = v pokemon.dirty = true @@ -260,7 +260,7 @@ func (pokemon *Pokemon) SetWeather(v null.Int) { func (pokemon *Pokemon) SetIsStrong(v null.Bool) { if pokemon.IsStrong != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsStrong:%v->%v", pokemon.IsStrong, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsStrong:%s->%s", FormatNull(pokemon.IsStrong), FormatNull(v))) } pokemon.IsStrong = v pokemon.dirty = true @@ -270,7 +270,7 @@ func (pokemon *Pokemon) SetIsStrong(v null.Bool) { func (pokemon *Pokemon) SetExpireTimestamp(v null.Int) { if pokemon.ExpireTimestamp != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("ExpireTimestamp:%v->%v", pokemon.ExpireTimestamp, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("ExpireTimestamp:%s->%s", FormatNull(pokemon.ExpireTimestamp), FormatNull(v))) } pokemon.ExpireTimestamp = v pokemon.dirty = true @@ -290,7 +290,7 @@ func (pokemon *Pokemon) SetExpireTimestampVerified(v bool) { func (pokemon *Pokemon) SetSeenType(v null.String) { if pokemon.SeenType != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SeenType:%v->%v", pokemon.SeenType, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SeenType:%s->%s", FormatNull(pokemon.SeenType), FormatNull(v))) } pokemon.SeenType = v pokemon.dirty = true @@ -307,7 +307,7 @@ func (pokemon *Pokemon) SetUsername(v null.String) { func (pokemon *Pokemon) SetCellId(v null.Int) { if pokemon.CellId != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("CellId:%v->%v", pokemon.CellId, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("CellId:%s->%s", FormatNull(pokemon.CellId), FormatNull(v))) } pokemon.CellId = v pokemon.dirty = true @@ -327,7 +327,7 @@ func (pokemon *Pokemon) SetIsEvent(v int8) { func (pokemon *Pokemon) SetShiny(v null.Bool) { if pokemon.Shiny != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Shiny:%v->%v", pokemon.Shiny, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Shiny:%s->%s", FormatNull(pokemon.Shiny), FormatNull(v))) } pokemon.Shiny = v pokemon.dirty = true @@ -337,7 +337,7 @@ func (pokemon *Pokemon) SetShiny(v null.Bool) { func (pokemon *Pokemon) SetCp(v null.Int) { if pokemon.Cp != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Cp:%v->%v", pokemon.Cp, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Cp:%s->%s", FormatNull(pokemon.Cp), FormatNull(v))) } pokemon.Cp = v pokemon.dirty = true @@ -347,7 +347,7 @@ func (pokemon *Pokemon) SetCp(v null.Int) { func (pokemon *Pokemon) SetLevel(v null.Int) { if pokemon.Level != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Level:%v->%v", pokemon.Level, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Level:%s->%s", FormatNull(pokemon.Level), FormatNull(v))) } pokemon.Level = v pokemon.dirty = true @@ -357,7 +357,7 @@ func (pokemon *Pokemon) SetLevel(v null.Int) { func (pokemon *Pokemon) SetMove1(v null.Int) { if pokemon.Move1 != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move1:%v->%v", pokemon.Move1, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move1:%s->%s", FormatNull(pokemon.Move1), FormatNull(v))) } pokemon.Move1 = v pokemon.dirty = true @@ -367,7 +367,7 @@ func (pokemon *Pokemon) SetMove1(v null.Int) { func (pokemon *Pokemon) SetMove2(v null.Int) { if pokemon.Move2 != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move2:%v->%v", pokemon.Move2, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move2:%s->%s", FormatNull(pokemon.Move2), FormatNull(v))) } pokemon.Move2 = v pokemon.dirty = true @@ -377,7 +377,7 @@ func (pokemon *Pokemon) SetMove2(v null.Int) { func (pokemon *Pokemon) SetHeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Height, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Height:%v->%v", pokemon.Height, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Height:%s->%s", FormatNull(pokemon.Height), FormatNull(v))) } pokemon.Height = v pokemon.dirty = true @@ -387,7 +387,7 @@ func (pokemon *Pokemon) SetHeight(v null.Float) { func (pokemon *Pokemon) SetWeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Weight, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weight:%v->%v", pokemon.Weight, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weight:%s->%s", FormatNull(pokemon.Weight), FormatNull(v))) } pokemon.Weight = v pokemon.dirty = true @@ -397,7 +397,7 @@ func (pokemon *Pokemon) SetWeight(v null.Float) { func (pokemon *Pokemon) SetSize(v null.Int) { if pokemon.Size != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Size:%v->%v", pokemon.Size, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Size:%s->%s", FormatNull(pokemon.Size), FormatNull(v))) } pokemon.Size = v pokemon.dirty = true @@ -417,7 +417,7 @@ func (pokemon *Pokemon) SetIsDitto(v bool) { func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { if pokemon.DisplayPokemonId != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("DisplayPokemonId:%v->%v", pokemon.DisplayPokemonId, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("DisplayPokemonId:%s->%s", FormatNull(pokemon.DisplayPokemonId), FormatNull(v))) } pokemon.DisplayPokemonId = v pokemon.dirty = true @@ -427,7 +427,7 @@ func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { func (pokemon *Pokemon) SetPvp(v null.String) { if pokemon.Pvp != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Pvp:%v->%v", pokemon.Pvp, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Pvp:%s->%s", FormatNull(pokemon.Pvp), FormatNull(v))) } pokemon.Pvp = v pokemon.dirty = true @@ -437,7 +437,7 @@ func (pokemon *Pokemon) SetPvp(v null.String) { func (pokemon *Pokemon) SetCapture1(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture1, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture1:%v->%v", pokemon.Capture1, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture1:%s->%s", FormatNull(pokemon.Capture1), FormatNull(v))) } pokemon.Capture1 = v pokemon.dirty = true @@ -447,7 +447,7 @@ func (pokemon *Pokemon) SetCapture1(v null.Float) { func (pokemon *Pokemon) SetCapture2(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture2, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture2:%v->%v", pokemon.Capture2, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture2:%s->%s", FormatNull(pokemon.Capture2), FormatNull(v))) } pokemon.Capture2 = v pokemon.dirty = true @@ -457,7 +457,7 @@ func (pokemon *Pokemon) SetCapture2(v null.Float) { func (pokemon *Pokemon) SetCapture3(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture3, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture3:%v->%v", pokemon.Capture3, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture3:%s->%s", FormatNull(pokemon.Capture3), FormatNull(v))) } pokemon.Capture3 = v pokemon.dirty = true @@ -467,7 +467,7 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { func (pokemon *Pokemon) SetUpdated(v null.Int) { if pokemon.Updated != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Updated:%v->%v", pokemon.Updated, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Updated:%s->%s", FormatNull(pokemon.Updated), FormatNull(v))) } pokemon.Updated = v pokemon.dirty = true diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 8d97c8a1..20d771b7 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -191,7 +191,7 @@ func (p *Pokestop) SetLon(v float64) { func (p *Pokestop) SetName(v null.String) { if p.Name != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Name:%v->%v", p.Name, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Name:%s->%s", FormatNull(p.Name), FormatNull(v))) } p.Name = v p.dirty = true @@ -201,7 +201,7 @@ func (p *Pokestop) SetName(v null.String) { func (p *Pokestop) SetUrl(v null.String) { if p.Url != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Url:%v->%v", p.Url, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Url:%s->%s", FormatNull(p.Url), FormatNull(v))) } p.Url = v p.dirty = true @@ -211,7 +211,7 @@ func (p *Pokestop) SetUrl(v null.String) { func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { if p.LureExpireTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LureExpireTimestamp:%v->%v", p.LureExpireTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LureExpireTimestamp:%s->%s", FormatNull(p.LureExpireTimestamp), FormatNull(v))) } p.LureExpireTimestamp = v p.dirty = true @@ -221,7 +221,7 @@ func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { if p.LastModifiedTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LastModifiedTimestamp:%v->%v", p.LastModifiedTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LastModifiedTimestamp:%s->%s", FormatNull(p.LastModifiedTimestamp), FormatNull(v))) } p.LastModifiedTimestamp = v p.dirty = true @@ -231,7 +231,7 @@ func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { func (p *Pokestop) SetEnabled(v null.Bool) { if p.Enabled != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Enabled:%v->%v", p.Enabled, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Enabled:%s->%s", FormatNull(p.Enabled), FormatNull(v))) } p.Enabled = v p.dirty = true @@ -241,7 +241,7 @@ func (p *Pokestop) SetEnabled(v null.Bool) { func (p *Pokestop) SetQuestType(v null.Int) { if p.QuestType != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestType:%v->%v", p.QuestType, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestType:%s->%s", FormatNull(p.QuestType), FormatNull(v))) } p.QuestType = v p.dirty = true @@ -251,7 +251,7 @@ func (p *Pokestop) SetQuestType(v null.Int) { func (p *Pokestop) SetQuestTimestamp(v null.Int) { if p.QuestTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTimestamp:%v->%v", p.QuestTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTimestamp:%s->%s", FormatNull(p.QuestTimestamp), FormatNull(v))) } p.QuestTimestamp = v p.dirty = true @@ -261,7 +261,7 @@ func (p *Pokestop) SetQuestTimestamp(v null.Int) { func (p *Pokestop) SetQuestTarget(v null.Int) { if p.QuestTarget != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTarget:%v->%v", p.QuestTarget, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTarget:%s->%s", FormatNull(p.QuestTarget), FormatNull(v))) } p.QuestTarget = v p.dirty = true @@ -271,7 +271,7 @@ func (p *Pokestop) SetQuestTarget(v null.Int) { func (p *Pokestop) SetQuestConditions(v null.String) { if p.QuestConditions != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestConditions:%v->%v", p.QuestConditions, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestConditions:%s->%s", FormatNull(p.QuestConditions), FormatNull(v))) } p.QuestConditions = v p.dirty = true @@ -281,7 +281,7 @@ func (p *Pokestop) SetQuestConditions(v null.String) { func (p *Pokestop) SetQuestRewards(v null.String) { if p.QuestRewards != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestRewards:%v->%v", p.QuestRewards, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestRewards:%s->%s", FormatNull(p.QuestRewards), FormatNull(v))) } p.QuestRewards = v p.dirty = true @@ -291,7 +291,7 @@ func (p *Pokestop) SetQuestRewards(v null.String) { func (p *Pokestop) SetQuestTemplate(v null.String) { if p.QuestTemplate != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTemplate:%v->%v", p.QuestTemplate, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTemplate:%s->%s", FormatNull(p.QuestTemplate), FormatNull(v))) } p.QuestTemplate = v p.dirty = true @@ -301,7 +301,7 @@ func (p *Pokestop) SetQuestTemplate(v null.String) { func (p *Pokestop) SetQuestTitle(v null.String) { if p.QuestTitle != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTitle:%v->%v", p.QuestTitle, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTitle:%s->%s", FormatNull(p.QuestTitle), FormatNull(v))) } p.QuestTitle = v p.dirty = true @@ -311,7 +311,7 @@ func (p *Pokestop) SetQuestTitle(v null.String) { func (p *Pokestop) SetQuestExpiry(v null.Int) { if p.QuestExpiry != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestExpiry:%v->%v", p.QuestExpiry, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestExpiry:%s->%s", FormatNull(p.QuestExpiry), FormatNull(v))) } p.QuestExpiry = v p.dirty = true @@ -321,7 +321,7 @@ func (p *Pokestop) SetQuestExpiry(v null.Int) { func (p *Pokestop) SetCellId(v null.Int) { if p.CellId != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CellId:%v->%v", p.CellId, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CellId:%s->%s", FormatNull(p.CellId), FormatNull(v))) } p.CellId = v p.dirty = true @@ -361,7 +361,7 @@ func (p *Pokestop) SetFirstSeenTimestamp(v int16) { func (p *Pokestop) SetSponsorId(v null.Int) { if p.SponsorId != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("SponsorId:%v->%v", p.SponsorId, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("SponsorId:%s->%s", FormatNull(p.SponsorId), FormatNull(v))) } p.SponsorId = v p.dirty = true @@ -371,7 +371,7 @@ func (p *Pokestop) SetSponsorId(v null.Int) { func (p *Pokestop) SetPartnerId(v null.String) { if p.PartnerId != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PartnerId:%v->%v", p.PartnerId, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PartnerId:%s->%s", FormatNull(p.PartnerId), FormatNull(v))) } p.PartnerId = v p.dirty = true @@ -381,7 +381,7 @@ func (p *Pokestop) SetPartnerId(v null.String) { func (p *Pokestop) SetArScanEligible(v null.Int) { if p.ArScanEligible != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ArScanEligible:%v->%v", p.ArScanEligible, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ArScanEligible:%s->%s", FormatNull(p.ArScanEligible), FormatNull(v))) } p.ArScanEligible = v p.dirty = true @@ -391,7 +391,7 @@ func (p *Pokestop) SetArScanEligible(v null.Int) { func (p *Pokestop) SetPowerUpLevel(v null.Int) { if p.PowerUpLevel != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpLevel:%v->%v", p.PowerUpLevel, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpLevel:%s->%s", FormatNull(p.PowerUpLevel), FormatNull(v))) } p.PowerUpLevel = v p.dirty = true @@ -401,7 +401,7 @@ func (p *Pokestop) SetPowerUpLevel(v null.Int) { func (p *Pokestop) SetPowerUpPoints(v null.Int) { if p.PowerUpPoints != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpPoints:%v->%v", p.PowerUpPoints, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpPoints:%s->%s", FormatNull(p.PowerUpPoints), FormatNull(v))) } p.PowerUpPoints = v p.dirty = true @@ -411,7 +411,7 @@ func (p *Pokestop) SetPowerUpPoints(v null.Int) { func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { if p.PowerUpEndTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%v->%v", p.PowerUpEndTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%s->%s", FormatNull(p.PowerUpEndTimestamp), FormatNull(v))) } p.PowerUpEndTimestamp = v p.dirty = true @@ -421,7 +421,7 @@ func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { func (p *Pokestop) SetAlternativeQuestType(v null.Int) { if p.AlternativeQuestType != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestType:%v->%v", p.AlternativeQuestType, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestType:%s->%s", FormatNull(p.AlternativeQuestType), FormatNull(v))) } p.AlternativeQuestType = v p.dirty = true @@ -431,7 +431,7 @@ func (p *Pokestop) SetAlternativeQuestType(v null.Int) { func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { if p.AlternativeQuestTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTimestamp:%v->%v", p.AlternativeQuestTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTimestamp:%s->%s", FormatNull(p.AlternativeQuestTimestamp), FormatNull(v))) } p.AlternativeQuestTimestamp = v p.dirty = true @@ -441,7 +441,7 @@ func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { if p.AlternativeQuestTarget != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTarget:%v->%v", p.AlternativeQuestTarget, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTarget:%s->%s", FormatNull(p.AlternativeQuestTarget), FormatNull(v))) } p.AlternativeQuestTarget = v p.dirty = true @@ -451,7 +451,7 @@ func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { if p.AlternativeQuestConditions != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestConditions:%v->%v", p.AlternativeQuestConditions, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestConditions:%s->%s", FormatNull(p.AlternativeQuestConditions), FormatNull(v))) } p.AlternativeQuestConditions = v p.dirty = true @@ -461,7 +461,7 @@ func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { if p.AlternativeQuestRewards != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestRewards:%v->%v", p.AlternativeQuestRewards, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestRewards:%s->%s", FormatNull(p.AlternativeQuestRewards), FormatNull(v))) } p.AlternativeQuestRewards = v p.dirty = true @@ -471,7 +471,7 @@ func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { if p.AlternativeQuestTemplate != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTemplate:%v->%v", p.AlternativeQuestTemplate, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTemplate:%s->%s", FormatNull(p.AlternativeQuestTemplate), FormatNull(v))) } p.AlternativeQuestTemplate = v p.dirty = true @@ -481,7 +481,7 @@ func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { if p.AlternativeQuestTitle != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTitle:%v->%v", p.AlternativeQuestTitle, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTitle:%s->%s", FormatNull(p.AlternativeQuestTitle), FormatNull(v))) } p.AlternativeQuestTitle = v p.dirty = true @@ -491,7 +491,7 @@ func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { if p.AlternativeQuestExpiry != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestExpiry:%v->%v", p.AlternativeQuestExpiry, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestExpiry:%s->%s", FormatNull(p.AlternativeQuestExpiry), FormatNull(v))) } p.AlternativeQuestExpiry = v p.dirty = true @@ -501,7 +501,7 @@ func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { func (p *Pokestop) SetDescription(v null.String) { if p.Description != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Description:%v->%v", p.Description, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Description:%s->%s", FormatNull(p.Description), FormatNull(v))) } p.Description = v p.dirty = true @@ -511,7 +511,7 @@ func (p *Pokestop) SetDescription(v null.String) { func (p *Pokestop) SetShowcaseFocus(v null.String) { if p.ShowcaseFocus != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseFocus:%v->%v", p.ShowcaseFocus, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseFocus:%s->%s", FormatNull(p.ShowcaseFocus), FormatNull(v))) } p.ShowcaseFocus = v p.dirty = true @@ -521,7 +521,7 @@ func (p *Pokestop) SetShowcaseFocus(v null.String) { func (p *Pokestop) SetShowcasePokemon(v null.Int) { if p.ShowcasePokemon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemon:%v->%v", p.ShowcasePokemon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemon:%s->%s", FormatNull(p.ShowcasePokemon), FormatNull(v))) } p.ShowcasePokemon = v p.dirty = true @@ -531,7 +531,7 @@ func (p *Pokestop) SetShowcasePokemon(v null.Int) { func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { if p.ShowcasePokemonForm != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonForm:%v->%v", p.ShowcasePokemonForm, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonForm:%s->%s", FormatNull(p.ShowcasePokemonForm), FormatNull(v))) } p.ShowcasePokemonForm = v p.dirty = true @@ -541,7 +541,7 @@ func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { func (p *Pokestop) SetShowcasePokemonType(v null.Int) { if p.ShowcasePokemonType != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonType:%v->%v", p.ShowcasePokemonType, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonType:%s->%s", FormatNull(p.ShowcasePokemonType), FormatNull(v))) } p.ShowcasePokemonType = v p.dirty = true @@ -551,7 +551,7 @@ func (p *Pokestop) SetShowcasePokemonType(v null.Int) { func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { if p.ShowcaseRankingStandard != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankingStandard:%v->%v", p.ShowcaseRankingStandard, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankingStandard:%s->%s", FormatNull(p.ShowcaseRankingStandard), FormatNull(v))) } p.ShowcaseRankingStandard = v p.dirty = true @@ -561,7 +561,7 @@ func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { func (p *Pokestop) SetShowcaseExpiry(v null.Int) { if p.ShowcaseExpiry != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseExpiry:%v->%v", p.ShowcaseExpiry, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseExpiry:%s->%s", FormatNull(p.ShowcaseExpiry), FormatNull(v))) } p.ShowcaseExpiry = v p.dirty = true @@ -571,7 +571,7 @@ func (p *Pokestop) SetShowcaseExpiry(v null.Int) { func (p *Pokestop) SetShowcaseRankings(v null.String) { if p.ShowcaseRankings != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankings:%v->%v", p.ShowcaseRankings, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankings:%s->%s", FormatNull(p.ShowcaseRankings), FormatNull(v))) } p.ShowcaseRankings = v p.dirty = true diff --git a/decoder/routes.go b/decoder/routes.go index 0fa7ee6a..567a7adb 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -245,7 +245,7 @@ func (r *Route) SetStartLon(v float64) { func (r *Route) SetTags(v null.String) { if r.Tags != v { if dbDebugEnabled { - r.changedFields = append(r.changedFields, fmt.Sprintf("Tags:%v->%v", r.Tags, v)) + r.changedFields = append(r.changedFields, fmt.Sprintf("Tags:%s->%s", FormatNull(r.Tags), FormatNull(v))) } r.Tags = v r.dirty = true diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 9171a852..0f5f0f3b 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -99,7 +99,7 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { // Handle validity changes if (s.DespawnSec.Valid && !v.Valid) || (!s.DespawnSec.Valid && v.Valid) { if dbDebugEnabled { - s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%v->%v", s.DespawnSec, v)) + s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%s->%s", FormatNull(s.DespawnSec), FormatNull(v))) } s.DespawnSec = v s.dirty = true @@ -126,7 +126,7 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { // Allow 2-second tolerance for despawn time if Abs(oldVal-newVal) > 2 { if dbDebugEnabled { - s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%v->%v", s.DespawnSec, v)) + s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%s->%s", FormatNull(s.DespawnSec), FormatNull(v))) } s.DespawnSec = v s.dirty = true diff --git a/decoder/station.go b/decoder/station.go index 8eb1e307..8a7b7bcd 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -204,7 +204,7 @@ func (station *Station) SetIsInactive(v bool) { func (station *Station) SetBattleLevel(v null.Int) { if station.BattleLevel != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattleLevel:%v->%v", station.BattleLevel, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleLevel:%s->%s", FormatNull(station.BattleLevel), FormatNull(v))) } station.BattleLevel = v station.dirty = true @@ -214,7 +214,7 @@ func (station *Station) SetBattleLevel(v null.Int) { func (station *Station) SetBattleStart(v null.Int) { if station.BattleStart != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattleStart:%v->%v", station.BattleStart, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleStart:%s->%s", FormatNull(station.BattleStart), FormatNull(v))) } station.BattleStart = v station.dirty = true @@ -224,7 +224,7 @@ func (station *Station) SetBattleStart(v null.Int) { func (station *Station) SetBattleEnd(v null.Int) { if station.BattleEnd != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattleEnd:%v->%v", station.BattleEnd, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleEnd:%s->%s", FormatNull(station.BattleEnd), FormatNull(v))) } station.BattleEnd = v station.dirty = true @@ -234,7 +234,7 @@ func (station *Station) SetBattleEnd(v null.Int) { func (station *Station) SetBattlePokemonId(v null.Int) { if station.BattlePokemonId != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonId:%v->%v", station.BattlePokemonId, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonId:%s->%s", FormatNull(station.BattlePokemonId), FormatNull(v))) } station.BattlePokemonId = v station.dirty = true @@ -244,7 +244,7 @@ func (station *Station) SetBattlePokemonId(v null.Int) { func (station *Station) SetBattlePokemonForm(v null.Int) { if station.BattlePokemonForm != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonForm:%v->%v", station.BattlePokemonForm, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonForm:%s->%s", FormatNull(station.BattlePokemonForm), FormatNull(v))) } station.BattlePokemonForm = v station.dirty = true @@ -254,7 +254,7 @@ func (station *Station) SetBattlePokemonForm(v null.Int) { func (station *Station) SetBattlePokemonCostume(v null.Int) { if station.BattlePokemonCostume != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCostume:%v->%v", station.BattlePokemonCostume, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCostume:%s->%s", FormatNull(station.BattlePokemonCostume), FormatNull(v))) } station.BattlePokemonCostume = v station.dirty = true @@ -264,7 +264,7 @@ func (station *Station) SetBattlePokemonCostume(v null.Int) { func (station *Station) SetBattlePokemonGender(v null.Int) { if station.BattlePokemonGender != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonGender:%v->%v", station.BattlePokemonGender, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonGender:%s->%s", FormatNull(station.BattlePokemonGender), FormatNull(v))) } station.BattlePokemonGender = v station.dirty = true @@ -274,7 +274,7 @@ func (station *Station) SetBattlePokemonGender(v null.Int) { func (station *Station) SetBattlePokemonAlignment(v null.Int) { if station.BattlePokemonAlignment != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonAlignment:%v->%v", station.BattlePokemonAlignment, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonAlignment:%s->%s", FormatNull(station.BattlePokemonAlignment), FormatNull(v))) } station.BattlePokemonAlignment = v station.dirty = true @@ -284,7 +284,7 @@ func (station *Station) SetBattlePokemonAlignment(v null.Int) { func (station *Station) SetBattlePokemonBreadMode(v null.Int) { if station.BattlePokemonBreadMode != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonBreadMode:%v->%v", station.BattlePokemonBreadMode, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonBreadMode:%s->%s", FormatNull(station.BattlePokemonBreadMode), FormatNull(v))) } station.BattlePokemonBreadMode = v station.dirty = true @@ -294,7 +294,7 @@ func (station *Station) SetBattlePokemonBreadMode(v null.Int) { func (station *Station) SetBattlePokemonMove1(v null.Int) { if station.BattlePokemonMove1 != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove1:%v->%v", station.BattlePokemonMove1, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove1:%s->%s", FormatNull(station.BattlePokemonMove1), FormatNull(v))) } station.BattlePokemonMove1 = v station.dirty = true @@ -304,7 +304,7 @@ func (station *Station) SetBattlePokemonMove1(v null.Int) { func (station *Station) SetBattlePokemonMove2(v null.Int) { if station.BattlePokemonMove2 != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove2:%v->%v", station.BattlePokemonMove2, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove2:%s->%s", FormatNull(station.BattlePokemonMove2), FormatNull(v))) } station.BattlePokemonMove2 = v station.dirty = true @@ -314,7 +314,7 @@ func (station *Station) SetBattlePokemonMove2(v null.Int) { func (station *Station) SetBattlePokemonStamina(v null.Int) { if station.BattlePokemonStamina != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonStamina:%v->%v", station.BattlePokemonStamina, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonStamina:%s->%s", FormatNull(station.BattlePokemonStamina), FormatNull(v))) } station.BattlePokemonStamina = v station.dirty = true @@ -324,7 +324,7 @@ func (station *Station) SetBattlePokemonStamina(v null.Int) { func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { if !nullFloatAlmostEqual(station.BattlePokemonCpMultiplier, v, floatTolerance) { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCpMultiplier:%v->%v", station.BattlePokemonCpMultiplier, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCpMultiplier:%s->%s", FormatNull(station.BattlePokemonCpMultiplier), FormatNull(v))) } station.BattlePokemonCpMultiplier = v station.dirty = true @@ -334,7 +334,7 @@ func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { func (station *Station) SetTotalStationedPokemon(v null.Int) { if station.TotalStationedPokemon != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedPokemon:%v->%v", station.TotalStationedPokemon, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedPokemon:%s->%s", FormatNull(station.TotalStationedPokemon), FormatNull(v))) } station.TotalStationedPokemon = v station.dirty = true @@ -344,7 +344,7 @@ func (station *Station) SetTotalStationedPokemon(v null.Int) { func (station *Station) SetTotalStationedGmax(v null.Int) { if station.TotalStationedGmax != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedGmax:%v->%v", station.TotalStationedGmax, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedGmax:%s->%s", FormatNull(station.TotalStationedGmax), FormatNull(v))) } station.TotalStationedGmax = v station.dirty = true @@ -354,7 +354,7 @@ func (station *Station) SetTotalStationedGmax(v null.Int) { func (station *Station) SetStationedPokemon(v null.String) { if station.StationedPokemon != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("StationedPokemon:%v->%v", station.StationedPokemon, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("StationedPokemon:%s->%s", FormatNull(station.StationedPokemon), FormatNull(v))) } station.StationedPokemon = v station.dirty = true diff --git a/decoder/tappable.go b/decoder/tappable.go index 98573571..20ff851d 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -80,7 +80,7 @@ func (ta *Tappable) SetLon(v float64) { func (ta *Tappable) SetFortId(v null.String) { if ta.FortId != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("FortId:%v->%v", ta.FortId, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("FortId:%s->%s", FormatNull(ta.FortId), FormatNull(v))) } ta.FortId = v ta.dirty = true @@ -90,7 +90,7 @@ func (ta *Tappable) SetFortId(v null.String) { func (ta *Tappable) SetSpawnId(v null.Int) { if ta.SpawnId != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("SpawnId:%v->%v", ta.SpawnId, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("SpawnId:%s->%s", FormatNull(ta.SpawnId), FormatNull(v))) } ta.SpawnId = v ta.dirty = true @@ -110,7 +110,7 @@ func (ta *Tappable) SetType(v string) { func (ta *Tappable) SetEncounter(v null.Int) { if ta.Encounter != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("Encounter:%v->%v", ta.Encounter, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Encounter:%s->%s", FormatNull(ta.Encounter), FormatNull(v))) } ta.Encounter = v ta.dirty = true @@ -120,7 +120,7 @@ func (ta *Tappable) SetEncounter(v null.Int) { func (ta *Tappable) SetItemId(v null.Int) { if ta.ItemId != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("ItemId:%v->%v", ta.ItemId, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ItemId:%s->%s", FormatNull(ta.ItemId), FormatNull(v))) } ta.ItemId = v ta.dirty = true @@ -130,7 +130,7 @@ func (ta *Tappable) SetItemId(v null.Int) { func (ta *Tappable) SetCount(v null.Int) { if ta.Count != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("Count:%v->%v", ta.Count, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Count:%s->%s", FormatNull(ta.Count), FormatNull(v))) } ta.Count = v ta.dirty = true @@ -140,7 +140,7 @@ func (ta *Tappable) SetCount(v null.Int) { func (ta *Tappable) SetExpireTimestamp(v null.Int) { if ta.ExpireTimestamp != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("ExpireTimestamp:%v->%v", ta.ExpireTimestamp, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ExpireTimestamp:%s->%s", FormatNull(ta.ExpireTimestamp), FormatNull(v))) } ta.ExpireTimestamp = v ta.dirty = true From a724af67d89a2923b2a866276125db13dbfd1194 Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 3 Feb 2026 13:27:11 +0000 Subject: [PATCH 35/78] Initial merge of DNF api and pre-load --- config.toml.example | 6 + config/config.go | 3 +- decoder/api_fort.go | 396 +++++++++++++++++++++++++++++ decoder/fortRtree.go | 260 ++++++++++++------- decoder/fort_preload.go | 158 ++++++++++++ decoder/fort_tracker.go | 21 ++ decoder/gym_state.go | 21 +- decoder/main.go | 13 + decoder/pokestop.go | 196 ++++++++++---- decoder/pokestop_decode.go | 62 ++++- decoder/pokestop_state.go | 65 +++-- main.go | 24 +- routes.go | 36 +++ sql/50_quest_reward_columns.up.sql | 44 ++++ 14 files changed, 1141 insertions(+), 164 deletions(-) create mode 100644 decoder/api_fort.go create mode 100644 decoder/fort_preload.go create mode 100644 sql/50_quest_reward_columns.up.sql diff --git a/config.toml.example b/config.toml.example index 2c018fcb..ed23dabf 100644 --- a/config.toml.example +++ b/config.toml.example @@ -11,6 +11,12 @@ reduce_updates = false pokemon_memory_only = false # Use in-memory storage for pokemon only +# Fort loading options: +# preload_forts: Pre-load all forts into cache on startup (faster initial responses) +# fort_in_memory: Keep forts in memory with rtree for spatial lookups +preload_forts = false +fort_in_memory = false + [koji] url = "http://{koji_url}/api/v1/geofence/feature-collection/{golbat_project}" bearer_token = "secret" diff --git a/config/config.go b/config/config.go index 6eacd7da..e0e4e0be 100644 --- a/config/config.go +++ b/config/config.go @@ -17,7 +17,8 @@ type configDefinition struct { Prometheus Prometheus `koanf:"prometheus"` PokemonMemoryOnly bool `koanf:"pokemon_memory_only"` PokemonInternalToDb bool `koanf:"pokemon_internal_to_db"` - TestFortInMemory bool `koanf:"test_fort_in_memory"` + PreloadForts bool `koanf:"preload_forts"` // Pre-load all forts into cache on startup + FortInMemory bool `koanf:"fort_in_memory"` // Keep forts in memory with rtree for spatial lookups Cleanup cleanup `koanf:"cleanup"` RawBearer string `koanf:"raw_bearer"` ApiSecret string `koanf:"api_secret"` diff --git a/decoder/api_fort.go b/decoder/api_fort.go new file mode 100644 index 00000000..d9f03d30 --- /dev/null +++ b/decoder/api_fort.go @@ -0,0 +1,396 @@ +package decoder + +import ( + "context" + "slices" + "time" + + log "github.com/sirupsen/logrus" + + "golbat/config" + "golbat/db" + "golbat/geo" +) + +type ApiFortScan struct { + Min geo.Location `json:"min"` + Max geo.Location `json:"max"` + Limit int `json:"limit"` + DnfFilters []ApiFortDnfFilter `json:"filters"` +} + +type ApiFortDnfFilter struct { + PowerUpLevel *ApiFortDnfMinMax8 `json:"power_up_level"` + IsArScanEligible bool `json:"is_ar_scan_eligible"` + + AvailableSlots *ApiFortDnfMinMax8 `json:"available_slots"` + TeamId []*int8 `json:"team_id"` + InBattle bool `json:"in_battle"` + RaidLevel []*int8 `json:"raid_level"` + RaidPokemon []ApiDnfId `json:"raid_pokemon_id"` + + LureId []*int16 `json:"lure_id"` + + ArQuestRewardType []*int16 `json:"ar_quest_reward_type"` + ArQuestRewardAmount *ApiFortDnfMinMax16 `json:"ar_quest_reward_amount"` + ArQuestRewardItemId []*int16 `json:"ar_quest_reward_item_id"` + ArQuestRewardPokemon []ApiDnfId `json:"ar_quest_reward_pokemon"` + ArQuestType []*int16 `json:"ar_quest_type"` + ArQuestTarget []*int16 `json:"ar_quest_target"` + ArQuestTemplate []string `json:"ar_quest_template"` + + NoArQuestRewardType []*int16 `json:"noar_quest_reward_type"` + NoArQuestRewardAmount *ApiFortDnfMinMax16 `json:"noar_quest_reward_amount"` + NoArQuestRewardItemId []*int16 `json:"noar_quest_reward_item_id"` + NoArQuestRewardPokemon []ApiDnfId `json:"noar_quest_reward_pokemon"` + NoArQuestType []*int16 `json:"noar_quest_type"` + NoArQuestTarget []*int16 `json:"noar_quest_target"` + NoArQuestTemplate []string `json:"noar_quest_template"` + + IncidentDisplayType []*int8 `json:"incident_display_type"` + IncidentStyle []*int8 `json:"incident_style"` + IncidentCharacter []*int16 `json:"incident_character"` + IncidentSlot1 []ApiDnfId `json:"incident_slot_1"` + IncidentSlot2 []ApiDnfId `json:"incident_slot_2"` + IncidentSlot3 []ApiDnfId `json:"incident_slot_3"` + ContestPokemon []ApiDnfId `json:"contest_pokemon"` + ContestPokemonType1 []*int8 `json:"contest_pokemon_type_1"` + ContestPokemonType2 []*int8 `json:"contest_pokemon_type_2"` + ContestRankingStandard []*int8 `json:"contest_ranking_standard"` + ContestTotalEntries *ApiFortDnfMinMax16 `json:"contest_total_entries"` +} + +type ApiDnfId struct { + Pokemon int16 `json:"pokemon_id"` + Form *int16 `json:"form"` +} + +type ApiFortDnfMinMax8 struct { + Min int8 `json:"min"` + Max int8 `json:"max"` +} + +type ApiFortDnfMinMax16 struct { + Min int16 `json:"min"` + Max int16 `json:"max"` +} + +type ApiGymScanResult struct { + Gyms []*Gym `json:"gyms"` + Examined int `json:"examined"` + Skipped int `json:"skipped"` + Total int `json:"total"` +} + +type ApiPokestopScanResult struct { + Pokestops []*Pokestop `json:"pokestops"` + Examined int `json:"examined"` + Skipped int `json:"skipped"` + Total int `json:"total"` +} + +func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDnfFilter) bool { + if fortType != fortLookup.FortType { + return false + } + if filter.PowerUpLevel != nil && (fortLookup.PowerUpLevel < filter.PowerUpLevel.Min || fortLookup.PowerUpLevel > filter.PowerUpLevel.Max) { + return false + } + if filter.IsArScanEligible && !fortLookup.IsArScanEligible { + return false + } + + if fortLookup.FortType == GYM { + if filter.AvailableSlots != nil && (fortLookup.AvailableSlots < filter.AvailableSlots.Min || fortLookup.AvailableSlots > filter.AvailableSlots.Max) { + return false + } + if filter.TeamId != nil && !slices.Contains(filter.TeamId, &fortLookup.TeamId) { + return false + } + if filter.InBattle && !fortLookup.InBattle { + return false + } + if filter.RaidLevel != nil && !slices.Contains(filter.RaidLevel, &fortLookup.RaidLevel) { + return false + } + if filter.RaidPokemon != nil { + raidPokemonMatch := false + for _, raidPokemon := range filter.RaidPokemon { + if raidPokemon.Pokemon == fortLookup.RaidPokemonId && (raidPokemon.Form == nil || *raidPokemon.Form == fortLookup.RaidPokemonForm) { + raidPokemonMatch = true + break + } + } + if !raidPokemonMatch { + return false + } + } + } else if fortLookup.FortType == POKESTOP { + if filter.LureId != nil && !slices.Contains(filter.LureId, &fortLookup.LureId) { + return false + } + // AR Quest Filters + if filter.ArQuestRewardType != nil && !slices.Contains(filter.ArQuestRewardType, &fortLookup.QuestArRewardType) { + return false + } + if filter.ArQuestRewardAmount != nil && (fortLookup.QuestArRewardAmount < filter.ArQuestRewardAmount.Min || fortLookup.QuestArRewardAmount > filter.ArQuestRewardAmount.Max) { + return false + } + if filter.ArQuestRewardItemId != nil && !slices.Contains(filter.ArQuestRewardItemId, &fortLookup.QuestArRewardItemId) { + return false + } + if filter.ArQuestType != nil && !slices.Contains(filter.ArQuestType, &fortLookup.QuestArType) { + return false + } + if filter.ArQuestTarget != nil && !slices.Contains(filter.ArQuestTarget, &fortLookup.QuestArTarget) { + return false + } + if filter.ArQuestTemplate != nil && !slices.Contains(filter.ArQuestTemplate, fortLookup.QuestArTemplate) { + return false + } + if filter.ArQuestRewardPokemon != nil { + match := false + for _, pkm := range filter.ArQuestRewardPokemon { + if pkm.Pokemon == fortLookup.QuestArRewardPokemonId && (pkm.Form == nil || *pkm.Form == fortLookup.QuestArRewardPokemonForm) { + match = true + break + } + } + if !match { + return false + } + } + + // No-AR Quest Filters + if filter.NoArQuestRewardType != nil && !slices.Contains(filter.NoArQuestRewardType, &fortLookup.QuestNoArRewardType) { + return false + } + if filter.NoArQuestRewardAmount != nil && (fortLookup.QuestNoArRewardAmount < filter.NoArQuestRewardAmount.Min || fortLookup.QuestNoArRewardAmount > filter.NoArQuestRewardAmount.Max) { + return false + } + if filter.NoArQuestRewardItemId != nil && !slices.Contains(filter.NoArQuestRewardItemId, &fortLookup.QuestNoArRewardItemId) { + return false + } + if filter.NoArQuestType != nil && !slices.Contains(filter.NoArQuestType, &fortLookup.QuestNoArType) { + return false + } + if filter.NoArQuestTarget != nil && !slices.Contains(filter.NoArQuestTarget, &fortLookup.QuestNoArTarget) { + return false + } + if filter.NoArQuestTemplate != nil && !slices.Contains(filter.NoArQuestTemplate, fortLookup.QuestNoArTemplate) { + return false + } + if filter.NoArQuestRewardPokemon != nil { + match := false + for _, pkm := range filter.NoArQuestRewardPokemon { + if pkm.Pokemon == fortLookup.QuestNoArRewardPokemonId && (pkm.Form == nil || *pkm.Form == fortLookup.QuestNoArRewardPokemonForm) { + match = true + break + } + } + if !match { + return false + } + } + + // Contest Filters + if filter.ContestPokemonType1 != nil && !slices.Contains(filter.ContestPokemonType1, &fortLookup.ContestPokemonType1) { + return false + } + if filter.ContestPokemonType2 != nil && !slices.Contains(filter.ContestPokemonType2, &fortLookup.ContestPokemonType2) { + return false + } + if filter.ContestRankingStandard != nil && !slices.Contains(filter.ContestRankingStandard, &fortLookup.ContestRankingStandard) { + return false + } + if filter.ContestTotalEntries != nil && (fortLookup.ContestTotalEntries < filter.ContestTotalEntries.Min || fortLookup.ContestTotalEntries > filter.ContestTotalEntries.Max) { + return false + } + if filter.ContestPokemon != nil { + match := false + for _, pkm := range filter.ContestPokemon { + if pkm.Pokemon == fortLookup.ContestPokemonId && (pkm.Form == nil || *pkm.Form == fortLookup.ContestPokemonForm) { + match = true + break + } + } + if !match { + return false + } + } + + // Incident Filters + if len(filter.IncidentDisplayType) > 0 || len(filter.IncidentStyle) > 0 || len(filter.IncidentCharacter) > 0 || len(filter.IncidentSlot1) > 0 || len(filter.IncidentSlot2) > 0 || len(filter.IncidentSlot3) > 0 { + incidentMatch := false + for _, incident := range fortLookup.Incidents { + incidentFilterMatch := true + if filter.IncidentDisplayType != nil && !slices.Contains(filter.IncidentDisplayType, &incident.DisplayType) { + incidentFilterMatch = false + } + if incidentFilterMatch && filter.IncidentStyle != nil && !slices.Contains(filter.IncidentStyle, &incident.Style) { + incidentFilterMatch = false + } + if incidentFilterMatch && filter.IncidentCharacter != nil && !slices.Contains(filter.IncidentCharacter, &incident.Character) { + incidentFilterMatch = false + } + if incidentFilterMatch && filter.IncidentSlot1 != nil { + slotMatch := false + for _, pkm := range filter.IncidentSlot1 { + if pkm.Pokemon == incident.Slot1PokemonId && (pkm.Form == nil || *pkm.Form == incident.Slot1PokemonForm) { + slotMatch = true + break + } + } + if !slotMatch { + incidentFilterMatch = false + } + } + if incidentFilterMatch && filter.IncidentSlot2 != nil { + slotMatch := false + for _, pkm := range filter.IncidentSlot2 { + if pkm.Pokemon == incident.Slot2PokemonId && (pkm.Form == nil || *pkm.Form == incident.Slot2PokemonForm) { + slotMatch = true + break + } + } + if !slotMatch { + incidentFilterMatch = false + } + } + if incidentFilterMatch && filter.IncidentSlot3 != nil { + slotMatch := false + for _, pkm := range filter.IncidentSlot3 { + if pkm.Pokemon == incident.Slot3PokemonId && (pkm.Form == nil || *pkm.Form == incident.Slot3PokemonForm) { + slotMatch = true + break + } + } + if !slotMatch { + incidentFilterMatch = false + } + } + if incidentFilterMatch { + incidentMatch = true + break + } + } + if !incidentMatch { + return false + } + } + } + return true +} + +func internalGetForts(fortType FortType, retrieveParameters ApiFortScan) ([]string, int, int, int) { + start := time.Now() + + minLocation := retrieveParameters.Min + maxLocation := retrieveParameters.Max + + maxForts := config.Config.Tuning.MaxPokemonResults + if retrieveParameters.Limit > 0 && retrieveParameters.Limit < maxForts { + maxForts = retrieveParameters.Limit + } + + fortsExamined := 0 + fortsSkipped := 0 + + fortTreeMutex.RLock() + fortTreeCopy := fortTree.Copy() + fortTreeMutex.RUnlock() + + lockedTime := time.Since(start) + + var returnKeys []string + + fortTreeCopy.Search([2]float64{minLocation.Longitude, minLocation.Latitude}, [2]float64{maxLocation.Longitude, maxLocation.Latitude}, + func(min, max [2]float64, fortId string) bool { + fortsExamined++ + + fortLookup, found := fortLookupCache.Load(fortId) + if !found { + fortsSkipped++ + return true + } + + matched := false + if len(retrieveParameters.DnfFilters) == 0 { + matched = fortType == fortLookup.FortType + } else { + for i := 0; i < len(retrieveParameters.DnfFilters); i++ { + if isFortDnfMatch(fortType, &fortLookup, &retrieveParameters.DnfFilters[i]) { + matched = true + break + } + } + } + + if matched { + returnKeys = append(returnKeys, fortId) + if len(returnKeys) >= maxForts { + log.Infof("GetFortsInArea - result would exceed maximum size (%d), stopping scan", maxForts) + return false + } + } + + return true + }) + + log.Infof("GetFortsInArea - scan time %s (locked time %s), %d scanned, %d skipped, %d returned, tree size %d", + time.Since(start), lockedTime, fortsExamined, fortsSkipped, len(returnKeys), fortTreeCopy.Len()) + + return returnKeys, fortsExamined, fortsSkipped, fortTreeCopy.Len() +} + +func GymScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiGymScanResult { + returnKeys, examined, skipped, total := internalGetForts(GYM, retrieveParameters) + results := make([]*Gym, 0, len(returnKeys)) + start := time.Now() + + for _, key := range returnKeys { + gym, unlock, err := GetGymRecordReadOnly(context.Background(), dbDetails, key) + if err == nil && gym != nil { + // Make a copy to avoid holding locks + gymCopy := *gym + results = append(results, &gymCopy) + } + if unlock != nil { + unlock() + } + } + log.Infof("GymScan - result buffer time %s, %d added", time.Since(start), len(results)) + + return &ApiGymScanResult{ + Gyms: results, + Examined: examined, + Skipped: skipped, + Total: total, + } +} + +func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiPokestopScanResult { + returnKeys, examined, skipped, total := internalGetForts(POKESTOP, retrieveParameters) + results := make([]*Pokestop, 0, len(returnKeys)) + start := time.Now() + + for _, key := range returnKeys { + pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key) + if err == nil && pokestop != nil { + // Make a copy to avoid holding locks + pokestopCopy := *pokestop + results = append(results, &pokestopCopy) + } + if unlock != nil { + unlock() + } + } + log.Infof("PokestopScan - result buffer time %s, %d added", time.Since(start), len(results)) + + return &ApiPokestopScanResult{ + Pokestops: results, + Examined: examined, + Skipped: skipped, + Total: total, + } +} diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index f914d73c..0c17f6cd 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -1,141 +1,229 @@ package decoder import ( - "context" + "encoding/json" + "sync" + + "github.com/guregu/null/v6" + "github.com/puzpuzpuz/xsync/v3" log "github.com/sirupsen/logrus" "github.com/tidwall/rtree" - "golbat/db" - "sync" ) +type IncidentLookup struct { + DisplayType int8 + Style int8 + Character int16 + Slot1PokemonId int16 + Slot1PokemonForm int16 + Slot2PokemonId int16 + Slot2PokemonForm int16 + Slot3PokemonId int16 + Slot3PokemonForm int16 +} + type FortLookup struct { - IsGym bool - Lure int16 + FortType FortType + PowerUpLevel int8 + IsArScanEligible bool + Lat float64 + Lon float64 + + // Gym-specific fields + AvailableSlots int8 + TeamId int8 + InBattle bool RaidLevel int8 RaidPokemonId int16 - QuestRewardType int16 - QuestRewardId int16 + RaidPokemonForm int16 + + // Pokestop-specific fields + LureId int16 + QuestNoArRewardType int16 + QuestNoArRewardAmount int16 + QuestNoArRewardItemId int16 + QuestNoArRewardPokemonId int16 + QuestNoArRewardPokemonForm int16 + QuestNoArType int16 + QuestNoArTarget int16 + QuestNoArTemplate string + QuestArRewardType int16 + QuestArRewardAmount int16 + QuestArRewardItemId int16 + QuestArRewardPokemonId int16 + QuestArRewardPokemonForm int16 + QuestArType int16 + QuestArTarget int16 + QuestArTemplate string + Incidents []*IncidentLookup + ContestPokemonId int16 + ContestPokemonForm int16 + ContestPokemonType1 int8 + ContestPokemonType2 int8 + ContestRankingStandard int8 + ContestTotalEntries int16 } -var fortLookupCache map[string]FortLookup +var fortLookupCache *xsync.MapOf[string, FortLookup] var fortTreeMutex sync.RWMutex var fortTree rtree.RTreeG[string] func initFortRtree() { - fortLookupCache = make(map[string]FortLookup) + fortLookupCache = xsync.NewMapOf[string, FortLookup]() } type IdRecord struct { Id string `db:"id"` } -func LoadAllPokestops(details db.DbDetails) { - var place IdRecord - rows, err := details.GeneralDb.Queryx("SELECT id FROM pokestop") - count := 0 - if err != nil { - log.Errorf("FortRTree: Load Pokestops %s", err) +// genericUpdateFort handles rtree updates for fort location changes and deletions +func genericUpdateFort(id string, lat float64, lon float64, deleted bool) { + oldFort, inMap := fortLookupCache.Load(id) + + if deleted { + if inMap { + fortLookupCache.Delete(id) + removeFortFromTree(id, oldFort.Lat, oldFort.Lon) + } return } - for rows.Next() { - if count%1000 == 0 { - log.Infof("Loaded %d pokestops", count) - } - count++ - err := rows.StructScan(&place) - if err != nil { - log.Fatalln(err) - } - _, unlock, _ := getPokestopRecordReadOnly(context.Background(), details, place.Id) - if unlock != nil { - unlock() - } + + if !inMap { + addFortToTree(id, lat, lon) + } else if lat != oldFort.Lat || lon != oldFort.Lon { + removeFortFromTree(id, oldFort.Lat, oldFort.Lon) + addFortToTree(id, lat, lon) } - log.Infof("Loaded %d pokestops [finished]", count) } -func LoadAllGyms(details db.DbDetails) { - var place IdRecord - rows, err := details.GeneralDb.Queryx("SELECT id FROM gym") - count := 0 - if err != nil { - log.Errorf("FortRTree: Load Gyms %s", err) - return - } - for rows.Next() { - if count%1000 == 0 { - log.Infof("Loaded %d gyms", count) - } - count++ - err := rows.StructScan(&place) - if err != nil { - log.Fatalln(err) - } - _, unlock, _ := GetGymRecordReadOnly(context.Background(), details, place.Id) - if unlock != nil { - unlock() - } - } - log.Infof("Loaded %d gyms [finished]", count) +// fortRtreeUpdatePokestopOnSave updates rtree and lookup cache when a pokestop is saved +func fortRtreeUpdatePokestopOnSave(pokestop *Pokestop) { + genericUpdateFort(pokestop.Id, pokestop.Lat, pokestop.Lon, pokestop.Deleted) + updatePokestopLookup(pokestop) +} + +// fortRtreeUpdateGymOnSave updates rtree and lookup cache when a gym is saved +func fortRtreeUpdateGymOnSave(gym *Gym) { + genericUpdateFort(gym.Id, gym.Lat, gym.Lon, gym.Deleted) + updateGymLookup(gym) } +// fortRtreeUpdatePokestopOnGet updates rtree when a pokestop is loaded from DB (legacy pattern) func fortRtreeUpdatePokestopOnGet(pokestop *Pokestop) { - fortTreeMutex.RLock() - _, inMap := fortLookupCache[pokestop.Id] - fortTreeMutex.RUnlock() + _, inMap := fortLookupCache.Load(pokestop.Id) if !inMap { - addPokestopToTree(pokestop) - // assumes lat,lon unchanged since ejected from cache, so do not add to rtree + addFortToTree(pokestop.Id, pokestop.Lat, pokestop.Lon) updatePokestopLookup(pokestop) } } +// fortRtreeUpdateGymOnGet updates rtree when a gym is loaded from DB (legacy pattern) func fortRtreeUpdateGymOnGet(gym *Gym) { - fortTreeMutex.RLock() - _, inMap := fortLookupCache[gym.Id] - fortTreeMutex.RUnlock() + _, inMap := fortLookupCache.Load(gym.Id) if !inMap { - addGymToTree(gym) - // assumes lat,lon unchanged since ejected from cache, so do not add to rtree + addFortToTree(gym.Id, gym.Lat, gym.Lon) updateGymLookup(gym) } } -func updatePokestopLookup(pokestop *Pokestop) { - fortTreeMutex.Lock() - fortLookupCache[pokestop.Id] = FortLookup{ - IsGym: false, - Lure: pokestop.LureId, - // RaidLevel: pokestop.RaidLevel, - // RaidPokemonId: pokestop.RaidPokemonId, - // QuestRewardType: pokestop.QuestRewardType, - // QuestRewardId: pokestop.QuestRewardId, +// getContestTotalEntries parses showcase rankings JSON to get total entries +func getContestTotalEntries(rankingsString null.String) int16 { + if !rankingsString.Valid { + return -1 } - fortTreeMutex.Unlock() + + type contestJson struct { + TotalEntries int `json:"total_entries"` + } + var cj contestJson + if json.Unmarshal([]byte(rankingsString.String), &cj) == nil { + return int16(cj.TotalEntries) + } + return -1 +} + +func updatePokestopLookup(pokestop *Pokestop) { + contestTotalEntries := getContestTotalEntries(pokestop.ShowcaseRankings) + + fortLookupCache.Store(pokestop.Id, FortLookup{ + FortType: POKESTOP, + PowerUpLevel: int8(valueOrMinus1(pokestop.PowerUpLevel)), + IsArScanEligible: pokestop.ArScanEligible.ValueOrZero() == 1, + Lat: pokestop.Lat, + Lon: pokestop.Lon, + LureId: pokestop.LureId, + QuestNoArRewardType: int16(pokestop.QuestRewardType.ValueOrZero()), + QuestNoArRewardAmount: int16(pokestop.QuestRewardAmount.ValueOrZero()), + QuestNoArRewardItemId: int16(pokestop.QuestItemId.ValueOrZero()), + QuestNoArRewardPokemonId: int16(pokestop.QuestPokemonId.ValueOrZero()), + QuestNoArRewardPokemonForm: int16(pokestop.QuestPokemonFormId.ValueOrZero()), + QuestNoArType: int16(valueOrMinus1(pokestop.QuestType)), + QuestNoArTarget: int16(valueOrMinus1(pokestop.QuestTarget)), + QuestNoArTemplate: pokestop.QuestTemplate.ValueOrZero(), + QuestArRewardType: int16(pokestop.AlternativeQuestRewardType.ValueOrZero()), + QuestArRewardAmount: int16(pokestop.AlternativeQuestRewardAmount.ValueOrZero()), + QuestArRewardItemId: int16(pokestop.AlternativeQuestItemId.ValueOrZero()), + QuestArRewardPokemonId: int16(pokestop.AlternativeQuestPokemonId.ValueOrZero()), + QuestArRewardPokemonForm: int16(pokestop.AlternativeQuestPokemonFormId.ValueOrZero()), + QuestArType: int16(valueOrMinus1(pokestop.AlternativeQuestType)), + QuestArTarget: int16(valueOrMinus1(pokestop.AlternativeQuestTarget)), + QuestArTemplate: pokestop.AlternativeQuestTemplate.ValueOrZero(), + ContestPokemonId: int16(pokestop.ShowcasePokemon.ValueOrZero()), + ContestPokemonForm: int16(pokestop.ShowcasePokemonForm.ValueOrZero()), + ContestPokemonType1: int8(pokestop.ShowcasePokemonType.ValueOrZero()), + ContestPokemonType2: -1, // TODO: this should probably be saved in the db + ContestRankingStandard: int8(pokestop.ShowcaseRankingStandard.ValueOrZero()), + ContestTotalEntries: contestTotalEntries, + }) } func updateGymLookup(gym *Gym) { + fortLookupCache.Store(gym.Id, FortLookup{ + FortType: GYM, + PowerUpLevel: int8(valueOrMinus1(gym.PowerUpLevel)), + Lat: gym.Lat, + Lon: gym.Lon, + IsArScanEligible: gym.ArScanEligible.ValueOrZero() == 1, + AvailableSlots: int8(gym.AvailableSlots.ValueOrZero()), + TeamId: int8(gym.TeamId.ValueOrZero()), + InBattle: gym.InBattle.ValueOrZero() == 1, + RaidLevel: int8(gym.RaidLevel.ValueOrZero()), + RaidPokemonId: int16(gym.RaidPokemonId.ValueOrZero()), + RaidPokemonForm: int16(gym.RaidPokemonForm.ValueOrZero()), + }) +} + +func addFortToTree(id string, lat float64, lon float64) { fortTreeMutex.Lock() - fortLookupCache[gym.Id] = FortLookup{ - IsGym: true, - RaidLevel: int8(gym.RaidLevel.ValueOrZero()), - RaidPokemonId: int16(gym.RaidPokemonId.ValueOrZero()), - } + fortTree.Insert([2]float64{lon, lat}, [2]float64{lon, lat}, id) fortTreeMutex.Unlock() } -func addPokestopToTree(pokestop *Pokestop) { - log.Infof("FortRtree - add pokestop %s, lat %f lon %f", pokestop.Id, pokestop.Lat, pokestop.Lon) - +func removeFortFromTree(fortId string, lat, lon float64) { fortTreeMutex.Lock() - fortTree.Insert([2]float64{pokestop.Lon, pokestop.Lat}, [2]float64{pokestop.Lon, pokestop.Lat}, pokestop.Id) + beforeLen := fortTree.Len() + fortTree.Delete([2]float64{lon, lat}, [2]float64{lon, lat}, fortId) + afterLen := fortTree.Len() fortTreeMutex.Unlock() + + if beforeLen != afterLen+1 { + log.Debugf("FortRtree - removing %s, lat %f lon %f size %d->%d", fortId, lat, lon, beforeLen, afterLen) + } } -func addGymToTree(gym *Gym) { - log.Infof("FortRtree - add gym %s, lat %f lon %f", gym.Id, gym.Lat, gym.Lon) +// GetFortLookup returns the FortLookup for the given fort ID, if it exists +func GetFortLookup(fortId string) (FortLookup, bool) { + return fortLookupCache.Load(fortId) +} - fortTreeMutex.Lock() - fortTree.Insert([2]float64{gym.Lon, gym.Lat}, [2]float64{gym.Lon, gym.Lat}, gym.Id) - fortTreeMutex.Unlock() +// GetFortsInBounds returns all fort IDs within the given bounding box +func GetFortsInBounds(minLat, minLon, maxLat, maxLon float64) []string { + var results []string + fortTreeMutex.RLock() + fortTree.Search([2]float64{minLon, minLat}, [2]float64{maxLon, maxLat}, func(min, max [2]float64, data string) bool { + results = append(results, data) + return true + }) + fortTreeMutex.RUnlock() + return results } diff --git a/decoder/fort_preload.go b/decoder/fort_preload.go new file mode 100644 index 00000000..1e134e5b --- /dev/null +++ b/decoder/fort_preload.go @@ -0,0 +1,158 @@ +package decoder + +import ( + "runtime" + "sync" + "sync/atomic" + "time" + + log "github.com/sirupsen/logrus" + + "golbat/db" +) + +// PreloadForts loads all forts from DB into cache. +// If populateRtree is true, also builds the rtree index. +// Fort tracker is always populated during preload. +func PreloadForts(dbDetails db.DbDetails, populateRtree bool) error { + startTime := time.Now() + + var wg sync.WaitGroup + var pokestopCount, gymCount int32 + + wg.Add(2) + go func() { + defer wg.Done() + pokestopCount = preloadPokestops(dbDetails, populateRtree) + }() + go func() { + defer wg.Done() + gymCount = preloadGyms(dbDetails, populateRtree) + }() + wg.Wait() + + log.Infof("PreloadForts: loaded %d pokestops and %d gyms in %v (rtree=%v)", + pokestopCount, gymCount, time.Since(startTime), populateRtree) + + return nil +} + +func preloadPokestops(dbDetails db.DbDetails, populateRtree bool) int32 { + query := "SELECT " + pokestopSelectColumns + " FROM pokestop WHERE deleted = 0" + rows, err := dbDetails.GeneralDb.Queryx(query) + if err != nil { + log.Errorf("PreloadForts: failed to query pokestops - %s", err) + return 0 + } + defer rows.Close() + + numWorkers := runtime.NumCPU() + jobs := make(chan *Pokestop, 100) + var wg sync.WaitGroup + var count int32 + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for pokestop := range jobs { + // Add to cache + pokestopCache.Set(pokestop.Id, pokestop, 0) // 0 = use default TTL + + // Update rtree if enabled + if populateRtree { + fortRtreeUpdatePokestopOnSave(pokestop) + } + + // Register with fort tracker + if fortTracker != nil && pokestop.CellId.Valid { + fortTracker.RegisterFort( + pokestop.Id, + uint64(pokestop.CellId.Int64), + false, + pokestop.Updated*1000, // convert to milliseconds + ) + } + + c := atomic.AddInt32(&count, 1) + if c%10000 == 0 { + log.Infof("PreloadForts: loaded %d pokestops...", c) + } + } + }() + } + + for rows.Next() { + var pokestop Pokestop + err := rows.StructScan(&pokestop) + if err != nil { + log.Errorf("PreloadForts: pokestop scan error - %s", err) + continue + } + jobs <- &pokestop + } + close(jobs) + wg.Wait() + + return count +} + +func preloadGyms(dbDetails db.DbDetails, populateRtree bool) int32 { + query := "SELECT " + gymSelectColumns + " FROM gym WHERE deleted = 0" + rows, err := dbDetails.GeneralDb.Queryx(query) + if err != nil { + log.Errorf("PreloadForts: failed to query gyms - %s", err) + return 0 + } + defer rows.Close() + + numWorkers := runtime.NumCPU() + jobs := make(chan *Gym, 100) + var wg sync.WaitGroup + var count int32 + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for gym := range jobs { + // Add to cache + gymCache.Set(gym.Id, gym, 0) // 0 = use default TTL + + // Update rtree if enabled + if populateRtree { + fortRtreeUpdateGymOnSave(gym) + } + + // Register with fort tracker + if fortTracker != nil && gym.CellId.Valid { + fortTracker.RegisterFort( + gym.Id, + uint64(gym.CellId.Int64), + true, + gym.Updated*1000, // convert to milliseconds + ) + } + + c := atomic.AddInt32(&count, 1) + if c%10000 == 0 { + log.Infof("PreloadForts: loaded %d gyms...", c) + } + } + }() + } + + for rows.Next() { + var gym Gym + err := rows.StructScan(&gym) + if err != nil { + log.Errorf("PreloadForts: gym scan error - %s", err) + continue + } + jobs <- &gym + } + close(jobs) + wg.Wait() + + return count +} diff --git a/decoder/fort_tracker.go b/decoder/fort_tracker.go index b3a2ae0d..f543fbc5 100644 --- a/decoder/fort_tracker.go +++ b/decoder/fort_tracker.go @@ -199,6 +199,27 @@ func (ft *FortTracker) getOrCreateCellLocked(cellId uint64) *FortTrackerCellStat return cell } +// RegisterFort registers a fort during bulk loading (e.g., from preload). +// Unlike UpdateFort, this uses the provided timestamp rather than "now". +func (ft *FortTracker) RegisterFort(fortId string, cellId uint64, isGym bool, updatedTimestamp int64) { + ft.mu.Lock() + defer ft.mu.Unlock() + + cell := ft.getOrCreateCellLocked(cellId) + + if isGym { + cell.gyms[fortId] = struct{}{} + } else { + cell.pokestops[fortId] = struct{}{} + } + + ft.forts[fortId] = &FortTrackerLastSeen{ + cellId: cellId, + lastSeen: updatedTimestamp, + isGym: isGym, + } +} + // UpdateFort updates tracking for a single fort seen in GMO func (ft *FortTracker) UpdateFort(fortId string, cellId uint64, isGym bool, now int64) { ft.mu.Lock() diff --git a/decoder/gym_state.go b/decoder/gym_state.go index e646ac1a..cee377cc 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -16,8 +16,18 @@ import ( log "github.com/sirupsen/logrus" ) +// gymSelectColumns defines the columns for gym queries. +// Used by both single-row and bulk load queries to keep them in sync. +const gymSelectColumns = `id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, + raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, + available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, + raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, + cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, + raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, + power_up_end_timestamp, description, defenders, rsvps` + func loadGymFromDatabase(ctx context.Context, db db.DbDetails, fortId string, gym *Gym) error { - err := db.GeneralDb.GetContext(ctx, gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) + err := db.GeneralDb.GetContext(ctx, gym, "SELECT "+gymSelectColumns+" FROM gym WHERE id = ?", fortId) statsCollector.IncDbQuery("select gym", err) return err } @@ -58,7 +68,7 @@ func GetGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) ( // we'll get their Gym and use that instead (ensuring same mutex) existingGym, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { // Only called if key doesn't exist - our Pokestop wins - if config.Config.TestFortInMemory { + if config.Config.FortInMemory { fortRtreeUpdateGymOnGet(&dbGym) } return &dbGym @@ -104,7 +114,7 @@ func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) ( // We loaded from DB gym.newRecord = false gym.ClearDirty() - if config.Config.TestFortInMemory { + if config.Config.FortInMemory { fortRtreeUpdateGymOnGet(gym) } } @@ -404,7 +414,10 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { } } - //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + if config.Config.FortInMemory { + fortRtreeUpdateGymOnSave(gym) + } + areas := MatchStatsGeofence(gym.Lat, gym.Lon) createGymWebhooks(gym, areas) createGymFortWebhooks(gym) diff --git a/decoder/main.go b/decoder/main.go index 3fcb043a..106ce299 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -1,6 +1,7 @@ package decoder import ( + "context" "fmt" "math" "runtime" @@ -94,10 +95,22 @@ func initDataCache() { TTL: 60 * time.Minute, KeyToShard: StringKeyToShard, }) + if config.Config.FortInMemory { + pokestopCache.OnEviction(func(ctx context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *Pokestop]) { + p := item.Value() + removeFortFromTree(p.Id, p.Lat, p.Lon) + }) + } gymCache = ttlcache.New[string, *Gym]( ttlcache.WithTTL[string, *Gym](60 * time.Minute), ) + if config.Config.FortInMemory { + gymCache.OnEviction(func(ctx context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *Gym]) { + g := item.Value() + removeFortFromTree(g.Id, g.Lat, g.Lon) + }) + } go gymCache.Start() stationCache = ttlcache.New[string, *Station]( diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 20d771b7..e77f75e0 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -11,49 +11,59 @@ import ( type Pokestop struct { mu sync.Mutex `db:"-"` // Object-level mutex - Id string `db:"id"` - Lat float64 `db:"lat"` - Lon float64 `db:"lon"` - Name null.String `db:"name"` - Url null.String `db:"url"` - LureExpireTimestamp null.Int `db:"lure_expire_timestamp"` - LastModifiedTimestamp null.Int `db:"last_modified_timestamp"` - Updated int64 `db:"updated"` - Enabled null.Bool `db:"enabled"` - QuestType null.Int `db:"quest_type"` - QuestTimestamp null.Int `db:"quest_timestamp"` - QuestTarget null.Int `db:"quest_target"` - QuestConditions null.String `db:"quest_conditions"` - QuestRewards null.String `db:"quest_rewards"` - QuestTemplate null.String `db:"quest_template"` - QuestTitle null.String `db:"quest_title"` - QuestExpiry null.Int `db:"quest_expiry"` - CellId null.Int `db:"cell_id"` - Deleted bool `db:"deleted"` - LureId int16 `db:"lure_id"` - FirstSeenTimestamp int16 `db:"first_seen_timestamp"` - SponsorId null.Int `db:"sponsor_id"` - PartnerId null.String `db:"partner_id"` - ArScanEligible null.Int `db:"ar_scan_eligible"` // is an 8 - PowerUpLevel null.Int `db:"power_up_level"` - PowerUpPoints null.Int `db:"power_up_points"` - PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp"` - AlternativeQuestType null.Int `db:"alternative_quest_type"` - AlternativeQuestTimestamp null.Int `db:"alternative_quest_timestamp"` - AlternativeQuestTarget null.Int `db:"alternative_quest_target"` - AlternativeQuestConditions null.String `db:"alternative_quest_conditions"` - AlternativeQuestRewards null.String `db:"alternative_quest_rewards"` - AlternativeQuestTemplate null.String `db:"alternative_quest_template"` - AlternativeQuestTitle null.String `db:"alternative_quest_title"` - AlternativeQuestExpiry null.Int `db:"alternative_quest_expiry"` - Description null.String `db:"description"` - ShowcaseFocus null.String `db:"showcase_focus"` - ShowcasePokemon null.Int `db:"showcase_pokemon_id"` - ShowcasePokemonForm null.Int `db:"showcase_pokemon_form_id"` - ShowcasePokemonType null.Int `db:"showcase_pokemon_type_id"` - ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard"` - ShowcaseExpiry null.Int `db:"showcase_expiry"` - ShowcaseRankings null.String `db:"showcase_rankings"` + Id string `db:"id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + Name null.String `db:"name"` + Url null.String `db:"url"` + LureExpireTimestamp null.Int `db:"lure_expire_timestamp"` + LastModifiedTimestamp null.Int `db:"last_modified_timestamp"` + Updated int64 `db:"updated"` + Enabled null.Bool `db:"enabled"` + QuestType null.Int `db:"quest_type"` + QuestTimestamp null.Int `db:"quest_timestamp"` + QuestTarget null.Int `db:"quest_target"` + QuestConditions null.String `db:"quest_conditions"` + QuestRewards null.String `db:"quest_rewards"` + QuestTemplate null.String `db:"quest_template"` + QuestTitle null.String `db:"quest_title"` + QuestExpiry null.Int `db:"quest_expiry"` + QuestRewardType null.Int `db:"quest_reward_type"` + QuestItemId null.Int `db:"quest_item_id"` + QuestRewardAmount null.Int `db:"quest_reward_amount"` + QuestPokemonId null.Int `db:"quest_pokemon_id"` + QuestPokemonFormId null.Int `db:"quest_pokemon_form_id"` + CellId null.Int `db:"cell_id"` + Deleted bool `db:"deleted"` + LureId int16 `db:"lure_id"` + FirstSeenTimestamp int16 `db:"first_seen_timestamp"` + SponsorId null.Int `db:"sponsor_id"` + PartnerId null.String `db:"partner_id"` + ArScanEligible null.Int `db:"ar_scan_eligible"` // is an 8 + PowerUpLevel null.Int `db:"power_up_level"` + PowerUpPoints null.Int `db:"power_up_points"` + PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp"` + AlternativeQuestType null.Int `db:"alternative_quest_type"` + AlternativeQuestTimestamp null.Int `db:"alternative_quest_timestamp"` + AlternativeQuestTarget null.Int `db:"alternative_quest_target"` + AlternativeQuestConditions null.String `db:"alternative_quest_conditions"` + AlternativeQuestRewards null.String `db:"alternative_quest_rewards"` + AlternativeQuestTemplate null.String `db:"alternative_quest_template"` + AlternativeQuestTitle null.String `db:"alternative_quest_title"` + AlternativeQuestExpiry null.Int `db:"alternative_quest_expiry"` + AlternativeQuestRewardType null.Int `db:"alternative_quest_reward_type"` + AlternativeQuestItemId null.Int `db:"alternative_quest_item_id"` + AlternativeQuestRewardAmount null.Int `db:"alternative_quest_reward_amount"` + AlternativeQuestPokemonId null.Int `db:"alternative_quest_pokemon_id"` + AlternativeQuestPokemonFormId null.Int `db:"alternative_quest_pokemon_form_id"` + Description null.String `db:"description"` + ShowcaseFocus null.String `db:"showcase_focus"` + ShowcasePokemon null.Int `db:"showcase_pokemon_id"` + ShowcasePokemonForm null.Int `db:"showcase_pokemon_form_id"` + ShowcasePokemonType null.Int `db:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `db:"showcase_expiry"` + ShowcaseRankings null.String `db:"showcase_rankings"` dirty bool `db:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-"` // Not persisted - tracks if this is a new record @@ -318,6 +328,56 @@ func (p *Pokestop) SetQuestExpiry(v null.Int) { } } +func (p *Pokestop) SetQuestRewardType(v null.Int) { + if p.QuestRewardType != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestRewardType:%s->%s", FormatNull(p.QuestRewardType), FormatNull(v))) + } + p.QuestRewardType = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestItemId(v null.Int) { + if p.QuestItemId != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestItemId:%s->%s", FormatNull(p.QuestItemId), FormatNull(v))) + } + p.QuestItemId = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestRewardAmount(v null.Int) { + if p.QuestRewardAmount != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestRewardAmount:%s->%s", FormatNull(p.QuestRewardAmount), FormatNull(v))) + } + p.QuestRewardAmount = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestPokemonId(v null.Int) { + if p.QuestPokemonId != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestPokemonId:%s->%s", FormatNull(p.QuestPokemonId), FormatNull(v))) + } + p.QuestPokemonId = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestPokemonFormId(v null.Int) { + if p.QuestPokemonFormId != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestPokemonFormId:%s->%s", FormatNull(p.QuestPokemonFormId), FormatNull(v))) + } + p.QuestPokemonFormId = v + p.dirty = true + } +} + func (p *Pokestop) SetCellId(v null.Int) { if p.CellId != v { if dbDebugEnabled { @@ -498,6 +558,56 @@ func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { } } +func (p *Pokestop) SetAlternativeQuestRewardType(v null.Int) { + if p.AlternativeQuestRewardType != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestRewardType:%s->%s", FormatNull(p.AlternativeQuestRewardType), FormatNull(v))) + } + p.AlternativeQuestRewardType = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestItemId(v null.Int) { + if p.AlternativeQuestItemId != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestItemId:%s->%s", FormatNull(p.AlternativeQuestItemId), FormatNull(v))) + } + p.AlternativeQuestItemId = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestRewardAmount(v null.Int) { + if p.AlternativeQuestRewardAmount != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestRewardAmount:%s->%s", FormatNull(p.AlternativeQuestRewardAmount), FormatNull(v))) + } + p.AlternativeQuestRewardAmount = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestPokemonId(v null.Int) { + if p.AlternativeQuestPokemonId != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestPokemonId:%s->%s", FormatNull(p.AlternativeQuestPokemonId), FormatNull(v))) + } + p.AlternativeQuestPokemonId = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestPokemonFormId(v null.Int) { + if p.AlternativeQuestPokemonFormId != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestPokemonFormId:%s->%s", FormatNull(p.AlternativeQuestPokemonFormId), FormatNull(v))) + } + p.AlternativeQuestPokemonFormId = v + p.dirty = true + } +} + func (p *Pokestop) SetDescription(v null.String) { if p.Description != v { if dbDebugEnabled { diff --git a/decoder/pokestop_decode.go b/decoder/pokestop_decode.go index 87941c73..86746bd9 100644 --- a/decoder/pokestop_decode.go +++ b/decoder/pokestop_decode.go @@ -238,34 +238,68 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu conditions = append(conditions, condition) } - for _, rewardData := range questData.QuestRewards { + // Extract first reward details for indexed columns + var rewardType, rewardAmount, rewardItemId, rewardPokemonId, rewardPokemonFormId null.Int + + for i, rewardData := range questData.QuestRewards { reward := make(map[string]any) infoData := make(map[string]any) reward["type"] = int(rewardData.Type) + + // For the first reward, also populate the indexed column values + isFirst := i == 0 + if isFirst { + rewardType = null.IntFrom(int64(rewardData.Type)) + } + switch rewardData.Type { case pogo.QuestRewardProto_EXPERIENCE: infoData["amount"] = rewardData.GetExp() + if isFirst { + rewardAmount = null.IntFrom(int64(rewardData.GetExp())) + } case pogo.QuestRewardProto_ITEM: info := rewardData.GetItem() infoData["amount"] = info.Amount infoData["item_id"] = int(info.Item) + if isFirst { + rewardAmount = null.IntFrom(int64(info.Amount)) + rewardItemId = null.IntFrom(int64(info.Item)) + } case pogo.QuestRewardProto_STARDUST: infoData["amount"] = rewardData.GetStardust() + if isFirst { + rewardAmount = null.IntFrom(int64(rewardData.GetStardust())) + } case pogo.QuestRewardProto_CANDY: info := rewardData.GetCandy() infoData["amount"] = info.Amount infoData["pokemon_id"] = int(info.PokemonId) + if isFirst { + rewardAmount = null.IntFrom(int64(info.Amount)) + rewardPokemonId = null.IntFrom(int64(info.PokemonId)) + } case pogo.QuestRewardProto_XL_CANDY: info := rewardData.GetXlCandy() infoData["amount"] = info.Amount infoData["pokemon_id"] = int(info.PokemonId) + if isFirst { + rewardAmount = null.IntFrom(int64(info.Amount)) + rewardPokemonId = null.IntFrom(int64(info.PokemonId)) + } case pogo.QuestRewardProto_POKEMON_ENCOUNTER: info := rewardData.GetPokemonEncounter() if info.IsHiddenDitto { infoData["pokemon_id"] = 132 infoData["pokemon_id_display"] = int(info.GetPokemonId()) + if isFirst { + rewardPokemonId = null.IntFrom(132) + } } else { infoData["pokemon_id"] = int(info.GetPokemonId()) + if isFirst { + rewardPokemonId = null.IntFrom(int64(info.GetPokemonId())) + } } if info.ShinyProbability > 0.0 { infoData["shiny_probability"] = info.ShinyProbability @@ -276,6 +310,9 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu } if formId := int(display.Form); formId != 0 { infoData["form_id"] = formId + if isFirst { + rewardPokemonFormId = null.IntFrom(int64(formId)) + } } if genderId := int(display.Gender); genderId != 0 { infoData["gender_id"] = genderId @@ -289,19 +326,27 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu if breadMode := int(display.BreadModeEnum); breadMode != 0 { infoData["bread_mode"] = breadMode } - } else { - } case pogo.QuestRewardProto_POKECOIN: infoData["amount"] = rewardData.GetPokecoin() + if isFirst { + rewardAmount = null.IntFrom(int64(rewardData.GetPokecoin())) + } case pogo.QuestRewardProto_STICKER: info := rewardData.GetSticker() infoData["amount"] = info.Amount infoData["sticker_id"] = info.StickerId + if isFirst { + rewardAmount = null.IntFrom(int64(info.Amount)) + } case pogo.QuestRewardProto_MEGA_RESOURCE: info := rewardData.GetMegaResource() infoData["amount"] = info.Amount infoData["pokemon_id"] = int(info.PokemonId) + if isFirst { + rewardAmount = null.IntFrom(int64(info.Amount)) + rewardPokemonId = null.IntFrom(int64(info.PokemonId)) + } case pogo.QuestRewardProto_AVATAR_CLOTHING: case pogo.QuestRewardProto_QUEST: case pogo.QuestRewardProto_LEVEL_CAP: @@ -309,7 +354,6 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu case pogo.QuestRewardProto_PLAYER_ATTRIBUTE: default: break - } reward["info"] = infoData rewards = append(rewards, reward) @@ -347,6 +391,11 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu stop.SetAlternativeQuestRewards(null.StringFrom(string(questRewards))) stop.SetAlternativeQuestTimestamp(null.IntFrom(questTimestamp)) stop.SetAlternativeQuestExpiry(questExpiry) + stop.SetAlternativeQuestRewardType(rewardType) + stop.SetAlternativeQuestItemId(rewardItemId) + stop.SetAlternativeQuestRewardAmount(rewardAmount) + stop.SetAlternativeQuestPokemonId(rewardPokemonId) + stop.SetAlternativeQuestPokemonFormId(rewardPokemonFormId) } else { stop.SetQuestType(null.IntFrom(questType)) stop.SetQuestTarget(null.IntFrom(questTarget)) @@ -356,6 +405,11 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu stop.SetQuestRewards(null.StringFrom(string(questRewards))) stop.SetQuestTimestamp(null.IntFrom(questTimestamp)) stop.SetQuestExpiry(questExpiry) + stop.SetQuestRewardType(rewardType) + stop.SetQuestItemId(rewardItemId) + stop.SetQuestRewardAmount(rewardAmount) + stop.SetQuestPokemonId(rewardPokemonId) + stop.SetQuestPokemonFormId(rewardPokemonFormId) } return questTitle diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index 9fced113..b39e4d0c 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -16,19 +16,25 @@ import ( "golbat/webhooks" ) +// pokestopSelectColumns defines the columns for pokestop queries. +// Used by both single-row and bulk load queries to keep them in sync. +const pokestopSelectColumns = `id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, + updated, quest_type, quest_timestamp, quest_target, quest_conditions, + quest_rewards, quest_template, quest_title, quest_expiry, + quest_reward_type, quest_item_id, quest_reward_amount, quest_pokemon_id, quest_pokemon_form_id, + alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, + alternative_quest_conditions, alternative_quest_rewards, + alternative_quest_template, alternative_quest_title, alternative_quest_expiry, + alternative_quest_reward_type, alternative_quest_item_id, alternative_quest_reward_amount, + alternative_quest_pokemon_id, alternative_quest_pokemon_form_id, + cell_id, deleted, lure_id, sponsor_id, partner_id, + ar_scan_eligible, power_up_points, power_up_level, power_up_end_timestamp, + description, showcase_pokemon_id, showcase_pokemon_form_id, + showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings` + func loadPokestopFromDatabase(ctx context.Context, db db.DbDetails, fortId string, pokestop *Pokestop) error { err := db.GeneralDb.GetContext(ctx, pokestop, - `SELECT pokestop.id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, - pokestop.updated, quest_type, quest_timestamp, quest_target, quest_conditions, - quest_rewards, quest_template, quest_title, - alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, - alternative_quest_conditions, alternative_quest_rewards, - alternative_quest_template, alternative_quest_title, cell_id, deleted, lure_id, sponsor_id, partner_id, - ar_scan_eligible, power_up_points, power_up_level, power_up_end_timestamp, - quest_expiry, alternative_quest_expiry, description, showcase_pokemon_id, showcase_pokemon_form_id, - showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings - FROM pokestop - WHERE pokestop.id = ? `, fortId) + `SELECT `+pokestopSelectColumns+` FROM pokestop WHERE id = ?`, fortId) statsCollector.IncDbQuery("select pokestop", err) return err } @@ -70,7 +76,7 @@ func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId stri // we'll get their Pokestop and use that instead (ensuring same mutex) existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { // Only called if key doesn't exist - our Pokestop wins - if config.Config.TestFortInMemory { + if config.Config.FortInMemory { fortRtreeUpdatePokestopOnGet(&dbPokestop) } return &dbPokestop @@ -116,7 +122,7 @@ func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId stri // We loaded from DB pokestop.newRecord = false pokestop.ClearDirty() - if config.Config.TestFortInMemory { + if config.Config.FortInMemory { fortRtreeUpdatePokestopOnGet(pokestop) } } @@ -286,22 +292,28 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop INSERT INTO pokestop ( id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, quest_timestamp, quest_target, quest_conditions, quest_rewards, quest_template, quest_title, + quest_expiry, quest_reward_type, quest_item_id, quest_reward_amount, quest_pokemon_id, quest_pokemon_form_id, alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, alternative_quest_conditions, alternative_quest_rewards, alternative_quest_template, - alternative_quest_title, cell_id, lure_id, sponsor_id, partner_id, ar_scan_eligible, + alternative_quest_title, alternative_quest_expiry, alternative_quest_reward_type, alternative_quest_item_id, + alternative_quest_reward_amount, alternative_quest_pokemon_id, alternative_quest_pokemon_form_id, + cell_id, lure_id, sponsor_id, partner_id, ar_scan_eligible, power_up_points, power_up_level, power_up_end_timestamp, updated, first_seen_timestamp, - quest_expiry, alternative_quest_expiry, description, showcase_focus, showcase_pokemon_id, + description, showcase_focus, showcase_pokemon_id, showcase_pokemon_form_id, showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings ) VALUES ( :id, :lat, :lon, :name, :url, :enabled, :lure_expire_timestamp, :last_modified_timestamp, :quest_type, :quest_timestamp, :quest_target, :quest_conditions, :quest_rewards, :quest_template, :quest_title, + :quest_expiry, :quest_reward_type, :quest_item_id, :quest_reward_amount, :quest_pokemon_id, :quest_pokemon_form_id, :alternative_quest_type, :alternative_quest_timestamp, :alternative_quest_target, :alternative_quest_conditions, :alternative_quest_rewards, :alternative_quest_template, - :alternative_quest_title, :cell_id, :lure_id, :sponsor_id, :partner_id, :ar_scan_eligible, + :alternative_quest_title, :alternative_quest_expiry, :alternative_quest_reward_type, :alternative_quest_item_id, + :alternative_quest_reward_amount, :alternative_quest_pokemon_id, :alternative_quest_pokemon_form_id, + :cell_id, :lure_id, :sponsor_id, :partner_id, :ar_scan_eligible, :power_up_points, :power_up_level, :power_up_end_timestamp, UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), - :quest_expiry, :alternative_quest_expiry, :description, :showcase_focus, :showcase_pokemon_id, + :description, :showcase_focus, :showcase_pokemon_id, :showcase_pokemon_form_id, :showcase_pokemon_type_id, :showcase_ranking_standard, :showcase_expiry, :showcase_rankings)`, pokestop) @@ -334,6 +346,12 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop quest_rewards = :quest_rewards, quest_template = :quest_template, quest_title = :quest_title, + quest_expiry = :quest_expiry, + quest_reward_type = :quest_reward_type, + quest_item_id = :quest_item_id, + quest_reward_amount = :quest_reward_amount, + quest_pokemon_id = :quest_pokemon_id, + quest_pokemon_form_id = :quest_pokemon_form_id, alternative_quest_type = :alternative_quest_type, alternative_quest_timestamp = :alternative_quest_timestamp, alternative_quest_target = :alternative_quest_target, @@ -341,6 +359,12 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop alternative_quest_rewards = :alternative_quest_rewards, alternative_quest_template = :alternative_quest_template, alternative_quest_title = :alternative_quest_title, + alternative_quest_expiry = :alternative_quest_expiry, + alternative_quest_reward_type = :alternative_quest_reward_type, + alternative_quest_item_id = :alternative_quest_item_id, + alternative_quest_reward_amount = :alternative_quest_reward_amount, + alternative_quest_pokemon_id = :alternative_quest_pokemon_id, + alternative_quest_pokemon_form_id = :alternative_quest_pokemon_form_id, cell_id = :cell_id, lure_id = :lure_id, deleted = :deleted, @@ -350,8 +374,6 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop power_up_points = :power_up_points, power_up_level = :power_up_level, power_up_end_timestamp = :power_up_end_timestamp, - quest_expiry = :quest_expiry, - alternative_quest_expiry = :alternative_quest_expiry, description = :description, showcase_focus = :showcase_focus, showcase_pokemon_id = :showcase_pokemon_id, @@ -371,11 +393,14 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } _ = res } - //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) if dbDebugEnabled { pokestop.changedFields = pokestop.changedFields[:0] } + if config.Config.FortInMemory { + fortRtreeUpdatePokestopOnSave(pokestop) + } + createPokestopWebhooks(pokestop) createPokestopFortWebhooks(pokestop) if pokestop.IsNewRecord() { diff --git a/main.go b/main.go index f838c496..a95b47b0 100644 --- a/main.go +++ b/main.go @@ -256,13 +256,23 @@ func main() { staleThreshold = 3600 // def 1 hour } decoder.InitFortTracker(staleThreshold) - if err := decoder.LoadFortsFromDB(ctx, dbDetails); err != nil { - log.Errorf("failed to load forts into tracker: %s", err) - } - if cfg.TestFortInMemory { - go decoder.LoadAllPokestops(dbDetails) - go decoder.LoadAllGyms(dbDetails) + // Determine fort loading strategy + // FortInMemory enables rtree spatial lookups; PreloadForts just warms the cache + // TestFortInMemory is deprecated but still supported (treated as FortInMemory) + fortInMemory := cfg.FortInMemory + fullPreload := cfg.PreloadForts || fortInMemory + + if fullPreload { + // Full preload: loads into cache, registers with fort tracker, optionally builds rtree + if err := decoder.PreloadForts(dbDetails, fortInMemory); err != nil { + log.Errorf("failed to preload forts: %s", err) + } + } else { + // No preload: fort tracker loads its own minimal data + if err := decoder.LoadFortsFromDB(ctx, dbDetails); err != nil { + log.Errorf("failed to load forts into tracker: %s", err) + } } // Start the GRPC receiver @@ -296,6 +306,8 @@ func main() { apiGroup.GET("/gym/id/:gym_id", GetGym) apiGroup.POST("/gym/query", GetGyms) apiGroup.POST("/gym/search", SearchGyms) + apiGroup.POST("/gym/scan", GymScan) + apiGroup.POST("/pokestop/scan", PokestopScan) apiGroup.POST("/reload-geojson", ReloadGeojson) apiGroup.GET("/reload-geojson", ReloadGeojson) diff --git a/routes.go b/routes.go index 0cf7efff..75719d36 100644 --- a/routes.go +++ b/routes.go @@ -744,6 +744,42 @@ func SearchGyms(c *gin.Context) { c.JSON(http.StatusOK, out) } +// POST /api/gym/scan +// In-memory fort scan with DNF filters (requires fort_in_memory=true) +func GymScan(c *gin.Context) { + if !config.Config.FortInMemory { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "fort_in_memory not enabled"}) + return + } + + var params decoder.ApiFortScan + if err := c.ShouldBindJSON(¶ms); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid JSON body"}) + return + } + + result := decoder.GymScanEndpoint(params, dbDetails) + c.JSON(http.StatusOK, result) +} + +// POST /api/pokestop/scan +// In-memory fort scan with DNF filters (requires fort_in_memory=true) +func PokestopScan(c *gin.Context) { + if !config.Config.FortInMemory { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "fort_in_memory not enabled"}) + return + } + + var params decoder.ApiFortScan + if err := c.ShouldBindJSON(¶ms); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid JSON body"}) + return + } + + result := decoder.PokestopScanEndpoint(params, dbDetails) + c.JSON(http.StatusOK, result) +} + func GetTappable(c *gin.Context) { id := c.Param("tappable_id") tappableId, err := strconv.ParseUint(id, 10, 64) diff --git a/sql/50_quest_reward_columns.up.sql b/sql/50_quest_reward_columns.up.sql new file mode 100644 index 00000000..62ab0121 --- /dev/null +++ b/sql/50_quest_reward_columns.up.sql @@ -0,0 +1,44 @@ +-- Convert quest reward generated columns to regular columns +-- These will now be calculated in code when receiving quest protos + +-- -- Drop indexes first +-- ALTER TABLE `pokestop` +-- DROP INDEX `ix_quest_reward_type`, +-- DROP INDEX `ix_quest_item_id`, +-- DROP INDEX `ix_quest_pokemon_id`, +-- DROP INDEX `ix_alternative_quest_alternative_quest_pokemon_id`, +-- DROP INDEX `ix_alternative_quest_reward_type`, +-- DROP INDEX `ix_alternative_quest_item_id`; + +-- Drop generated columns +ALTER TABLE `pokestop` + DROP COLUMN `quest_reward_type`, + DROP COLUMN `quest_item_id`, + DROP COLUMN `quest_reward_amount`, + DROP COLUMN `quest_pokemon_id`, + DROP COLUMN `alternative_quest_reward_type`, + DROP COLUMN `alternative_quest_item_id`, + DROP COLUMN `alternative_quest_reward_amount`, + DROP COLUMN `alternative_quest_pokemon_id`; + +-- Re-add as regular columns +ALTER TABLE `pokestop` + ADD COLUMN `quest_reward_type` smallint unsigned DEFAULT NULL, + ADD COLUMN `quest_item_id` smallint unsigned DEFAULT NULL, + ADD COLUMN `quest_reward_amount` smallint unsigned DEFAULT NULL, + ADD COLUMN `quest_pokemon_id` smallint unsigned DEFAULT NULL, + ADD COLUMN `quest_pokemon_form_id` smallint unsigned DEFAULT NULL, + ADD COLUMN `alternative_quest_reward_type` smallint unsigned DEFAULT NULL, + ADD COLUMN `alternative_quest_item_id` smallint unsigned DEFAULT NULL, + ADD COLUMN `alternative_quest_reward_amount` smallint unsigned DEFAULT NULL, + ADD COLUMN `alternative_quest_pokemon_id` smallint unsigned DEFAULT NULL, + ADD COLUMN `alternative_quest_pokemon_form_id` smallint unsigned DEFAULT NULL; + +-- -- Re-add indexes +-- ALTER TABLE `pokestop` +-- ADD INDEX `ix_quest_reward_type` (`quest_reward_type`), +-- ADD INDEX `ix_quest_item_id` (`quest_item_id`), +-- ADD INDEX `ix_quest_pokemon_id` (`quest_pokemon_id`), +-- ADD INDEX `ix_alternative_quest_reward_type` (`alternative_quest_reward_type`), +-- ADD INDEX `ix_alternative_quest_item_id` (`alternative_quest_item_id`), +-- ADD INDEX `ix_alternative_quest_pokemon_id` (`alternative_quest_pokemon_id`); From 5e4697bcd130b89988ca6c6db21ccc4629752d1e Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 3 Feb 2026 14:41:45 +0000 Subject: [PATCH 36/78] Use correct result buffer --- decoder/api_fort.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/decoder/api_fort.go b/decoder/api_fort.go index d9f03d30..f35ee7ae 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -76,17 +76,17 @@ type ApiFortDnfMinMax16 struct { } type ApiGymScanResult struct { - Gyms []*Gym `json:"gyms"` - Examined int `json:"examined"` - Skipped int `json:"skipped"` - Total int `json:"total"` + Gyms []*ApiGymResult `json:"gyms"` + Examined int `json:"examined"` + Skipped int `json:"skipped"` + Total int `json:"total"` } type ApiPokestopScanResult struct { - Pokestops []*Pokestop `json:"pokestops"` - Examined int `json:"examined"` - Skipped int `json:"skipped"` - Total int `json:"total"` + Pokestops []*ApiPokestopResult `json:"pokestops"` + Examined int `json:"examined"` + Skipped int `json:"skipped"` + Total int `json:"total"` } func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDnfFilter) bool { @@ -345,14 +345,14 @@ func internalGetForts(fortType FortType, retrieveParameters ApiFortScan) ([]stri func GymScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiGymScanResult { returnKeys, examined, skipped, total := internalGetForts(GYM, retrieveParameters) - results := make([]*Gym, 0, len(returnKeys)) + results := make([]*ApiGymResult, 0, len(returnKeys)) start := time.Now() for _, key := range returnKeys { gym, unlock, err := GetGymRecordReadOnly(context.Background(), dbDetails, key) if err == nil && gym != nil { // Make a copy to avoid holding locks - gymCopy := *gym + gymCopy := buildGymResult(gym) results = append(results, &gymCopy) } if unlock != nil { @@ -371,14 +371,14 @@ func GymScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *Ap func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiPokestopScanResult { returnKeys, examined, skipped, total := internalGetForts(POKESTOP, retrieveParameters) - results := make([]*Pokestop, 0, len(returnKeys)) + results := make([]*ApiPokestopResult, 0, len(returnKeys)) start := time.Now() for _, key := range returnKeys { pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key) if err == nil && pokestop != nil { // Make a copy to avoid holding locks - pokestopCopy := *pokestop + pokestopCopy := buildPokestopResult(pokestop) results = append(results, &pokestopCopy) } if unlock != nil { From 66d87eec722af952a83b734ff33b815dc2ec39da Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 3 Feb 2026 15:49:10 +0000 Subject: [PATCH 37/78] Get it working --- decoder/api_fort.go | 72 ++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/decoder/api_fort.go b/decoder/api_fort.go index f35ee7ae..289fcc29 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -21,42 +21,42 @@ type ApiFortScan struct { type ApiFortDnfFilter struct { PowerUpLevel *ApiFortDnfMinMax8 `json:"power_up_level"` - IsArScanEligible bool `json:"is_ar_scan_eligible"` + IsArScanEligible *bool `json:"is_ar_scan_eligible"` AvailableSlots *ApiFortDnfMinMax8 `json:"available_slots"` - TeamId []*int8 `json:"team_id"` + TeamId []int8 `json:"team_id"` InBattle bool `json:"in_battle"` - RaidLevel []*int8 `json:"raid_level"` + RaidLevel []int8 `json:"raid_level"` RaidPokemon []ApiDnfId `json:"raid_pokemon_id"` - LureId []*int16 `json:"lure_id"` + LureId []int16 `json:"lure_id"` - ArQuestRewardType []*int16 `json:"ar_quest_reward_type"` + ArQuestRewardType []int16 `json:"ar_quest_reward_type"` ArQuestRewardAmount *ApiFortDnfMinMax16 `json:"ar_quest_reward_amount"` - ArQuestRewardItemId []*int16 `json:"ar_quest_reward_item_id"` + ArQuestRewardItemId []int16 `json:"ar_quest_reward_item_id"` ArQuestRewardPokemon []ApiDnfId `json:"ar_quest_reward_pokemon"` - ArQuestType []*int16 `json:"ar_quest_type"` - ArQuestTarget []*int16 `json:"ar_quest_target"` + ArQuestType []int16 `json:"ar_quest_type"` + ArQuestTarget []int16 `json:"ar_quest_target"` ArQuestTemplate []string `json:"ar_quest_template"` - NoArQuestRewardType []*int16 `json:"noar_quest_reward_type"` + NoArQuestRewardType []int16 `json:"noar_quest_reward_type"` NoArQuestRewardAmount *ApiFortDnfMinMax16 `json:"noar_quest_reward_amount"` - NoArQuestRewardItemId []*int16 `json:"noar_quest_reward_item_id"` + NoArQuestRewardItemId []int16 `json:"noar_quest_reward_item_id"` NoArQuestRewardPokemon []ApiDnfId `json:"noar_quest_reward_pokemon"` - NoArQuestType []*int16 `json:"noar_quest_type"` - NoArQuestTarget []*int16 `json:"noar_quest_target"` + NoArQuestType []int16 `json:"noar_quest_type"` + NoArQuestTarget []int16 `json:"noar_quest_target"` NoArQuestTemplate []string `json:"noar_quest_template"` - IncidentDisplayType []*int8 `json:"incident_display_type"` - IncidentStyle []*int8 `json:"incident_style"` - IncidentCharacter []*int16 `json:"incident_character"` + IncidentDisplayType []int8 `json:"incident_display_type"` + IncidentStyle []int8 `json:"incident_style"` + IncidentCharacter []int16 `json:"incident_character"` IncidentSlot1 []ApiDnfId `json:"incident_slot_1"` IncidentSlot2 []ApiDnfId `json:"incident_slot_2"` IncidentSlot3 []ApiDnfId `json:"incident_slot_3"` ContestPokemon []ApiDnfId `json:"contest_pokemon"` - ContestPokemonType1 []*int8 `json:"contest_pokemon_type_1"` - ContestPokemonType2 []*int8 `json:"contest_pokemon_type_2"` - ContestRankingStandard []*int8 `json:"contest_ranking_standard"` + ContestPokemonType1 []int8 `json:"contest_pokemon_type_1"` + ContestPokemonType2 []int8 `json:"contest_pokemon_type_2"` + ContestRankingStandard []int8 `json:"contest_ranking_standard"` ContestTotalEntries *ApiFortDnfMinMax16 `json:"contest_total_entries"` } @@ -96,7 +96,7 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn if filter.PowerUpLevel != nil && (fortLookup.PowerUpLevel < filter.PowerUpLevel.Min || fortLookup.PowerUpLevel > filter.PowerUpLevel.Max) { return false } - if filter.IsArScanEligible && !fortLookup.IsArScanEligible { + if filter.IsArScanEligible != nil && !fortLookup.IsArScanEligible { return false } @@ -104,13 +104,13 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn if filter.AvailableSlots != nil && (fortLookup.AvailableSlots < filter.AvailableSlots.Min || fortLookup.AvailableSlots > filter.AvailableSlots.Max) { return false } - if filter.TeamId != nil && !slices.Contains(filter.TeamId, &fortLookup.TeamId) { + if filter.TeamId != nil && !slices.Contains(filter.TeamId, fortLookup.TeamId) { return false } if filter.InBattle && !fortLookup.InBattle { return false } - if filter.RaidLevel != nil && !slices.Contains(filter.RaidLevel, &fortLookup.RaidLevel) { + if filter.RaidLevel != nil && !slices.Contains(filter.RaidLevel, fortLookup.RaidLevel) { return false } if filter.RaidPokemon != nil { @@ -126,23 +126,23 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn } } } else if fortLookup.FortType == POKESTOP { - if filter.LureId != nil && !slices.Contains(filter.LureId, &fortLookup.LureId) { + if filter.LureId != nil && !slices.Contains(filter.LureId, fortLookup.LureId) { return false } // AR Quest Filters - if filter.ArQuestRewardType != nil && !slices.Contains(filter.ArQuestRewardType, &fortLookup.QuestArRewardType) { + if filter.ArQuestRewardType != nil && !slices.Contains(filter.ArQuestRewardType, fortLookup.QuestArRewardType) { return false } if filter.ArQuestRewardAmount != nil && (fortLookup.QuestArRewardAmount < filter.ArQuestRewardAmount.Min || fortLookup.QuestArRewardAmount > filter.ArQuestRewardAmount.Max) { return false } - if filter.ArQuestRewardItemId != nil && !slices.Contains(filter.ArQuestRewardItemId, &fortLookup.QuestArRewardItemId) { + if filter.ArQuestRewardItemId != nil && !slices.Contains(filter.ArQuestRewardItemId, fortLookup.QuestArRewardItemId) { return false } - if filter.ArQuestType != nil && !slices.Contains(filter.ArQuestType, &fortLookup.QuestArType) { + if filter.ArQuestType != nil && !slices.Contains(filter.ArQuestType, fortLookup.QuestArType) { return false } - if filter.ArQuestTarget != nil && !slices.Contains(filter.ArQuestTarget, &fortLookup.QuestArTarget) { + if filter.ArQuestTarget != nil && !slices.Contains(filter.ArQuestTarget, fortLookup.QuestArTarget) { return false } if filter.ArQuestTemplate != nil && !slices.Contains(filter.ArQuestTemplate, fortLookup.QuestArTemplate) { @@ -162,19 +162,19 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn } // No-AR Quest Filters - if filter.NoArQuestRewardType != nil && !slices.Contains(filter.NoArQuestRewardType, &fortLookup.QuestNoArRewardType) { + if filter.NoArQuestRewardType != nil && !slices.Contains(filter.NoArQuestRewardType, fortLookup.QuestNoArRewardType) { return false } if filter.NoArQuestRewardAmount != nil && (fortLookup.QuestNoArRewardAmount < filter.NoArQuestRewardAmount.Min || fortLookup.QuestNoArRewardAmount > filter.NoArQuestRewardAmount.Max) { return false } - if filter.NoArQuestRewardItemId != nil && !slices.Contains(filter.NoArQuestRewardItemId, &fortLookup.QuestNoArRewardItemId) { + if filter.NoArQuestRewardItemId != nil && !slices.Contains(filter.NoArQuestRewardItemId, fortLookup.QuestNoArRewardItemId) { return false } - if filter.NoArQuestType != nil && !slices.Contains(filter.NoArQuestType, &fortLookup.QuestNoArType) { + if filter.NoArQuestType != nil && !slices.Contains(filter.NoArQuestType, fortLookup.QuestNoArType) { return false } - if filter.NoArQuestTarget != nil && !slices.Contains(filter.NoArQuestTarget, &fortLookup.QuestNoArTarget) { + if filter.NoArQuestTarget != nil && !slices.Contains(filter.NoArQuestTarget, fortLookup.QuestNoArTarget) { return false } if filter.NoArQuestTemplate != nil && !slices.Contains(filter.NoArQuestTemplate, fortLookup.QuestNoArTemplate) { @@ -194,13 +194,13 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn } // Contest Filters - if filter.ContestPokemonType1 != nil && !slices.Contains(filter.ContestPokemonType1, &fortLookup.ContestPokemonType1) { + if filter.ContestPokemonType1 != nil && !slices.Contains(filter.ContestPokemonType1, fortLookup.ContestPokemonType1) { return false } - if filter.ContestPokemonType2 != nil && !slices.Contains(filter.ContestPokemonType2, &fortLookup.ContestPokemonType2) { + if filter.ContestPokemonType2 != nil && !slices.Contains(filter.ContestPokemonType2, fortLookup.ContestPokemonType2) { return false } - if filter.ContestRankingStandard != nil && !slices.Contains(filter.ContestRankingStandard, &fortLookup.ContestRankingStandard) { + if filter.ContestRankingStandard != nil && !slices.Contains(filter.ContestRankingStandard, fortLookup.ContestRankingStandard) { return false } if filter.ContestTotalEntries != nil && (fortLookup.ContestTotalEntries < filter.ContestTotalEntries.Min || fortLookup.ContestTotalEntries > filter.ContestTotalEntries.Max) { @@ -224,13 +224,13 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn incidentMatch := false for _, incident := range fortLookup.Incidents { incidentFilterMatch := true - if filter.IncidentDisplayType != nil && !slices.Contains(filter.IncidentDisplayType, &incident.DisplayType) { + if filter.IncidentDisplayType != nil && !slices.Contains(filter.IncidentDisplayType, incident.DisplayType) { incidentFilterMatch = false } - if incidentFilterMatch && filter.IncidentStyle != nil && !slices.Contains(filter.IncidentStyle, &incident.Style) { + if incidentFilterMatch && filter.IncidentStyle != nil && !slices.Contains(filter.IncidentStyle, incident.Style) { incidentFilterMatch = false } - if incidentFilterMatch && filter.IncidentCharacter != nil && !slices.Contains(filter.IncidentCharacter, &incident.Character) { + if incidentFilterMatch && filter.IncidentCharacter != nil && !slices.Contains(filter.IncidentCharacter, incident.Character) { incidentFilterMatch = false } if incidentFilterMatch && filter.IncidentSlot1 != nil { From 882f1b071bfe81c9b02d2d5a300b5f322de9b983 Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 3 Feb 2026 16:16:59 +0000 Subject: [PATCH 38/78] Convert station/gym to sharded cache --- decoder/main.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/decoder/main.go b/decoder/main.go index 106ce299..112747e5 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -55,8 +55,8 @@ type webhooksSenderInterface interface { var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector var pokestopCache *ShardedCache[string, *Pokestop] -var gymCache *ttlcache.Cache[string, *Gym] -var stationCache *ttlcache.Cache[string, *Station] +var gymCache *ShardedCache[string, *Gym] +var stationCache *ShardedCache[string, *Station] var tappableCache *ttlcache.Cache[uint64, *Tappable] var weatherCache *ttlcache.Cache[int64, *Weather] var weatherConsensusCache *ttlcache.Cache[int64, *WeatherConsensusState] @@ -102,21 +102,23 @@ func initDataCache() { }) } - gymCache = ttlcache.New[string, *Gym]( - ttlcache.WithTTL[string, *Gym](60 * time.Minute), - ) + gymCache = NewShardedCache(ShardedCacheConfig[string, *Gym]{ + NumShards: runtime.NumCPU(), + TTL: 60 * time.Minute, + KeyToShard: StringKeyToShard, + }) if config.Config.FortInMemory { gymCache.OnEviction(func(ctx context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *Gym]) { g := item.Value() removeFortFromTree(g.Id, g.Lat, g.Lon) }) } - go gymCache.Start() - stationCache = ttlcache.New[string, *Station]( - ttlcache.WithTTL[string, *Station](60 * time.Minute), - ) - go stationCache.Start() + stationCache = NewShardedCache(ShardedCacheConfig[string, *Station]{ + NumShards: runtime.NumCPU(), + TTL: 60 * time.Minute, + KeyToShard: StringKeyToShard, + }) tappableCache = ttlcache.New[uint64, *Tappable]( ttlcache.WithTTL[uint64, *Tappable](60 * time.Minute), From 81c01074d992beeb8990feda170b0be75cf3f132 Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 3 Feb 2026 16:34:12 +0000 Subject: [PATCH 39/78] preload more things --- config/config.go | 2 +- decoder/fort_preload.go | 158 ---------------------- decoder/preload.go | 281 +++++++++++++++++++++++++++++++++++++++ decoder/spawnpoint.go | 6 +- decoder/station_state.go | 12 +- main.go | 17 ++- 6 files changed, 307 insertions(+), 169 deletions(-) delete mode 100644 decoder/fort_preload.go create mode 100644 decoder/preload.go diff --git a/config/config.go b/config/config.go index e0e4e0be..099d786a 100644 --- a/config/config.go +++ b/config/config.go @@ -17,7 +17,7 @@ type configDefinition struct { Prometheus Prometheus `koanf:"prometheus"` PokemonMemoryOnly bool `koanf:"pokemon_memory_only"` PokemonInternalToDb bool `koanf:"pokemon_internal_to_db"` - PreloadForts bool `koanf:"preload_forts"` // Pre-load all forts into cache on startup + Preload bool `koanf:"preload"` // Pre-load forts, stations, spawnpoints into cache on startup FortInMemory bool `koanf:"fort_in_memory"` // Keep forts in memory with rtree for spatial lookups Cleanup cleanup `koanf:"cleanup"` RawBearer string `koanf:"raw_bearer"` diff --git a/decoder/fort_preload.go b/decoder/fort_preload.go deleted file mode 100644 index 1e134e5b..00000000 --- a/decoder/fort_preload.go +++ /dev/null @@ -1,158 +0,0 @@ -package decoder - -import ( - "runtime" - "sync" - "sync/atomic" - "time" - - log "github.com/sirupsen/logrus" - - "golbat/db" -) - -// PreloadForts loads all forts from DB into cache. -// If populateRtree is true, also builds the rtree index. -// Fort tracker is always populated during preload. -func PreloadForts(dbDetails db.DbDetails, populateRtree bool) error { - startTime := time.Now() - - var wg sync.WaitGroup - var pokestopCount, gymCount int32 - - wg.Add(2) - go func() { - defer wg.Done() - pokestopCount = preloadPokestops(dbDetails, populateRtree) - }() - go func() { - defer wg.Done() - gymCount = preloadGyms(dbDetails, populateRtree) - }() - wg.Wait() - - log.Infof("PreloadForts: loaded %d pokestops and %d gyms in %v (rtree=%v)", - pokestopCount, gymCount, time.Since(startTime), populateRtree) - - return nil -} - -func preloadPokestops(dbDetails db.DbDetails, populateRtree bool) int32 { - query := "SELECT " + pokestopSelectColumns + " FROM pokestop WHERE deleted = 0" - rows, err := dbDetails.GeneralDb.Queryx(query) - if err != nil { - log.Errorf("PreloadForts: failed to query pokestops - %s", err) - return 0 - } - defer rows.Close() - - numWorkers := runtime.NumCPU() - jobs := make(chan *Pokestop, 100) - var wg sync.WaitGroup - var count int32 - - for i := 0; i < numWorkers; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for pokestop := range jobs { - // Add to cache - pokestopCache.Set(pokestop.Id, pokestop, 0) // 0 = use default TTL - - // Update rtree if enabled - if populateRtree { - fortRtreeUpdatePokestopOnSave(pokestop) - } - - // Register with fort tracker - if fortTracker != nil && pokestop.CellId.Valid { - fortTracker.RegisterFort( - pokestop.Id, - uint64(pokestop.CellId.Int64), - false, - pokestop.Updated*1000, // convert to milliseconds - ) - } - - c := atomic.AddInt32(&count, 1) - if c%10000 == 0 { - log.Infof("PreloadForts: loaded %d pokestops...", c) - } - } - }() - } - - for rows.Next() { - var pokestop Pokestop - err := rows.StructScan(&pokestop) - if err != nil { - log.Errorf("PreloadForts: pokestop scan error - %s", err) - continue - } - jobs <- &pokestop - } - close(jobs) - wg.Wait() - - return count -} - -func preloadGyms(dbDetails db.DbDetails, populateRtree bool) int32 { - query := "SELECT " + gymSelectColumns + " FROM gym WHERE deleted = 0" - rows, err := dbDetails.GeneralDb.Queryx(query) - if err != nil { - log.Errorf("PreloadForts: failed to query gyms - %s", err) - return 0 - } - defer rows.Close() - - numWorkers := runtime.NumCPU() - jobs := make(chan *Gym, 100) - var wg sync.WaitGroup - var count int32 - - for i := 0; i < numWorkers; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for gym := range jobs { - // Add to cache - gymCache.Set(gym.Id, gym, 0) // 0 = use default TTL - - // Update rtree if enabled - if populateRtree { - fortRtreeUpdateGymOnSave(gym) - } - - // Register with fort tracker - if fortTracker != nil && gym.CellId.Valid { - fortTracker.RegisterFort( - gym.Id, - uint64(gym.CellId.Int64), - true, - gym.Updated*1000, // convert to milliseconds - ) - } - - c := atomic.AddInt32(&count, 1) - if c%10000 == 0 { - log.Infof("PreloadForts: loaded %d gyms...", c) - } - } - }() - } - - for rows.Next() { - var gym Gym - err := rows.StructScan(&gym) - if err != nil { - log.Errorf("PreloadForts: gym scan error - %s", err) - continue - } - jobs <- &gym - } - close(jobs) - wg.Wait() - - return count -} diff --git a/decoder/preload.go b/decoder/preload.go new file mode 100644 index 00000000..55c1417f --- /dev/null +++ b/decoder/preload.go @@ -0,0 +1,281 @@ +package decoder + +import ( + "runtime" + "sync" + "sync/atomic" + "time" + + log "github.com/sirupsen/logrus" + + "golbat/db" +) + +// Preload loads forts, stations, and recent spawnpoints from DB into cache. +// If populateRtree is true, also builds the rtree index for forts. +func Preload(dbDetails db.DbDetails, populateRtree bool) { + startTime := time.Now() + + var wg sync.WaitGroup + var pokestopCount, gymCount, stationCount, spawnpointCount int32 + + wg.Add(4) + go func() { + defer wg.Done() + pokestopCount = preloadPokestops(dbDetails, populateRtree) + }() + go func() { + defer wg.Done() + gymCount = preloadGyms(dbDetails, populateRtree) + }() + go func() { + defer wg.Done() + stationCount = preloadStations(dbDetails) + }() + go func() { + defer wg.Done() + spawnpointCount = preloadSpawnpoints(dbDetails) + }() + wg.Wait() + + log.Infof("Preload: loaded %d pokestops, %d gyms, %d stations, %d spawnpoints in %v (rtree=%v)", + pokestopCount, gymCount, stationCount, spawnpointCount, time.Since(startTime), populateRtree) +} + +// PreloadForts loads all forts from DB into cache. +// If populateRtree is true, also builds the rtree index. +// Fort tracker is always populated during preload. +func PreloadForts(dbDetails db.DbDetails, populateRtree bool) error { + startTime := time.Now() + + var wg sync.WaitGroup + var pokestopCount, gymCount int32 + + wg.Add(2) + go func() { + defer wg.Done() + pokestopCount = preloadPokestops(dbDetails, populateRtree) + }() + go func() { + defer wg.Done() + gymCount = preloadGyms(dbDetails, populateRtree) + }() + wg.Wait() + + log.Infof("PreloadForts: loaded %d pokestops and %d gyms in %v (rtree=%v)", + pokestopCount, gymCount, time.Since(startTime), populateRtree) + + return nil +} + +func preloadPokestops(dbDetails db.DbDetails, populateRtree bool) int32 { + query := "SELECT " + pokestopSelectColumns + " FROM pokestop WHERE deleted = 0" + rows, err := dbDetails.GeneralDb.Queryx(query) + if err != nil { + log.Errorf("Preload: failed to query pokestops - %s", err) + return 0 + } + defer rows.Close() + + numWorkers := runtime.NumCPU() + jobs := make(chan *Pokestop, 100) + var wg sync.WaitGroup + var count int32 + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for pokestop := range jobs { + // Add to cache + pokestopCache.Set(pokestop.Id, pokestop, 0) // 0 = use default TTL + + // Update rtree if enabled + if populateRtree { + fortRtreeUpdatePokestopOnSave(pokestop) + } + + // Register with fort tracker + if fortTracker != nil && pokestop.CellId.Valid { + fortTracker.RegisterFort( + pokestop.Id, + uint64(pokestop.CellId.Int64), + false, + pokestop.Updated*1000, // convert to milliseconds + ) + } + + c := atomic.AddInt32(&count, 1) + if c%10000 == 0 { + log.Infof("Preload: loaded %d pokestops...", c) + } + } + }() + } + + for rows.Next() { + var pokestop Pokestop + err := rows.StructScan(&pokestop) + if err != nil { + log.Errorf("Preload: pokestop scan error - %s", err) + continue + } + jobs <- &pokestop + } + close(jobs) + wg.Wait() + + return count +} + +func preloadGyms(dbDetails db.DbDetails, populateRtree bool) int32 { + query := "SELECT " + gymSelectColumns + " FROM gym WHERE deleted = 0" + rows, err := dbDetails.GeneralDb.Queryx(query) + if err != nil { + log.Errorf("Preload: failed to query gyms - %s", err) + return 0 + } + defer rows.Close() + + numWorkers := runtime.NumCPU() + jobs := make(chan *Gym, 100) + var wg sync.WaitGroup + var count int32 + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for gym := range jobs { + // Add to cache + gymCache.Set(gym.Id, gym, 0) // 0 = use default TTL + + // Update rtree if enabled + if populateRtree { + fortRtreeUpdateGymOnSave(gym) + } + + // Register with fort tracker + if fortTracker != nil && gym.CellId.Valid { + fortTracker.RegisterFort( + gym.Id, + uint64(gym.CellId.Int64), + true, + gym.Updated*1000, // convert to milliseconds + ) + } + + c := atomic.AddInt32(&count, 1) + if c%10000 == 0 { + log.Infof("Preload: loaded %d gyms...", c) + } + } + }() + } + + for rows.Next() { + var gym Gym + err := rows.StructScan(&gym) + if err != nil { + log.Errorf("Preload: gym scan error - %s", err) + continue + } + jobs <- &gym + } + close(jobs) + wg.Wait() + + return count +} + +func preloadStations(dbDetails db.DbDetails) int32 { + query := "SELECT " + stationSelectColumns + " FROM station" + rows, err := dbDetails.GeneralDb.Queryx(query) + if err != nil { + log.Errorf("Preload: failed to query stations - %s", err) + return 0 + } + defer rows.Close() + + numWorkers := runtime.NumCPU() + jobs := make(chan *Station, 100) + var wg sync.WaitGroup + var count int32 + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for station := range jobs { + // Add to cache + stationCache.Set(station.Id, station, 0) // 0 = use default TTL + + c := atomic.AddInt32(&count, 1) + if c%10000 == 0 { + log.Infof("Preload: loaded %d stations...", c) + } + } + }() + } + + for rows.Next() { + var station Station + err := rows.StructScan(&station) + if err != nil { + log.Errorf("Preload: station scan error - %s", err) + continue + } + jobs <- &station + } + close(jobs) + wg.Wait() + + return count +} + +func preloadSpawnpoints(dbDetails db.DbDetails) int32 { + // Load spawnpoints seen in the last 48 hours + cutoff := time.Now().Unix() - 48*60*60 + query := "SELECT " + spawnpointSelectColumns + " FROM spawnpoint WHERE last_seen > ?" + rows, err := dbDetails.GeneralDb.Queryx(query, cutoff) + if err != nil { + log.Errorf("Preload: failed to query spawnpoints - %s", err) + return 0 + } + defer rows.Close() + + numWorkers := runtime.NumCPU() + jobs := make(chan *Spawnpoint, 100) + var wg sync.WaitGroup + var count int32 + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for spawnpoint := range jobs { + // Add to cache + spawnpointCache.Set(spawnpoint.Id, spawnpoint, 0) // 0 = use default TTL + + c := atomic.AddInt32(&count, 1) + if c%10000 == 0 { + log.Infof("Preload: loaded %d spawnpoints...", c) + } + } + }() + } + + for rows.Next() { + var spawnpoint Spawnpoint + err := rows.StructScan(&spawnpoint) + if err != nil { + log.Errorf("Preload: spawnpoint scan error - %s", err) + continue + } + jobs <- &spawnpoint + } + close(jobs) + wg.Wait() + + return count +} diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 0f5f0f3b..cab83009 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -17,6 +17,10 @@ import ( log "github.com/sirupsen/logrus" ) +// spawnpointSelectColumns defines the columns for spawnpoint queries. +// Used by both single-row and bulk load queries to keep them in sync. +const spawnpointSelectColumns = `id, lat, lon, updated, last_seen, despawn_sec` + // Spawnpoint struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Spawnpoint struct { @@ -155,7 +159,7 @@ func (s *Spawnpoint) SetLastSeen(v int64) { func loadSpawnpointFromDatabase(ctx context.Context, db db.DbDetails, spawnpointId int64, spawnpoint *Spawnpoint) error { err := db.GeneralDb.GetContext(ctx, spawnpoint, - "SELECT id, lat, lon, updated, last_seen, despawn_sec FROM spawnpoint WHERE id = ?", spawnpointId) + "SELECT "+spawnpointSelectColumns+" FROM spawnpoint WHERE id = ?", spawnpointId) statsCollector.IncDbQuery("select spawnpoint", err) return err } diff --git a/decoder/station_state.go b/decoder/station_state.go index b038936d..cd3e4b2f 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -14,6 +14,15 @@ import ( "golbat/webhooks" ) +// stationSelectColumns defines the columns for station queries. +// Used by both single-row and bulk load queries to keep them in sync. +const stationSelectColumns = `id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, + is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, + battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, + battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, + battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, + stationed_pokemon` + type StationWebhook struct { Id string `json:"id"` Latitude float64 `json:"latitude"` @@ -40,8 +49,7 @@ type StationWebhook struct { func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId string, station *Station) error { err := db.GeneralDb.GetContext(ctx, station, - `SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon - FROM station WHERE id = ?`, stationId) + `SELECT `+stationSelectColumns+` FROM station WHERE id = ?`, stationId) statsCollector.IncDbQuery("select station", err) return err } diff --git a/main.go b/main.go index a95b47b0..5214c5e7 100644 --- a/main.go +++ b/main.go @@ -257,15 +257,18 @@ func main() { } decoder.InitFortTracker(staleThreshold) - // Determine fort loading strategy - // FortInMemory enables rtree spatial lookups; PreloadForts just warms the cache - // TestFortInMemory is deprecated but still supported (treated as FortInMemory) + // Determine loading strategy + // Preload: warms cache for forts, stations, and recent spawnpoints + // FortInMemory: enables rtree spatial lookups (only loads forts) fortInMemory := cfg.FortInMemory - fullPreload := cfg.PreloadForts || fortInMemory - if fullPreload { - // Full preload: loads into cache, registers with fort tracker, optionally builds rtree - if err := decoder.PreloadForts(dbDetails, fortInMemory); err != nil { + if cfg.Preload { + // Full preload: loads forts, stations, spawnpoints into cache + // Registers forts with fort tracker, optionally builds rtree + decoder.Preload(dbDetails, fortInMemory) + } else if fortInMemory { + // Fort in memory only: loads forts into cache with rtree + if err := decoder.PreloadForts(dbDetails, true); err != nil { log.Errorf("failed to preload forts: %s", err) } } else { From 62556889879ee1568118ff90b644f93614ddf764 Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 3 Feb 2026 18:38:51 +0000 Subject: [PATCH 40/78] Preload incidents --- decoder/incident_state.go | 7 +++-- decoder/preload.go | 61 +++++++++++++++++++++++++++++++++++---- decoder/station_decode.go | 2 +- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/decoder/incident_state.go b/decoder/incident_state.go index 641bc06b..fda4cc29 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -13,10 +13,13 @@ import ( "golbat/webhooks" ) +// incidentSelectColumns defines the columns for incident queries. +// Used by both single-row and bulk load queries to keep them in sync. +const incidentSelectColumns = "id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form" + func loadIncidentFromDatabase(ctx context.Context, db db.DbDetails, incidentId string, incident *Incident) error { err := db.GeneralDb.GetContext(ctx, incident, - "SELECT id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form "+ - "FROM incident WHERE incident.id = ?", incidentId) + "SELECT "+incidentSelectColumns+" FROM incident WHERE id = ?", incidentId) statsCollector.IncDbQuery("select incident", err) return err } diff --git a/decoder/preload.go b/decoder/preload.go index 55c1417f..b659acae 100644 --- a/decoder/preload.go +++ b/decoder/preload.go @@ -11,15 +11,15 @@ import ( "golbat/db" ) -// Preload loads forts, stations, and recent spawnpoints from DB into cache. +// Preload loads forts, stations, active incidents, and recent spawnpoints from DB into cache. // If populateRtree is true, also builds the rtree index for forts. func Preload(dbDetails db.DbDetails, populateRtree bool) { startTime := time.Now() var wg sync.WaitGroup - var pokestopCount, gymCount, stationCount, spawnpointCount int32 + var pokestopCount, gymCount, stationCount, incidentCount, spawnpointCount int32 - wg.Add(4) + wg.Add(5) go func() { defer wg.Done() pokestopCount = preloadPokestops(dbDetails, populateRtree) @@ -32,14 +32,18 @@ func Preload(dbDetails db.DbDetails, populateRtree bool) { defer wg.Done() stationCount = preloadStations(dbDetails) }() + go func() { + defer wg.Done() + incidentCount = preloadIncidents(dbDetails) + }() go func() { defer wg.Done() spawnpointCount = preloadSpawnpoints(dbDetails) }() wg.Wait() - log.Infof("Preload: loaded %d pokestops, %d gyms, %d stations, %d spawnpoints in %v (rtree=%v)", - pokestopCount, gymCount, stationCount, spawnpointCount, time.Since(startTime), populateRtree) + log.Infof("Preload: loaded %d pokestops, %d gyms, %d stations, %d incidents, %d spawnpoints in %v (rtree=%v)", + pokestopCount, gymCount, stationCount, incidentCount, spawnpointCount, time.Since(startTime), populateRtree) } // PreloadForts loads all forts from DB into cache. @@ -233,6 +237,53 @@ func preloadStations(dbDetails db.DbDetails) int32 { return count } +func preloadIncidents(dbDetails db.DbDetails) int32 { + // Load active incidents (not yet expired) + now := time.Now().Unix() + query := "SELECT " + incidentSelectColumns + " FROM incident WHERE expiration > ?" + rows, err := dbDetails.GeneralDb.Queryx(query, now) + if err != nil { + log.Errorf("Preload: failed to query incidents - %s", err) + return 0 + } + defer rows.Close() + + numWorkers := runtime.NumCPU() + jobs := make(chan *Incident, 100) + var wg sync.WaitGroup + var count int32 + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for incident := range jobs { + // Add to cache + incidentCache.Set(incident.Id, incident, 0) // 0 = use default TTL + + c := atomic.AddInt32(&count, 1) + if c%10000 == 0 { + log.Infof("Preload: loaded %d incidents...", c) + } + } + }() + } + + for rows.Next() { + var incident Incident + err := rows.StructScan(&incident) + if err != nil { + log.Errorf("Preload: incident scan error - %s", err) + continue + } + jobs <- &incident + } + close(jobs) + wg.Wait() + + return count +} + func preloadSpawnpoints(dbDetails db.DbDetails) int32 { // Load spawnpoints seen in the last 48 hours cutoff := time.Now().Unix() - 48*60*60 diff --git a/decoder/station_decode.go b/decoder/station_decode.go index da341b9a..6e9f54fa 100644 --- a/decoder/station_decode.go +++ b/decoder/station_decode.go @@ -16,7 +16,7 @@ func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, // NOTE: Some names have more than 255 runes, which won't fit in our // varchar(255). if truncateStr, truncated := util.TruncateUTF8(stationProto.Name, 255); truncated { - log.Warnf("truncating name for station id '%s'. Orig name: %s", + log.Debugf("truncating name for station id '%s'. Orig name: %s", stationProto.Id, stationProto.Name, ) From f70e5b5bdad6c11937490482017280501a9b3059 Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 3 Feb 2026 23:02:18 +0000 Subject: [PATCH 41/78] Write behind database queue implementation --- config/config.go | 3 + config/reader.go | 3 + db/pokestop.go | 89 -------- decoder/gmo_decode.go | 27 +-- decoder/gym_state.go | 151 +++++++------ decoder/gym_writeable.go | 27 +++ decoder/incident_state.go | 73 ++++--- decoder/incident_writeable.go | 27 +++ decoder/main.go | 63 ++++++ decoder/pending_pokemon.go | 168 --------------- decoder/pokemon_process.go | 5 - decoder/pokemon_state.go | 207 +++++++++--------- decoder/pokemon_writeable.go | 29 +++ decoder/pokestop_process.go | 5 +- decoder/pokestop_state.go | 245 +++++++++++++++++++--- decoder/pokestop_writeable.go | 27 +++ decoder/routes_state.go | 51 +++-- decoder/routes_writeable.go | 27 +++ decoder/s2cell.go | 56 +++-- decoder/spawnpoint.go | 39 +++- decoder/spawnpoint_writeable.go | 29 +++ decoder/station_state.go | 54 +++-- decoder/station_writeable.go | 27 +++ decoder/tappable_state.go | 52 +++-- decoder/tappable_writeable.go | 29 +++ decoder/writebehind/processor.go | 74 +++++++ decoder/writebehind/queue.go | 197 +++++++++++++++++ decoder/writebehind/queue_test.go | 200 ++++++++++++++++++ decoder/writebehind/ratelimit.go | 116 ++++++++++ decoder/writebehind/s2cell_accumulator.go | 192 +++++++++++++++++ decoder/writebehind/types.go | 37 ++++ main.go | 11 +- stats.go | 50 +---- stats_collector/noop.go | 9 + stats_collector/prometheus.go | 92 ++++++++ stats_collector/stats_collector.go | 11 + 36 files changed, 1888 insertions(+), 614 deletions(-) create mode 100644 decoder/gym_writeable.go create mode 100644 decoder/incident_writeable.go delete mode 100644 decoder/pending_pokemon.go create mode 100644 decoder/pokemon_writeable.go create mode 100644 decoder/pokestop_writeable.go create mode 100644 decoder/routes_writeable.go create mode 100644 decoder/spawnpoint_writeable.go create mode 100644 decoder/station_writeable.go create mode 100644 decoder/tappable_writeable.go create mode 100644 decoder/writebehind/processor.go create mode 100644 decoder/writebehind/queue.go create mode 100644 decoder/writebehind/queue_test.go create mode 100644 decoder/writebehind/ratelimit.go create mode 100644 decoder/writebehind/s2cell_accumulator.go create mode 100644 decoder/writebehind/types.go diff --git a/config/config.go b/config/config.go index 099d786a..95120c58 100644 --- a/config/config.go +++ b/config/config.go @@ -126,6 +126,9 @@ type tuning struct { ProfileRoutes bool `koanf:"profile_routes"` MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` ReduceUpdates bool `koanf:"reduce_updates"` + WriteBehindStartupDelay int `koanf:"write_behind_startup_delay"` // seconds, default: 120 + WriteBehindRateLimit int `koanf:"write_behind_rate_limit"` // writes/sec, 0=unlimited + WriteBehindBurstCapacity int `koanf:"write_behind_burst_capacity"` // default: 100 } type scanRule struct { diff --git a/config/reader.go b/config/reader.go index 57935895..1038475d 100644 --- a/config/reader.go +++ b/config/reader.go @@ -55,6 +55,9 @@ func ReadConfig() (configDefinition, error) { MaxPokemonDistance: 100, MaxConcurrentProactiveIVSwitch: 6, ReduceUpdates: false, + WriteBehindStartupDelay: 120, // 2 minutes + WriteBehindRateLimit: 0, // unlimited + WriteBehindBurstCapacity: 100, }, Weather: weather{ ProactiveIVSwitching: true, diff --git a/db/pokestop.go b/db/pokestop.go index f450d449..897289a1 100644 --- a/db/pokestop.go +++ b/db/pokestop.go @@ -3,7 +3,6 @@ package db import ( "context" "database/sql" - "errors" "github.com/jmoiron/sqlx" "github.com/paulmach/orb/geojson" @@ -49,94 +48,6 @@ func GetPokestopPositions(db DbDetails, fence *geojson.Feature) ([]QuestLocation return areas, nil } -func RemoveQuests(ctx context.Context, db DbDetails, fence *geojson.Feature) (int64, error) { - const updateChunkSize = 500 - - //goland:noinspection GoPreferNilSlice - allIdsToUpdate := []string{} - var removedQuestsCount int64 - - bbox := fence.Geometry.Bound() - bytes, err := fence.MarshalJSON() - if err != nil { - statsCollector.IncDbQuery("remove quests", err) - return removedQuestsCount, err - } - - idQueryString := "SELECT `id` FROM `pokestop` " + - "WHERE lat >= ? and lon >= ? and lat <= ? and lon <= ? and enabled = 1 " + - "AND ST_CONTAINS(ST_GeomFromGeoJSON('" + string(bytes) + "', 2, 0), POINT(lon, lat))" - - //log.Debugf("Clear quests query: %s", idQueryString) - - // collect allIdsToUpdate - err = db.GeneralDb.Select(&allIdsToUpdate, idQueryString, - bbox.Min.Lat(), bbox.Min.Lon(), bbox.Max.Lat(), bbox.Max.Lon(), - ) - - if errors.Is(err, sql.ErrNoRows) { - statsCollector.IncDbQuery("remove quests", err) - return removedQuestsCount, nil - } - - if err != nil { - statsCollector.IncDbQuery("remove quests", err) - return removedQuestsCount, err - } - - for { - // take at most updateChunkSize elements from allIdsToUpdate - updateIdsCount := len(allIdsToUpdate) - - if updateIdsCount == 0 { - break - } - - if updateIdsCount > updateChunkSize { - updateIdsCount = updateChunkSize - } - - updateIds := allIdsToUpdate[:updateIdsCount] - - // remove processed elements from allIdsToUpdate - allIdsToUpdate = allIdsToUpdate[updateIdsCount:] - - query, args, _ := sqlx.In("UPDATE pokestop "+ - "SET "+ - "quest_type = NULL,"+ - "quest_timestamp = NULL,"+ - "quest_target = NULL,"+ - "quest_conditions = NULL,"+ - "quest_rewards = NULL,"+ - "quest_template = NULL,"+ - "quest_title = NULL, "+ - "quest_expiry = NULL, "+ - "alternative_quest_type = NULL,"+ - "alternative_quest_timestamp = NULL,"+ - "alternative_quest_target = NULL,"+ - "alternative_quest_conditions = NULL,"+ - "alternative_quest_rewards = NULL,"+ - "alternative_quest_template = NULL,"+ - "alternative_quest_title = NULL, "+ - "alternative_quest_expiry = NULL "+ - "WHERE id IN (?)", updateIds) - - query = db.GeneralDb.Rebind(query) - res, err := db.GeneralDb.ExecContext(ctx, query, args...) - - if err != nil { - statsCollector.IncDbQuery("remove quests", err) - return removedQuestsCount, err - } - - rowsAffected, _ := res.RowsAffected() - removedQuestsCount += rowsAffected - } - - statsCollector.IncDbQuery("remove quests", err) - return removedQuestsCount, err -} - func ClearOldPokestops(ctx context.Context, db DbDetails, stopIds []string) error { query, args, _ := sqlx.In("UPDATE pokestop SET deleted = 1 WHERE id IN (?);", stopIds) query = db.GeneralDb.Rebind(query) diff --git a/decoder/gmo_decode.go b/decoder/gmo_decode.go index 8f2a7f54..44b633e1 100644 --- a/decoder/gmo_decode.go +++ b/decoder/gmo_decode.go @@ -111,33 +111,18 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca spawnpointUpdateFromWild(ctx, db, wild.Data, wild.Timestamp) if scanParameters.ProcessWild { - // Use read-only getter - we're only checking if update is needed, then queuing - pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { - log.Errorf("getPokemonRecordReadOnly: %s", err) + log.Errorf("getOrCreatePokemonRecord: %s", err) continue } updateTime := wild.Timestamp / 1000 - shouldQueue := pokemon == nil || pokemon.wildSignificantUpdate(wild.Data, updateTime) - - if unlock != nil { - unlock() - } - - if shouldQueue { - // The sweeper will process it after timeout if no encounter arrives - pending := &PendingPokemon{ - EncounterId: encounterId, - WildPokemon: wild.Data, - CellId: int64(wild.Cell), - TimestampMs: wild.Timestamp, - UpdateTime: updateTime, - WeatherLookup: weatherLookup, - Username: username, - } - pokemonPendingQueue.AddPending(pending) + if pokemon.isNewRecord() || pokemon.wildSignificantUpdate(wild.Data, updateTime) { + pokemon.updateFromWild(ctx, db, wild.Data, int64(wild.Cell), weatherLookup, wild.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, updateTime) } + unlock() } } diff --git a/decoder/gym_state.go b/decoder/gym_state.go index cee377cc..2e927a59 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -345,75 +345,26 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { } gym.SetUpdated(now) - if gym.IsDirty() { - if gym.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ - "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) - - statsCollector.IncDbQuery("insert gym", err) - if err != nil { - log.Errorf("insert gym: %s", err) - return - } + // Capture isNewRecord before state changes + isNewRecord := gym.IsNewRecord() - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ - "lat = :lat, "+ - "lon = :lon, "+ - "name = :name, "+ - "url = :url, "+ - "last_modified_timestamp = :last_modified_timestamp, "+ - "raid_end_timestamp = :raid_end_timestamp, "+ - "raid_spawn_timestamp = :raid_spawn_timestamp, "+ - "raid_battle_timestamp = :raid_battle_timestamp, "+ - "updated = :updated, "+ - "raid_pokemon_id = :raid_pokemon_id, "+ - "guarding_pokemon_id = :guarding_pokemon_id, "+ - "guarding_pokemon_display = :guarding_pokemon_display, "+ - "available_slots = :available_slots, "+ - "team_id = :team_id, "+ - "raid_level = :raid_level, "+ - "enabled = :enabled, "+ - "ex_raid_eligible = :ex_raid_eligible, "+ - "in_battle = :in_battle, "+ - "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ - "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ - "raid_pokemon_form = :raid_pokemon_form, "+ - "raid_pokemon_alignment = :raid_pokemon_alignment, "+ - "raid_pokemon_cp = :raid_pokemon_cp, "+ - "raid_is_exclusive = :raid_is_exclusive, "+ - "cell_id = :cell_id, "+ - "deleted = :deleted, "+ - "total_cp = :total_cp, "+ - "raid_pokemon_gender = :raid_pokemon_gender, "+ - "sponsor_id = :sponsor_id, "+ - "partner_id = :partner_id, "+ - "raid_pokemon_costume = :raid_pokemon_costume, "+ - "raid_pokemon_evolution = :raid_pokemon_evolution, "+ - "ar_scan_eligible = :ar_scan_eligible, "+ - "power_up_level = :power_up_level, "+ - "power_up_points = :power_up_points, "+ - "power_up_end_timestamp = :power_up_end_timestamp,"+ - "description = :description,"+ - "defenders = :defenders,"+ - "rsvps = :rsvps "+ - "WHERE id = :id", gym, - ) - statsCollector.IncDbQuery("update gym", err) - if err != nil { - log.Errorf("Update gym %s", err) - } - _, _ = res, err + // Debug logging before queueing + if dbDebugEnabled { + if isNewRecord { + dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) + } else if gym.IsDirty() { + dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) } } + // Queue the write through the write-behind system + if writeBehindQueue != nil && gym.IsDirty() { + writeBehindQueue.Enqueue(gym, isNewRecord, 0) + } else if gym.IsDirty() { + // Fallback to direct write if queue not initialized + _ = gymWriteDB(db, gym, isNewRecord) + } + if config.Config.FortInMemory { fortRtreeUpdateGymOnSave(gym) } @@ -425,13 +376,81 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { if dbDebugEnabled { gym.changedFields = gym.changedFields[:0] } - if gym.IsNewRecord() { + if isNewRecord { gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) gym.newRecord = false } gym.ClearDirty() } +// gymWriteDB performs the actual database INSERT/UPDATE for a Gym +// This is called by both direct writes and the write-behind queue +func gymWriteDB(db db.DbDetails, gym *Gym, isNewRecord bool) error { + ctx := context.Background() + + if isNewRecord { + res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ + "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) + + statsCollector.IncDbQuery("insert gym", err) + if err != nil { + log.Errorf("insert gym: %s", err) + return err + } + _, _ = res, err + } else { + res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ + "lat = :lat, "+ + "lon = :lon, "+ + "name = :name, "+ + "url = :url, "+ + "last_modified_timestamp = :last_modified_timestamp, "+ + "raid_end_timestamp = :raid_end_timestamp, "+ + "raid_spawn_timestamp = :raid_spawn_timestamp, "+ + "raid_battle_timestamp = :raid_battle_timestamp, "+ + "updated = :updated, "+ + "raid_pokemon_id = :raid_pokemon_id, "+ + "guarding_pokemon_id = :guarding_pokemon_id, "+ + "guarding_pokemon_display = :guarding_pokemon_display, "+ + "available_slots = :available_slots, "+ + "team_id = :team_id, "+ + "raid_level = :raid_level, "+ + "enabled = :enabled, "+ + "ex_raid_eligible = :ex_raid_eligible, "+ + "in_battle = :in_battle, "+ + "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ + "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ + "raid_pokemon_form = :raid_pokemon_form, "+ + "raid_pokemon_alignment = :raid_pokemon_alignment, "+ + "raid_pokemon_cp = :raid_pokemon_cp, "+ + "raid_is_exclusive = :raid_is_exclusive, "+ + "cell_id = :cell_id, "+ + "deleted = :deleted, "+ + "total_cp = :total_cp, "+ + "raid_pokemon_gender = :raid_pokemon_gender, "+ + "sponsor_id = :sponsor_id, "+ + "partner_id = :partner_id, "+ + "raid_pokemon_costume = :raid_pokemon_costume, "+ + "raid_pokemon_evolution = :raid_pokemon_evolution, "+ + "ar_scan_eligible = :ar_scan_eligible, "+ + "power_up_level = :power_up_level, "+ + "power_up_points = :power_up_points, "+ + "power_up_end_timestamp = :power_up_end_timestamp,"+ + "description = :description,"+ + "defenders = :defenders,"+ + "rsvps = :rsvps "+ + "WHERE id = :id", gym, + ) + statsCollector.IncDbQuery("update gym", err) + if err != nil { + log.Errorf("Update gym %s", err) + return err + } + _, _ = res, err + } + return nil +} + func updateGymGetMapFortCache(gym *Gym, skipName bool) { storedGetMapFort := getMapFortsCache.Get(gym.Id) if storedGetMapFort != nil { diff --git a/decoder/gym_writeable.go b/decoder/gym_writeable.go new file mode 100644 index 00000000..91f16e1b --- /dev/null +++ b/decoder/gym_writeable.go @@ -0,0 +1,27 @@ +package decoder + +import ( + "golbat/db" + "golbat/decoder/writebehind" +) + +// Ensure Gym implements Writeable +var _ writebehind.Writeable = (*Gym)(nil) + +// WriteKey returns a unique key for this Gym (for squashing) +func (g *Gym) WriteKey() string { + return "gym:" + g.Id +} + +// WriteType returns the entity type name (for metrics) +func (g *Gym) WriteType() string { + return "gym" +} + +// WriteToDB performs the actual database write for this Gym +// This delegates to the shared direct write function +func (g *Gym) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { + g.Lock() + defer g.Unlock() + return gymWriteDB(dbDetails, g, isNewRecord) +} diff --git a/decoder/incident_state.go b/decoder/incident_state.go index fda4cc29..c56b43a7 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -116,23 +116,62 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident incident.SetUpdated(time.Now().Unix()) - if incident.IsNewRecord() { - if dbDebugEnabled { + // Capture isNewRecord before state changes + isNewRecord := incident.IsNewRecord() + + // Debug logging before queueing + if dbDebugEnabled { + if isNewRecord { dbDebugLog("INSERT", "Incident", incident.Id, incident.changedFields) + } else { + dbDebugLog("UPDATE", "Incident", incident.Id, incident.changedFields) } + } + + // Queue the write through the write-behind system + if writeBehindQueue != nil { + writeBehindQueue.Enqueue(incident, isNewRecord, 0) + } else { + // Fallback to direct write if queue not initialized + _ = incidentWriteDB(db, incident, isNewRecord) + } + + createIncidentWebhooks(ctx, db, incident) + + var stopLat, stopLon float64 + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + stopLat, stopLon = stop.Lat, stop.Lon + unlock() + } + + areas := MatchStatsGeofence(stopLat, stopLon) + updateIncidentStats(incident, areas) + + if dbDebugEnabled { + incident.changedFields = incident.changedFields[:0] + } + incident.ClearDirty() + if isNewRecord { + incident.newRecord = false + incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) + } +} + +// incidentWriteDB performs the actual database INSERT/UPDATE for an Incident +// This is called by both direct writes and the write-behind queue +func incidentWriteDB(db db.DbDetails, incident *Incident, isNewRecord bool) error { + if isNewRecord { res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) + statsCollector.IncDbQuery("insert incident", err) if err != nil { log.Errorf("insert incident: %s", err) - return + return err } - statsCollector.IncDbQuery("insert incident", err) _, _ = res, err } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Incident", incident.Id, incident.changedFields) - } res, err := db.GeneralDb.NamedExec("UPDATE incident SET "+ "start = :start, "+ "expiration = :expiration, "+ @@ -152,27 +191,11 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident statsCollector.IncDbQuery("update incident", err) if err != nil { log.Errorf("Update incident %s", err) + return err } _, _ = res, err } - - createIncidentWebhooks(ctx, db, incident) - - var stopLat, stopLon float64 - stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) - if stop != nil { - stopLat, stopLon = stop.Lat, stop.Lon - unlock() - } - - areas := MatchStatsGeofence(stopLat, stopLon) - updateIncidentStats(incident, areas) - - incident.ClearDirty() - if incident.IsNewRecord() { - incident.newRecord = false - incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) - } + return nil } func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { diff --git a/decoder/incident_writeable.go b/decoder/incident_writeable.go new file mode 100644 index 00000000..9924d7f6 --- /dev/null +++ b/decoder/incident_writeable.go @@ -0,0 +1,27 @@ +package decoder + +import ( + "golbat/db" + "golbat/decoder/writebehind" +) + +// Ensure Incident implements Writeable +var _ writebehind.Writeable = (*Incident)(nil) + +// WriteKey returns a unique key for this Incident (for squashing) +func (i *Incident) WriteKey() string { + return "incident:" + i.Id +} + +// WriteType returns the entity type name (for metrics) +func (i *Incident) WriteType() string { + return "incident" +} + +// WriteToDB performs the actual database write for this Incident +// This delegates to the shared direct write function +func (i *Incident) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { + i.Lock() + defer i.Unlock() + return incidentWriteDB(dbDetails, i, isNewRecord) +} diff --git a/decoder/main.go b/decoder/main.go index 112747e5..ef42e94c 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -13,6 +13,8 @@ import ( log "github.com/sirupsen/logrus" "golbat/config" + "golbat/db" + "golbat/decoder/writebehind" "golbat/geo" "golbat/pogo" "golbat/stats_collector" @@ -71,6 +73,12 @@ var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto var ProactiveIVSwitchSem chan bool +// writeBehindQueue is the global write-behind queue for database writes +var writeBehindQueue *writebehind.Queue + +// s2CellAccumulator is the global S2Cell batch accumulator +var s2CellAccumulator *writebehind.S2CellAccumulator + var ohbem *gohbem.Ohbem func init() { @@ -284,6 +292,61 @@ func SetStatsCollector(collector stats_collector.StatsCollector) { statsCollector = collector } +// InitWriteBehindQueue initializes the write-behind queue +// Should be called after SetStatsCollector +func InitWriteBehindQueue(ctx context.Context, dbDetails db.DbDetails) { + cfg := writebehind.QueueConfig{ + StartupDelaySeconds: config.Config.Tuning.WriteBehindStartupDelay, + RateLimit: config.Config.Tuning.WriteBehindRateLimit, + BurstCapacity: config.Config.Tuning.WriteBehindBurstCapacity, + } + + writeBehindQueue = writebehind.NewQueue(cfg, dbDetails, statsCollector) + + log.Infof("Write-behind queue initialized: startup_delay=%ds, rate_limit=%d/s, burst=%d", + cfg.StartupDelaySeconds, cfg.RateLimit, cfg.BurstCapacity) + + // Start the processing loop in a goroutine + go writeBehindQueue.ProcessLoop(ctx) +} + +// GetWriteBehindQueue returns the global write-behind queue +func GetWriteBehindQueue() *writebehind.Queue { + return writeBehindQueue +} + +// FlushWriteBehindQueue flushes all pending writes (for shutdown) +func FlushWriteBehindQueue() { + if writeBehindQueue != nil { + writeBehindQueue.Flush() + } +} + +// InitS2CellAccumulator initializes the S2Cell batch accumulator +// Should be called after SetStatsCollector +func InitS2CellAccumulator(ctx context.Context, dbDetails db.DbDetails) { + cfg := writebehind.QueueConfig{ + StartupDelaySeconds: config.Config.Tuning.WriteBehindStartupDelay, + RateLimit: config.Config.Tuning.WriteBehindRateLimit, + BurstCapacity: config.Config.Tuning.WriteBehindBurstCapacity, + } + + s2CellAccumulator = writebehind.NewS2CellAccumulator(cfg, dbDetails, statsCollector, s2CellBatchWrite) + + log.Infof("S2Cell accumulator initialized: startup_delay=%ds, rate_limit=%d/s, burst=%d", + cfg.StartupDelaySeconds, cfg.RateLimit, cfg.BurstCapacity) + + // Start the processing loop + s2CellAccumulator.Start(ctx) +} + +// FlushS2CellAccumulator flushes all pending S2Cell writes (for shutdown) +func FlushS2CellAccumulator() { + if s2CellAccumulator != nil { + s2CellAccumulator.Flush() + } +} + // GetUpdateThreshold returns the number of seconds that should be used as a // debounce/last-seen threshold. Pass the default seconds for normal operation // If ReduceUpdates is enabled in the loaded config.Config, this returns 43200 (12 hours). diff --git a/decoder/pending_pokemon.go b/decoder/pending_pokemon.go deleted file mode 100644 index 39042db3..00000000 --- a/decoder/pending_pokemon.go +++ /dev/null @@ -1,168 +0,0 @@ -package decoder - -import ( - "context" - "sync" - "time" - - "golbat/db" - "golbat/pogo" - - log "github.com/sirupsen/logrus" -) - -// PendingPokemon stores wild pokemon data awaiting a potential encounter -type PendingPokemon struct { - EncounterId uint64 - WildPokemon *pogo.WildPokemonProto - CellId int64 - TimestampMs int64 - UpdateTime int64 - WeatherLookup map[int64]pogo.GameplayWeatherProto_WeatherCondition - Username string - ReceivedAt time.Time -} - -// PokemonPendingQueue manages pokemon awaiting encounter data -type PokemonPendingQueue struct { - mu sync.RWMutex - pending map[uint64]*PendingPokemon - timeout time.Duration -} - -// NewPokemonPendingQueue creates a new pending queue with the specified timeout -func NewPokemonPendingQueue(timeout time.Duration) *PokemonPendingQueue { - return &PokemonPendingQueue{ - pending: make(map[uint64]*PendingPokemon), - timeout: timeout, - } -} - -// AddPending stores a wild pokemon awaiting encounter data. -// Returns true if the pokemon was added, false if it already exists. -func (q *PokemonPendingQueue) AddPending(p *PendingPokemon) bool { - q.mu.Lock() - defer q.mu.Unlock() - - // Only add if not already present (first sighting wins) - if _, exists := q.pending[p.EncounterId]; exists { - return false - } - - p.ReceivedAt = time.Now() - q.pending[p.EncounterId] = p - return true -} - -// TryComplete attempts to retrieve and remove a pending pokemon for an encounter. -// Returns the pending pokemon and true if found, nil and false otherwise. -func (q *PokemonPendingQueue) TryComplete(encounterId uint64) (*PendingPokemon, bool) { - q.mu.Lock() - defer q.mu.Unlock() - - p, exists := q.pending[encounterId] - if exists { - delete(q.pending, encounterId) - } - return p, exists -} - -// Remove removes a pending pokemon without processing it. -func (q *PokemonPendingQueue) Remove(encounterId uint64) { - q.mu.Lock() - delete(q.pending, encounterId) - q.mu.Unlock() -} - -// Size returns the current number of pending pokemon -func (q *PokemonPendingQueue) Size() int { - q.mu.RLock() - defer q.mu.RUnlock() - return len(q.pending) -} - -// collectExpired removes and returns all entries older than timeout -func (q *PokemonPendingQueue) collectExpired() []*PendingPokemon { - cutoff := time.Now().Add(-q.timeout) - - q.mu.Lock() - defer q.mu.Unlock() - - var expired []*PendingPokemon - for id, p := range q.pending { - if p.ReceivedAt.Before(cutoff) { - expired = append(expired, p) - delete(q.pending, id) - } - } - - return expired -} - -// StartSweeper starts a background goroutine that processes expired entries -func (q *PokemonPendingQueue) StartSweeper(ctx context.Context, interval time.Duration, dbDetails db.DbDetails) { - go func() { - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - log.Info("Pokemon pending queue sweeper stopped") - return - case <-ticker.C: - expired := q.collectExpired() - if len(expired) > 0 { - log.Debugf("Processing %d expired pending pokemon", len(expired)) - q.processExpired(ctx, dbDetails, expired) - } - } - } - }() -} - -// processExpired handles pokemon that didn't receive an encounter within the timeout -func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.DbDetails, expired []*PendingPokemon) { - for _, p := range expired { - // Check for shutdown signal between iterations - if ctx.Err() != nil { - log.Debug("Context cancelled, stopping expired pokemon processing") - return - } - - processCtx, cancel := context.WithTimeout(ctx, 3*time.Second) - - pokemon, unlock, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) - if err != nil { - log.Errorf("getOrCreatePokemonRecord in sweeper: %s", err) - cancel() - continue - } - - // Update if there is still a change required & this update is the most recent - if pokemon.wildSignificantUpdate(p.WildPokemon, p.UpdateTime) && pokemon.Updated.ValueOrZero() < p.UpdateTime { - log.Debugf("DELAYED UPDATE: Updating pokemon %d from wild (sweeper)", p.EncounterId) - - pokemon.updateFromWild(processCtx, dbDetails, p.WildPokemon, p.CellId, p.WeatherLookup, p.TimestampMs, p.Username) - savePokemonRecordAsAtTime(processCtx, dbDetails, pokemon, false, true, true, p.UpdateTime) - } - - unlock() - cancel() - } -} - -// Global pending queue instance -var pokemonPendingQueue *PokemonPendingQueue - -// InitPokemonPendingQueue initializes the global pending queue -func InitPokemonPendingQueue(ctx context.Context, dbDetails db.DbDetails, timeout time.Duration, sweepInterval time.Duration) { - pokemonPendingQueue = NewPokemonPendingQueue(timeout) - pokemonPendingQueue.StartSweeper(ctx, sweepInterval, dbDetails) - log.Infof("Pokemon pending queue started with %v timeout and %v sweep interval", timeout, sweepInterval) -} - -// GetPokemonPendingQueue returns the global pending queue instance -func GetPokemonPendingQueue() *PokemonPendingQueue { - return pokemonPendingQueue -} diff --git a/decoder/pokemon_process.go b/decoder/pokemon_process.go index 0d9585ad..ee6f460c 100644 --- a/decoder/pokemon_process.go +++ b/decoder/pokemon_process.go @@ -19,11 +19,6 @@ func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, encounterId := encounter.Pokemon.EncounterId - // Remove from pending queue - encounter arrived so no need for delayed wild update - if pokemonPendingQueue != nil { - pokemonPendingQueue.Remove(encounterId) - } - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Errorf("Error pokemon [%d]: %s", encounterId, err) diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index dcfb2840..44faac02 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "strconv" + "time" "golbat/config" "golbat/db" @@ -19,6 +20,9 @@ import ( "google.golang.org/protobuf/proto" ) +// wildPokemonDelay is how long wild Pokemon wait for encounter data before writing +const wildPokemonDelay = 30 * time.Second + // peekPokemonRecordReadOnly acquires lock, does NOT take snapshot. // Use for read-only checks which will not cause a backing database lookup // Caller must use returned unlock function @@ -178,14 +182,6 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po return } - // uncomment to debug excessive writes - //if !pokemon.isNewRecord() && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { - // log.Errorf("Why are we updating this? %s", cmp.Diff(oldPokemon, pokemon, cmp.Options{ - // ignoreNearFloats, ignoreNearNullFloats, - // cmpopts.IgnoreFields(Pokemon{}, "Username", "Iv", "Pvp"), - // })) - //} - if pokemon.FirstSeenTimestamp == 0 { pokemon.FirstSeenTimestamp = now } @@ -232,9 +228,12 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } log.Debugf("Updating pokemon [%d] from %s->%s - newRecord: %t", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero(), pokemon.isNewRecord()) - //log.Println(cmp.Diff(oldPokemon, pokemon)) + + // Capture isNewRecord before any state changes + isNewRecord := pokemon.isNewRecord() if writeDB && !config.Config.PokemonMemoryOnly { + // Prepare internal data if needed (must happen before queueing) if isEncounter && config.Config.PokemonInternalToDb { unboosted, boosted, strong := pokemon.locateAllScans() if unboosted != nil && boosted != nil { @@ -251,93 +250,30 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po log.Errorf("[POKEMON] Failed to marshal internal data for %d, data may be lost: %s", pokemon.Id, err) } } - if pokemon.isNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) - } - pvpField, pvpValue := "", "" - if changePvpField { - pvpField, pvpValue = "pvp, ", ":pvp, " - } - res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("INSERT INTO pokemon (id, pokemon_id, lat, lon,"+ - "spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, move_1, move_2,"+ - "gender, form, cp, level, strong, weather, costume, weight, height, size,"+ - "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id,"+ - "expire_timestamp_verified, shiny, username, %s is_event, seen_type) "+ - "VALUES (\"%d\", :pokemon_id, :lat, :lon, :spawn_id, :expire_timestamp, :atk_iv, :def_iv, :sta_iv,"+ - ":golbat_internal, :iv, :move_1, :move_2, :gender, :form, :cp, :level, :strong, :weather, :costume,"+ - ":weight, :height, :size, :display_pokemon_id, :is_ditto, :pokestop_id, :updated,"+ - ":first_seen_timestamp, :changed, :cell_id, :expire_timestamp_verified, :shiny, :username, %s :is_event,"+ - ":seen_type)", pvpField, pokemon.Id, pvpValue), pokemon) - - statsCollector.IncDbQuery("insert pokemon", err) - if err != nil { - log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) - log.Errorf("Full structure: %+v", pokemon) - pokemonCache.Delete(pokemon.Id) - // Force reload of pokemon from database - return - } - rows, rowsErr := res.RowsAffected() - log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) - } else { - if dbDebugEnabled { + // Debug logging happens here, before queueing + if dbDebugEnabled { + if isNewRecord { + dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } else { dbDebugLog("UPDATE", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) } - pvpUpdate := "" - if changePvpField { - pvpUpdate = "pvp = :pvp, " - } - res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("UPDATE pokemon SET "+ - "pokestop_id = :pokestop_id, "+ - "spawn_id = :spawn_id, "+ - "lat = :lat, "+ - "lon = :lon, "+ - "weight = :weight, "+ - "height = :height, "+ - "size = :size, "+ - "expire_timestamp = :expire_timestamp, "+ - "updated = :updated, "+ - "pokemon_id = :pokemon_id, "+ - "move_1 = :move_1, "+ - "move_2 = :move_2, "+ - "gender = :gender, "+ - "cp = :cp, "+ - "atk_iv = :atk_iv, "+ - "def_iv = :def_iv, "+ - "sta_iv = :sta_iv, "+ - "golbat_internal = :golbat_internal,"+ - "iv = :iv,"+ - "form = :form, "+ - "level = :level, "+ - "strong = :strong, "+ - "weather = :weather, "+ - "costume = :costume, "+ - "first_seen_timestamp = :first_seen_timestamp, "+ - "changed = :changed, "+ - "cell_id = :cell_id, "+ - "expire_timestamp_verified = :expire_timestamp_verified, "+ - "display_pokemon_id = :display_pokemon_id, "+ - "is_ditto = :is_ditto, "+ - "seen_type = :seen_type, "+ - "shiny = :shiny, "+ - "username = :username, "+ - "%s"+ - "is_event = :is_event "+ - "WHERE id = \"%d\"", pvpUpdate, pokemon.Id), pokemon, - ) - statsCollector.IncDbQuery("update pokemon", err) - if err != nil { - log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) - log.Errorf("Full structure: %+v", pokemon) - pokemonCache.Delete(pokemon.Id) - // Force reload of pokemon from database - - return + } + + // Queue the write through the write-behind system + if writeBehindQueue != nil { + // Determine delay based on seen type + // Wild/nearby Pokemon wait for potential encounter data, encounter writes immediately + delay := time.Duration(0) + seenType := pokemon.SeenType.ValueOrZero() + if seenType == SeenType_Wild || seenType == SeenType_LureWild || + seenType == SeenType_Cell || seenType == SeenType_NearbyStop { + delay = wildPokemonDelay } - rows, rowsErr := res.RowsAffected() - log.Debugf("Updating pokemon [%d] after update res = %d %v", pokemon.Id, rows, rowsErr) + writeBehindQueue.Enqueue(pokemon, isNewRecord, delay) + } else { + // Fallback to direct write if queue not initialized + _ = pokemonWriteDB(db, pokemon, isNewRecord) } } else { if dbDebugEnabled { @@ -345,8 +281,8 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } - // Update pokemon rtree - if pokemon.isNewRecord() { + // Update pokemon rtree (immediate, not queued) + if isNewRecord { addPokemonToTree(pokemon) } else if pokemon.Lat != pokemon.oldValues.Lat || pokemon.Lon != pokemon.oldValues.Lon { // Position changed - update R-tree by removing from old position and adding to new @@ -356,6 +292,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po updatePokemonLookup(pokemon, changePvpField, pvpResults) + // Webhooks and stats happen immediately (not queued) areas := MatchStatsGeofence(pokemon.Lat, pokemon.Lon) if webhook { createPokemonWebhooks(ctx, db, pokemon, areas) @@ -375,6 +312,88 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } +// pokemonWriteDB performs the actual database INSERT or UPDATE for a Pokemon +// This is called by both direct writes and the write-behind queue +func pokemonWriteDB(db db.DbDetails, pokemon *Pokemon, isNewRecord bool) error { + ctx := context.Background() + + if isNewRecord { + // Always include PVP field for inserts if it's set + pvpField, pvpValue := "", "" + if pokemon.Pvp.Valid { + pvpField, pvpValue = "pvp, ", ":pvp, " + } + _, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("INSERT INTO pokemon (id, pokemon_id, lat, lon,"+ + "spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, move_1, move_2,"+ + "gender, form, cp, level, strong, weather, costume, weight, height, size,"+ + "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id,"+ + "expire_timestamp_verified, shiny, username, %s is_event, seen_type) "+ + "VALUES (\"%d\", :pokemon_id, :lat, :lon, :spawn_id, :expire_timestamp, :atk_iv, :def_iv, :sta_iv,"+ + ":golbat_internal, :iv, :move_1, :move_2, :gender, :form, :cp, :level, :strong, :weather, :costume,"+ + ":weight, :height, :size, :display_pokemon_id, :is_ditto, :pokestop_id, :updated,"+ + ":first_seen_timestamp, :changed, :cell_id, :expire_timestamp_verified, :shiny, :username, %s :is_event,"+ + ":seen_type)", pvpField, pokemon.Id, pvpValue), pokemon) + + statsCollector.IncDbQuery("insert pokemon", err) + if err != nil { + log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) + pokemonCache.Delete(pokemon.Id) + return err + } + } else { + // Always include PVP field for updates if it's set + pvpUpdate := "" + if pokemon.Pvp.Valid { + pvpUpdate = "pvp = :pvp, " + } + _, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("UPDATE pokemon SET "+ + "pokestop_id = :pokestop_id, "+ + "spawn_id = :spawn_id, "+ + "lat = :lat, "+ + "lon = :lon, "+ + "weight = :weight, "+ + "height = :height, "+ + "size = :size, "+ + "expire_timestamp = :expire_timestamp, "+ + "updated = :updated, "+ + "pokemon_id = :pokemon_id, "+ + "move_1 = :move_1, "+ + "move_2 = :move_2, "+ + "gender = :gender, "+ + "cp = :cp, "+ + "atk_iv = :atk_iv, "+ + "def_iv = :def_iv, "+ + "sta_iv = :sta_iv, "+ + "golbat_internal = :golbat_internal,"+ + "iv = :iv,"+ + "form = :form, "+ + "level = :level, "+ + "strong = :strong, "+ + "weather = :weather, "+ + "costume = :costume, "+ + "first_seen_timestamp = :first_seen_timestamp, "+ + "changed = :changed, "+ + "cell_id = :cell_id, "+ + "expire_timestamp_verified = :expire_timestamp_verified, "+ + "display_pokemon_id = :display_pokemon_id, "+ + "is_ditto = :is_ditto, "+ + "seen_type = :seen_type, "+ + "shiny = :shiny, "+ + "username = :username, "+ + "%s"+ + "is_event = :is_event "+ + "WHERE id = \"%d\"", pvpUpdate, pokemon.Id), pokemon, + ) + statsCollector.IncDbQuery("update pokemon", err) + if err != nil { + log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) + pokemonCache.Delete(pokemon.Id) + return err + } + } + return nil +} + type PokemonWebhook struct { SpawnpointId string `json:"spawnpoint_id"` PokestopId string `json:"pokestop_id"` diff --git a/decoder/pokemon_writeable.go b/decoder/pokemon_writeable.go new file mode 100644 index 00000000..87f9f8c3 --- /dev/null +++ b/decoder/pokemon_writeable.go @@ -0,0 +1,29 @@ +package decoder + +import ( + "fmt" + + "golbat/db" + "golbat/decoder/writebehind" +) + +// Ensure Pokemon implements Writeable +var _ writebehind.Writeable = (*Pokemon)(nil) + +// WriteKey returns a unique key for this Pokemon (for squashing) +func (pokemon *Pokemon) WriteKey() string { + return fmt.Sprintf("pokemon:%d", pokemon.Id) +} + +// WriteType returns the entity type name (for metrics) +func (pokemon *Pokemon) WriteType() string { + return "pokemon" +} + +// WriteToDB performs the actual database write for this Pokemon +// This delegates to the shared direct write function +func (pokemon *Pokemon) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { + pokemon.Lock() + defer pokemon.Unlock() + return pokemonWriteDB(dbDetails, pokemon, isNewRecord) +} diff --git a/decoder/pokestop_process.go b/decoder/pokestop_process.go index 69dd78da..e2792fd2 100644 --- a/decoder/pokestop_process.go +++ b/decoder/pokestop_process.go @@ -61,13 +61,12 @@ func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.F func ClearQuestsWithinGeofence(ctx context.Context, dbDetails db.DbDetails, geofence *geojson.Feature) { started := time.Now() - rows, err := db.RemoveQuests(ctx, dbDetails, geofence) + count, err := RemoveQuestsWithinGeofence(ctx, dbDetails, geofence) if err != nil { log.Errorf("ClearQuest: Error removing quests: %s", err) return } - ClearPokestopCache() - log.Infof("ClearQuest: Removed quests from %d pokestops in %s", rows, time.Since(started)) + log.Infof("ClearQuest: Removed quests from %d pokestops in %s", count, time.Since(started)) } func GetQuestStatusWithGeofence(dbDetails db.DbDetails, geofence *geojson.Feature) db.QuestStatus { diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index b39e4d0c..28047201 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -9,6 +9,7 @@ import ( "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" + "github.com/paulmach/orb/geojson" log "github.com/sirupsen/logrus" "golbat/config" @@ -284,11 +285,52 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } pokestop.SetUpdated(now) - if pokestop.IsNewRecord() { - if dbDebugEnabled { + // Capture isNewRecord before state changes + isNewRecord := pokestop.IsNewRecord() + + // Debug logging happens here, before queueing + if dbDebugEnabled { + if isNewRecord { dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) + } else { + dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) } - res, err := db.GeneralDb.NamedExecContext(ctx, ` + } + + // Queue the write through the write-behind system (no delay for pokestops) + if writeBehindQueue != nil { + writeBehindQueue.Enqueue(pokestop, isNewRecord, 0) + } else { + // Fallback to direct write if queue not initialized + _ = pokestopWriteDB(db, pokestop, isNewRecord) + } + + if dbDebugEnabled { + pokestop.changedFields = pokestop.changedFields[:0] + } + + if config.Config.FortInMemory { + fortRtreeUpdatePokestopOnSave(pokestop) + } + + // Webhooks happen immediately (not queued) + createPokestopWebhooks(pokestop) + createPokestopFortWebhooks(pokestop) + + if isNewRecord { + pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + pokestop.newRecord = false + } + pokestop.ClearDirty() +} + +// pokestopWriteDB performs the actual database INSERT or UPDATE for a Pokestop +// This is called by both direct writes and the write-behind queue +func pokestopWriteDB(db db.DbDetails, pokestop *Pokestop, isNewRecord bool) error { + ctx := context.Background() + + if isNewRecord { + _, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO pokestop ( id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, quest_timestamp, quest_target, quest_conditions, quest_rewards, quest_template, quest_title, @@ -318,18 +360,12 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop pokestop) statsCollector.IncDbQuery("insert pokestop", err) - //log.Debugf("Insert pokestop %s %+v", pokestop.Id, pokestop) if err != nil { log.Errorf("insert pokestop: %s", err) - return + return err } - - _, _ = res, err } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, ` + _, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE pokestop SET lat = :lat, lon = :lon, @@ -386,29 +422,12 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop pokestop, ) statsCollector.IncDbQuery("update pokestop", err) - //log.Debugf("Update pokestop %s %+v", pokestop.Id, pokestop) if err != nil { log.Errorf("update pokestop %s: %s", pokestop.Id, err) - return + return err } - _ = res - } - if dbDebugEnabled { - pokestop.changedFields = pokestop.changedFields[:0] - } - - if config.Config.FortInMemory { - fortRtreeUpdatePokestopOnSave(pokestop) - } - - createPokestopWebhooks(pokestop) - createPokestopFortWebhooks(pokestop) - if pokestop.IsNewRecord() { - pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) - pokestop.newRecord = false } - pokestop.ClearDirty() - + return nil } func updatePokestopGetMapFortCache(pokestop *Pokestop) { @@ -420,3 +439,169 @@ func updatePokestopGetMapFortCache(pokestop *Pokestop) { log.Debugf("Updated Gym using stored getMapFort: %s", pokestop.Id) } } + +// RemoveQuestsWithinGeofence clears all quest fields for pokestops within a geofence +// Uses cache and write-behind queue for consistency +func RemoveQuestsWithinGeofence(ctx context.Context, dbDetails db.DbDetails, geofence *geojson.Feature) (int, error) { + bbox := geofence.Geometry.Bound() + bytes, err := geofence.MarshalJSON() + if err != nil { + return 0, err + } + + // Query for pokestop IDs within the geofence + var pokestopIds []string + err = dbDetails.GeneralDb.SelectContext(ctx, &pokestopIds, + "SELECT id FROM pokestop "+ + "WHERE lat >= ? AND lon >= ? AND lat <= ? AND lon <= ? AND enabled = 1 "+ + "AND ST_CONTAINS(ST_GeomFromGeoJSON('"+string(bytes)+"', 2, 0), POINT(lon, lat))", + bbox.Min.Lat(), bbox.Min.Lon(), bbox.Max.Lat(), bbox.Max.Lon()) + statsCollector.IncDbQuery("select pokestops for quest removal", err) + if err != nil { + return 0, err + } + + clearedCount := 0 + + for _, id := range pokestopIds { + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, dbDetails, id) + if err != nil { + log.Errorf("RemoveQuestsWithinGeofence: failed to get pokestop %s: %v", id, err) + continue + } + + // Clear regular quest fields + pokestop.SetQuestType(null.Int{}) + pokestop.SetQuestTimestamp(null.Int{}) + pokestop.SetQuestTarget(null.Int{}) + pokestop.SetQuestConditions(null.String{}) + pokestop.SetQuestRewards(null.String{}) + pokestop.SetQuestTemplate(null.String{}) + pokestop.SetQuestTitle(null.String{}) + pokestop.SetQuestExpiry(null.Int{}) + pokestop.SetQuestRewardType(null.Int{}) + pokestop.SetQuestItemId(null.Int{}) + pokestop.SetQuestRewardAmount(null.Int{}) + pokestop.SetQuestPokemonId(null.Int{}) + pokestop.SetQuestPokemonFormId(null.Int{}) + + // Clear alternative quest fields + pokestop.SetAlternativeQuestType(null.Int{}) + pokestop.SetAlternativeQuestTimestamp(null.Int{}) + pokestop.SetAlternativeQuestTarget(null.Int{}) + pokestop.SetAlternativeQuestConditions(null.String{}) + pokestop.SetAlternativeQuestRewards(null.String{}) + pokestop.SetAlternativeQuestTemplate(null.String{}) + pokestop.SetAlternativeQuestTitle(null.String{}) + pokestop.SetAlternativeQuestExpiry(null.Int{}) + pokestop.SetAlternativeQuestRewardType(null.Int{}) + pokestop.SetAlternativeQuestItemId(null.Int{}) + pokestop.SetAlternativeQuestRewardAmount(null.Int{}) + pokestop.SetAlternativeQuestPokemonId(null.Int{}) + pokestop.SetAlternativeQuestPokemonFormId(null.Int{}) + + if pokestop.IsDirty() { + savePokestopRecord(ctx, dbDetails, pokestop) + clearedCount++ + } + unlock() + } + + return clearedCount, nil +} + +// ExpireQuests finds pokestops with expired quests, clears quest fields, and saves through write-behind queue +func ExpireQuests(ctx context.Context, dbDetails db.DbDetails) (int, error) { + now := time.Now().Unix() + + // Query for pokestop IDs with expired regular quests + var expiredQuestIds []string + err := dbDetails.GeneralDb.SelectContext(ctx, &expiredQuestIds, + "SELECT id FROM pokestop WHERE quest_expiry IS NOT NULL AND quest_expiry < ?", now) + statsCollector.IncDbQuery("select expired quests", err) + if err != nil { + return 0, err + } + + // Query for pokestop IDs with expired alternative quests + var expiredAltQuestIds []string + err = dbDetails.GeneralDb.SelectContext(ctx, &expiredAltQuestIds, + "SELECT id FROM pokestop WHERE alternative_quest_expiry IS NOT NULL AND alternative_quest_expiry < ?", now) + statsCollector.IncDbQuery("select expired alt quests", err) + if err != nil { + return 0, err + } + + // Build sets for quick lookup + hasExpiredQuest := make(map[string]bool, len(expiredQuestIds)) + for _, id := range expiredQuestIds { + hasExpiredQuest[id] = true + } + + hasExpiredAltQuest := make(map[string]bool, len(expiredAltQuestIds)) + for _, id := range expiredAltQuestIds { + hasExpiredAltQuest[id] = true + } + + // Combine and deduplicate IDs + allIds := make(map[string]bool, len(expiredQuestIds)+len(expiredAltQuestIds)) + for _, id := range expiredQuestIds { + allIds[id] = true + } + for _, id := range expiredAltQuestIds { + allIds[id] = true + } + + expiredCount := 0 + + // Process each pokestop once, clearing both quest types if needed + for id := range allIds { + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, dbDetails, id) + if err != nil { + log.Errorf("ExpireQuests: failed to get pokestop %s: %v", id, err) + continue + } + + // Clear regular quest fields if expired + if hasExpiredQuest[id] { + pokestop.SetQuestType(null.Int{}) + pokestop.SetQuestTimestamp(null.Int{}) + pokestop.SetQuestTarget(null.Int{}) + pokestop.SetQuestConditions(null.String{}) + pokestop.SetQuestRewards(null.String{}) + pokestop.SetQuestTemplate(null.String{}) + pokestop.SetQuestTitle(null.String{}) + pokestop.SetQuestExpiry(null.Int{}) + pokestop.SetQuestRewardType(null.Int{}) + pokestop.SetQuestItemId(null.Int{}) + pokestop.SetQuestRewardAmount(null.Int{}) + pokestop.SetQuestPokemonId(null.Int{}) + pokestop.SetQuestPokemonFormId(null.Int{}) + } + + // Clear alternative quest fields if expired + if hasExpiredAltQuest[id] { + pokestop.SetAlternativeQuestType(null.Int{}) + pokestop.SetAlternativeQuestTimestamp(null.Int{}) + pokestop.SetAlternativeQuestTarget(null.Int{}) + pokestop.SetAlternativeQuestConditions(null.String{}) + pokestop.SetAlternativeQuestRewards(null.String{}) + pokestop.SetAlternativeQuestTemplate(null.String{}) + pokestop.SetAlternativeQuestTitle(null.String{}) + pokestop.SetAlternativeQuestExpiry(null.Int{}) + pokestop.SetAlternativeQuestRewardType(null.Int{}) + pokestop.SetAlternativeQuestItemId(null.Int{}) + pokestop.SetAlternativeQuestRewardAmount(null.Int{}) + pokestop.SetAlternativeQuestPokemonId(null.Int{}) + pokestop.SetAlternativeQuestPokemonFormId(null.Int{}) + } + + if pokestop.IsDirty() { + savePokestopRecord(ctx, dbDetails, pokestop) + expiredCount++ + } + unlock() + } + + return expiredCount, nil +} diff --git a/decoder/pokestop_writeable.go b/decoder/pokestop_writeable.go new file mode 100644 index 00000000..34a3fa4f --- /dev/null +++ b/decoder/pokestop_writeable.go @@ -0,0 +1,27 @@ +package decoder + +import ( + "golbat/db" + "golbat/decoder/writebehind" +) + +// Ensure Pokestop implements Writeable +var _ writebehind.Writeable = (*Pokestop)(nil) + +// WriteKey returns a unique key for this Pokestop (for squashing) +func (p *Pokestop) WriteKey() string { + return "pokestop:" + p.Id +} + +// WriteType returns the entity type name (for metrics) +func (p *Pokestop) WriteType() string { + return "pokestop" +} + +// WriteToDB performs the actual database write for this Pokestop +// This delegates to the shared direct write function +func (p *Pokestop) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { + p.Lock() + defer p.Unlock() + return pokestopWriteDB(dbDetails, p, isNewRecord) +} diff --git a/decoder/routes_state.go b/decoder/routes_state.go index 38a0d918..eefc5f6b 100644 --- a/decoder/routes_state.go +++ b/decoder/routes_state.go @@ -114,10 +114,45 @@ func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { route.SetUpdated(time.Now().Unix()) - if route.IsNewRecord() { - if dbDebugEnabled { + // Capture isNewRecord before state changes + isNewRecord := route.IsNewRecord() + + // Debug logging before queueing + if dbDebugEnabled { + if isNewRecord { dbDebugLog("INSERT", "Route", route.Id, route.changedFields) + } else { + dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) + } + } + + // Queue the write through the write-behind system + if writeBehindQueue != nil { + writeBehindQueue.Enqueue(route, isNewRecord, 0) + } else { + // Fallback to direct write if queue not initialized + if err := routeWriteDB(db, route, isNewRecord); err != nil { + return err } + } + + if dbDebugEnabled { + route.changedFields = route.changedFields[:0] + } + route.ClearDirty() + if isNewRecord { + routeCache.Set(route.Id, route, ttlcache.DefaultTTL) + route.newRecord = false + } + return nil +} + +// routeWriteDB performs the actual database INSERT/UPDATE for a Route +// This is called by both direct writes and the write-behind queue +func routeWriteDB(db db.DbDetails, route *Route, isNewRecord bool) error { + ctx := context.Background() + + if isNewRecord { _, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO route ( @@ -147,9 +182,6 @@ func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { return fmt.Errorf("insert route error: %w", err) } } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) - } _, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE route SET @@ -183,14 +215,5 @@ func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { return fmt.Errorf("update route error %w", err) } } - - if dbDebugEnabled { - route.changedFields = route.changedFields[:0] - } - route.ClearDirty() - if route.IsNewRecord() { - routeCache.Set(route.Id, route, ttlcache.DefaultTTL) - route.newRecord = false - } return nil } diff --git a/decoder/routes_writeable.go b/decoder/routes_writeable.go new file mode 100644 index 00000000..8f80d140 --- /dev/null +++ b/decoder/routes_writeable.go @@ -0,0 +1,27 @@ +package decoder + +import ( + "golbat/db" + "golbat/decoder/writebehind" +) + +// Ensure Route implements Writeable +var _ writebehind.Writeable = (*Route)(nil) + +// WriteKey returns a unique key for this Route (for squashing) +func (r *Route) WriteKey() string { + return "route:" + r.Id +} + +// WriteType returns the entity type name (for metrics) +func (r *Route) WriteType() string { + return "route" +} + +// WriteToDB performs the actual database write for this Route +// This delegates to the shared direct write function +func (r *Route) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { + r.Lock() + defer r.Unlock() + return routeWriteDB(dbDetails, r, isNewRecord) +} diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 60513492..12fe8ef2 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -7,6 +7,7 @@ import ( "time" "golbat/db" + "golbat/decoder/writebehind" "github.com/golang/geo/s2" "github.com/guregu/null/v6" @@ -33,7 +34,7 @@ type S2Cell struct { func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { now := time.Now().Unix() - var outputCellIds []*S2Cell + var cellsToWrite []*writebehind.S2CellData // prepare list of cells to update for _, cellId := range cellIds { @@ -57,36 +58,63 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { } s2Cell.Updated = now - outputCellIds = append(outputCellIds, s2Cell) + cellsToWrite = append(cellsToWrite, &writebehind.S2CellData{ + Id: s2Cell.Id, + Latitude: s2Cell.Latitude, + Longitude: s2Cell.Longitude, + Level: s2Cell.Level.ValueOrZero(), + Updated: s2Cell.Updated, + }) } - if len(outputCellIds) == 0 { + if len(cellsToWrite) == 0 { return } if dbDebugEnabled { var updatedCells []string - for _, s2cell := range outputCellIds { - updatedCells = append(updatedCells, strconv.FormatUint(s2cell.Id, 10)) + for _, cell := range cellsToWrite { + updatedCells = append(updatedCells, strconv.FormatUint(cell.Id, 10)) } log.Debugf("[DB_UPDATE] S2Cell Updated cells: %s", strings.Join(updatedCells, ",")) } - // run bulk query + // Queue through the accumulator if available + if s2CellAccumulator != nil { + s2CellAccumulator.Add(cellsToWrite) + } else { + // Fallback to direct write if accumulator not initialized + _ = s2CellBatchWrite(db, cellsToWrite) + } +} + +// s2CellBatchWrite performs the actual batch database write for S2Cells +// This is called by both direct writes and the accumulator +func s2CellBatchWrite(db db.DbDetails, cells []*writebehind.S2CellData) error { + ctx := context.Background() + + // Convert to slice of S2Cell for the query + s2Cells := make([]*S2Cell, len(cells)) + for i, cell := range cells { + s2Cells[i] = &S2Cell{ + Id: cell.Id, + Latitude: cell.Latitude, + Longitude: cell.Longitude, + Level: null.IntFrom(cell.Level), + Updated: cell.Updated, + } + } + _, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO s2cell (id, center_lat, center_lon, level, updated) VALUES (:id, :center_lat, :center_lon, :level, :updated) ON DUPLICATE KEY UPDATE updated=VALUES(updated) - `, outputCellIds) + `, s2Cells) statsCollector.IncDbQuery("insert s2cell", err) if err != nil { - log.Errorf("saveS2CellRecords: %s", err) - return + log.Errorf("s2CellBatchWrite: %s", err) + return err } - - // since cache is now a pointer, ttl will already have been updated - //for _, cellId := range outputCellIds { - // s2CellCache.Set(cellId.Id, cellId, ttlcache.DefaultTTL) - //} + return nil } diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index cab83009..0883360d 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -290,14 +290,42 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi spawnpoint.SetUpdated(time.Now().Unix()) // ensure future updates are set correctly spawnpoint.SetLastSeen(time.Now().Unix()) // ensure future updates are set correctly + // Capture isNewRecord before state changes + isNewRecord := spawnpoint.IsNewRecord() + + // Debug logging happens here, before queueing if dbDebugEnabled { - if spawnpoint.IsNewRecord() { + if isNewRecord { dbDebugLog("INSERT", "Spawnpoint", strconv.FormatInt(spawnpoint.Id, 10), spawnpoint.changedFields) } else { dbDebugLog("UPDATE", "Spawnpoint", strconv.FormatInt(spawnpoint.Id, 10), spawnpoint.changedFields) } } + // Queue the write through the write-behind system (no delay for spawnpoints) + if writeBehindQueue != nil { + writeBehindQueue.Enqueue(spawnpoint, isNewRecord, 0) + } else { + // Fallback to direct write if queue not initialized + _ = spawnpointWriteDB(db, spawnpoint) + } + + if dbDebugEnabled { + spawnpoint.changedFields = spawnpoint.changedFields[:0] + } + spawnpoint.ClearDirty() + if isNewRecord { + spawnpoint.newRecord = false + spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) + } +} + +// spawnpointWriteDB performs the actual database INSERT/UPDATE for a Spawnpoint +// This is called by both direct writes and the write-behind queue +// Spawnpoint uses UPSERT pattern so isNewRecord is not needed +func spawnpointWriteDB(db db.DbDetails, spawnpoint *Spawnpoint) error { + ctx := context.Background() + _, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO spawnpoint (id, lat, lon, updated, last_seen, despawn_sec)"+ "VALUES (:id, :lat, :lon, :updated, :last_seen, :despawn_sec)"+ "ON DUPLICATE KEY UPDATE "+ @@ -310,14 +338,9 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi statsCollector.IncDbQuery("insert spawnpoint", err) if err != nil { log.Errorf("Error updating spawnpoint %s", err) - return - } - - spawnpoint.ClearDirty() - if spawnpoint.IsNewRecord() { - spawnpoint.newRecord = false - spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) + return err } + return nil } // spawnpointSeen updates the last_seen timestamp for a spawnpoint. diff --git a/decoder/spawnpoint_writeable.go b/decoder/spawnpoint_writeable.go new file mode 100644 index 00000000..e85eb44d --- /dev/null +++ b/decoder/spawnpoint_writeable.go @@ -0,0 +1,29 @@ +package decoder + +import ( + "fmt" + + "golbat/db" + "golbat/decoder/writebehind" +) + +// Ensure Spawnpoint implements Writeable +var _ writebehind.Writeable = (*Spawnpoint)(nil) + +// WriteKey returns a unique key for this Spawnpoint (for squashing) +func (s *Spawnpoint) WriteKey() string { + return fmt.Sprintf("spawnpoint:%d", s.Id) +} + +// WriteType returns the entity type name (for metrics) +func (s *Spawnpoint) WriteType() string { + return "spawnpoint" +} + +// WriteToDB performs the actual database write for this Spawnpoint +// This delegates to the shared direct write function +func (s *Spawnpoint) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { + s.Lock() + defer s.Unlock() + return spawnpointWriteDB(dbDetails, s) +} diff --git a/decoder/station_state.go b/decoder/station_state.go index cd3e4b2f..e6ea9c86 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -150,10 +150,43 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { station.SetUpdated(now) - if station.IsNewRecord() { - if dbDebugEnabled { + // Capture isNewRecord before state changes + isNewRecord := station.IsNewRecord() + + // Debug logging before queueing + if dbDebugEnabled { + if isNewRecord { dbDebugLog("INSERT", "Station", station.Id, station.changedFields) + } else { + dbDebugLog("UPDATE", "Station", station.Id, station.changedFields) } + } + + // Queue the write through the write-behind system + if writeBehindQueue != nil { + writeBehindQueue.Enqueue(station, isNewRecord, 0) + } else { + // Fallback to direct write if queue not initialized + _ = stationWriteDB(db, station, isNewRecord) + } + + if dbDebugEnabled { + station.changedFields = station.changedFields[:0] + } + station.ClearDirty() + createStationWebhooks(station) + if isNewRecord { + stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + station.newRecord = false + } +} + +// stationWriteDB performs the actual database INSERT/UPDATE for a Station +// This is called by both direct writes and the write-behind queue +func stationWriteDB(db db.DbDetails, station *Station, isNewRecord bool) error { + ctx := context.Background() + + if isNewRecord { res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) @@ -163,13 +196,10 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { statsCollector.IncDbQuery("insert station", err) if err != nil { log.Errorf("insert station: %s", err) - return + return err } _, _ = res, err } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Station", station.Id, station.changedFields) - } res, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE station SET @@ -205,19 +235,11 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { statsCollector.IncDbQuery("update station", err) if err != nil { log.Errorf("Update station %s", err) + return err } _, _ = res, err } - - if dbDebugEnabled { - station.changedFields = station.changedFields[:0] - } - station.ClearDirty() - createStationWebhooks(station) - if station.IsNewRecord() { - stationCache.Set(station.Id, station, ttlcache.DefaultTTL) - station.newRecord = false - } + return nil } func createStationWebhooks(station *Station) { diff --git a/decoder/station_writeable.go b/decoder/station_writeable.go new file mode 100644 index 00000000..ab12094d --- /dev/null +++ b/decoder/station_writeable.go @@ -0,0 +1,27 @@ +package decoder + +import ( + "golbat/db" + "golbat/decoder/writebehind" +) + +// Ensure Station implements Writeable +var _ writebehind.Writeable = (*Station)(nil) + +// WriteKey returns a unique key for this Station (for squashing) +func (s *Station) WriteKey() string { + return "station:" + s.Id +} + +// WriteType returns the entity type name (for metrics) +func (s *Station) WriteType() string { + return "station" +} + +// WriteToDB performs the actual database write for this Station +// This delegates to the shared direct write function +func (s *Station) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { + s.Lock() + defer s.Unlock() + return stationWriteDB(dbDetails, s, isNewRecord) +} diff --git a/decoder/tappable_state.go b/decoder/tappable_state.go index 517c22ab..43e9dd4f 100644 --- a/decoder/tappable_state.go +++ b/decoder/tappable_state.go @@ -103,10 +103,42 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap now := time.Now().Unix() tappable.SetUpdated(now) - if tappable.IsNewRecord() { - if dbDebugEnabled { + // Capture isNewRecord before state changes + isNewRecord := tappable.IsNewRecord() + + // Debug logging before queueing + if dbDebugEnabled { + if isNewRecord { dbDebugLog("INSERT", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } else { + dbDebugLog("UPDATE", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) } + } + + // Queue the write through the write-behind system + if writeBehindQueue != nil { + writeBehindQueue.Enqueue(tappable, isNewRecord, 0) + } else { + // Fallback to direct write if queue not initialized + _ = tappableWriteDB(details, tappable, isNewRecord) + } + + if dbDebugEnabled { + tappable.changedFields = tappable.changedFields[:0] + } + tappable.ClearDirty() + if isNewRecord { + tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) + tappable.newRecord = false + } +} + +// tappableWriteDB performs the actual database INSERT/UPDATE for a Tappable +// This is called by both direct writes and the write-behind queue +func tappableWriteDB(details db.DbDetails, tappable *Tappable, isNewRecord bool) error { + ctx := context.Background() + + if isNewRecord { res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` INSERT INTO tappable ( id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated @@ -117,13 +149,10 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap statsCollector.IncDbQuery("insert tappable", err) if err != nil { log.Errorf("insert tappable %d: %s", tappable.Id, err) - return + return err } _ = res } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) - } res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` UPDATE tappable SET lat = :lat, @@ -142,16 +171,9 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap statsCollector.IncDbQuery("update tappable", err) if err != nil { log.Errorf("update tappable %d: %s", tappable.Id, err) - return + return err } _ = res } - if dbDebugEnabled { - tappable.changedFields = tappable.changedFields[:0] - } - tappable.ClearDirty() - if tappable.IsNewRecord() { - tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) - tappable.newRecord = false - } + return nil } diff --git a/decoder/tappable_writeable.go b/decoder/tappable_writeable.go new file mode 100644 index 00000000..c70d47d9 --- /dev/null +++ b/decoder/tappable_writeable.go @@ -0,0 +1,29 @@ +package decoder + +import ( + "fmt" + + "golbat/db" + "golbat/decoder/writebehind" +) + +// Ensure Tappable implements Writeable +var _ writebehind.Writeable = (*Tappable)(nil) + +// WriteKey returns a unique key for this Tappable (for squashing) +func (t *Tappable) WriteKey() string { + return fmt.Sprintf("tappable:%d", t.Id) +} + +// WriteType returns the entity type name (for metrics) +func (t *Tappable) WriteType() string { + return "tappable" +} + +// WriteToDB performs the actual database write for this Tappable +// This delegates to the shared direct write function +func (t *Tappable) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { + t.Lock() + defer t.Unlock() + return tappableWriteDB(dbDetails, t, isNewRecord) +} diff --git a/decoder/writebehind/processor.go b/decoder/writebehind/processor.go new file mode 100644 index 00000000..60e094c7 --- /dev/null +++ b/decoder/writebehind/processor.go @@ -0,0 +1,74 @@ +package writebehind + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" +) + +const ( + // processingInterval is how often the processor checks for work + processingInterval = 100 * time.Millisecond + + // batchSize is the maximum number of entries to process per tick + batchSize = 50 + + // statusLogInterval is how often to log queue status + statusLogInterval = 30 * time.Second +) + +// ProcessLoop runs the main processing loop for the queue +// This should be called in a goroutine +func (q *Queue) ProcessLoop(ctx context.Context) { + ticker := time.NewTicker(processingInterval) + defer ticker.Stop() + + statusTicker := time.NewTicker(statusLogInterval) + defer statusTicker.Stop() + + for { + select { + case <-ctx.Done(): + log.Info("Write-behind processor shutting down, flushing queue...") + q.Flush() + return + case <-statusTicker.C: + queueSize := q.Size() + log.Infof("Write-behind queue length: %d", queueSize) + case <-ticker.C: + if q.checkWarmup() { + q.processBatch(ctx) + } + } + } +} + +// processBatch processes a batch of entries from the queue +func (q *Queue) processBatch(ctx context.Context) { + entries := q.getReadyEntries(batchSize) + if len(entries) == 0 { + return + } + + for _, entry := range entries { + // Check for context cancellation + select { + case <-ctx.Done(): + return + default: + } + + // Apply rate limiting + if !q.rateLimiter.TryAcquire(1) { + q.stats.IncWriteBehindRateLimited(entry.Entity.WriteType()) + log.Debugf("Write-behind rate limited for %s", entry.Key) + waitTime := q.rateLimiter.WaitAcquire(1) + if waitTime > time.Second { + log.Warnf("Write-behind rate limited for %s, waited %v", entry.Key, waitTime) + } + } + + q.writeEntry(entry) + } +} diff --git a/decoder/writebehind/queue.go b/decoder/writebehind/queue.go new file mode 100644 index 00000000..8e679eb5 --- /dev/null +++ b/decoder/writebehind/queue.go @@ -0,0 +1,197 @@ +package writebehind + +import ( + "sync" + "time" + + "golbat/db" + "golbat/stats_collector" + + log "github.com/sirupsen/logrus" +) + +// Queue is the write-behind queue that buffers database writes +type Queue struct { + mu sync.RWMutex + pending map[string]*QueueEntry // key -> entry for O(1) lookup + orderedKeys []string // FIFO order for processing + + rateLimiter *TokenBucket + warmupComplete bool + startTime time.Time + + config QueueConfig + db db.DbDetails + stats stats_collector.StatsCollector +} + +// NewQueue creates a new write-behind queue +func NewQueue(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.StatsCollector) *Queue { + return &Queue{ + pending: make(map[string]*QueueEntry), + orderedKeys: make([]string, 0, 1024), + rateLimiter: NewTokenBucket(cfg.RateLimit, cfg.BurstCapacity), + warmupComplete: false, + startTime: time.Now(), + config: cfg, + db: dbDetails, + stats: stats, + } +} + +// Enqueue adds or updates an entity write with a specified delay +// If an entry already exists for the same key: +// - IsNewRecord is preserved if either is true (INSERT takes priority) +// - Delay is updated to the minimum of existing and new delay (0 means immediate) +// - QueuedAt is preserved (for total time tracking) +func (q *Queue) Enqueue(entity Writeable, isNewRecord bool, delay time.Duration) { + key := entity.WriteKey() + + q.mu.Lock() + defer q.mu.Unlock() + + if existing, ok := q.pending[key]; ok { + // Update existing entry with newer entity + existing.Entity = entity + existing.UpdatedAt = time.Now() + // Preserve INSERT status + existing.IsNewRecord = existing.IsNewRecord || isNewRecord + // Use minimum delay (0 means write immediately) + if delay < existing.Delay { + existing.Delay = delay + } + q.stats.IncWriteBehindSquashed(entity.WriteType()) + } else { + // New entry + q.pending[key] = &QueueEntry{ + Key: key, + Entity: entity, + QueuedAt: time.Now(), + UpdatedAt: time.Now(), + IsNewRecord: isNewRecord, + Delay: delay, + } + q.orderedKeys = append(q.orderedKeys, key) + } + + q.stats.SetWriteBehindQueueDepth(entity.WriteType(), float64(len(q.pending))) +} + +// Size returns the current queue size +func (q *Queue) Size() int { + q.mu.RLock() + defer q.mu.RUnlock() + return len(q.pending) +} + +// IsWarmupComplete returns true if the warmup period has elapsed +func (q *Queue) IsWarmupComplete() bool { + q.mu.RLock() + defer q.mu.RUnlock() + return q.warmupComplete +} + +// checkWarmup checks if warmup period has elapsed and updates state +func (q *Queue) checkWarmup() bool { + if q.warmupComplete { + return true + } + + elapsed := time.Since(q.startTime) + if elapsed >= time.Duration(q.config.StartupDelaySeconds)*time.Second { + q.mu.Lock() + if !q.warmupComplete { + q.warmupComplete = true + queueSize := len(q.pending) + q.mu.Unlock() + log.Infof("Write-behind warmup complete, processing %d queued writes", queueSize) + return true + } + q.mu.Unlock() + return true + } + return false +} + +// getReadyEntries returns entries that are ready to be written +// An entry is ready if its delay has elapsed since QueuedAt +func (q *Queue) getReadyEntries(maxCount int) []*QueueEntry { + q.mu.Lock() + defer q.mu.Unlock() + + if len(q.orderedKeys) == 0 { + return nil + } + + ready := make([]*QueueEntry, 0, maxCount) + remainingKeys := make([]string, 0, len(q.orderedKeys)) + now := time.Now() + + for _, key := range q.orderedKeys { + if len(ready) >= maxCount { + remainingKeys = append(remainingKeys, key) + continue + } + + entry, ok := q.pending[key] + if !ok { + continue + } + + // Check if delay has elapsed + if now.Sub(entry.QueuedAt) < entry.Delay { + remainingKeys = append(remainingKeys, key) + continue + } + + // Entry is ready + ready = append(ready, entry) + delete(q.pending, key) + } + + q.orderedKeys = remainingKeys + + return ready +} + +// Flush writes all pending entries immediately (used during shutdown) +func (q *Queue) Flush() { + q.mu.Lock() + entries := make([]*QueueEntry, 0, len(q.pending)) + for _, entry := range q.pending { + entries = append(entries, entry) + } + q.pending = make(map[string]*QueueEntry) + q.orderedKeys = q.orderedKeys[:0] + q.mu.Unlock() + + if len(entries) == 0 { + return + } + + log.Infof("Write-behind flushing %d entries", len(entries)) + + // Write all entries without rate limiting + for _, entry := range entries { + q.writeEntry(entry) + } + + log.Info("Write-behind flush complete") +} + +// writeEntry performs the actual database write for an entry +func (q *Queue) writeEntry(entry *QueueEntry) { + start := time.Now() + + err := entry.Entity.WriteToDB(q.db, entry.IsNewRecord) + latency := time.Since(start).Seconds() + + if err != nil { + q.stats.IncWriteBehindErrors(entry.Entity.WriteType()) + log.Errorf("Write-behind error for %s: %v", entry.Key, err) + } else { + q.stats.IncWriteBehindWrites(entry.Entity.WriteType()) + } + + q.stats.ObserveWriteBehindLatency(entry.Entity.WriteType(), latency) +} diff --git a/decoder/writebehind/queue_test.go b/decoder/writebehind/queue_test.go new file mode 100644 index 00000000..05012ca8 --- /dev/null +++ b/decoder/writebehind/queue_test.go @@ -0,0 +1,200 @@ +package writebehind + +import ( + "testing" + "time" + + "golbat/db" + "golbat/stats_collector" +) + +// mockWriteable implements Writeable for testing +type mockWriteable struct { + key string + writeType string + quality int + written bool +} + +func (m *mockWriteable) WriteKey() string { return m.key } +func (m *mockWriteable) WriteType() string { return m.writeType } +func (m *mockWriteable) WriteToDB(db db.DbDetails, isNewRecord bool) error { + m.written = true + return nil +} + +func TestQueueEnqueue(t *testing.T) { + stats := stats_collector.NewNoopStatsCollector() + cfg := QueueConfig{ + StartupDelaySeconds: 0, // No delay for tests + RateLimit: 0, // Unlimited + BurstCapacity: 100, + } + q := NewQueue(cfg, db.DbDetails{}, stats) + + entity := &mockWriteable{key: "test:1", writeType: "test", quality: 1} + q.Enqueue(entity, true, 0) + + if q.Size() != 1 { + t.Errorf("Expected queue size 1, got %d", q.Size()) + } +} + +func TestQueueSquashing(t *testing.T) { + stats := stats_collector.NewNoopStatsCollector() + cfg := QueueConfig{ + StartupDelaySeconds: 0, + RateLimit: 0, + BurstCapacity: 100, + } + q := NewQueue(cfg, db.DbDetails{}, stats) + + // Enqueue first entity + entity1 := &mockWriteable{key: "test:1", writeType: "test", quality: 1} + q.Enqueue(entity1, true, 0) + + // Enqueue second entity with same key + entity2 := &mockWriteable{key: "test:1", writeType: "test", quality: 2} + q.Enqueue(entity2, false, 0) + + // Should still only have 1 entry (squashed) + if q.Size() != 1 { + t.Errorf("Expected queue size 1 after squash, got %d", q.Size()) + } + + // The entry should use the newer entity (replaces old) + q.mu.RLock() + entry := q.pending["test:1"] + q.mu.RUnlock() + + if entry.Entity.(*mockWriteable).quality != 2 { + t.Errorf("Expected entity quality 2 (newer), got %d", entry.Entity.(*mockWriteable).quality) + } + + // IsNewRecord should be preserved (true || false = true) + if !entry.IsNewRecord { + t.Error("Expected IsNewRecord to be preserved as true after squash") + } +} + +func TestQueueNewRecordPreservation(t *testing.T) { + stats := stats_collector.NewNoopStatsCollector() + cfg := QueueConfig{ + StartupDelaySeconds: 0, + RateLimit: 0, + BurstCapacity: 100, + } + q := NewQueue(cfg, db.DbDetails{}, stats) + + // Enqueue as new record + entity1 := &mockWriteable{key: "test:1", writeType: "test", quality: 1} + q.Enqueue(entity1, true, 0) + + // Enqueue update (not new) + entity2 := &mockWriteable{key: "test:1", writeType: "test", quality: 2} + q.Enqueue(entity2, false, 0) + + q.mu.RLock() + entry := q.pending["test:1"] + q.mu.RUnlock() + + if !entry.IsNewRecord { + t.Error("IsNewRecord should be preserved as true when first entry was new") + } +} + +func TestQueueDelayHandling(t *testing.T) { + stats := stats_collector.NewNoopStatsCollector() + cfg := QueueConfig{ + StartupDelaySeconds: 0, + RateLimit: 0, + BurstCapacity: 100, + } + q := NewQueue(cfg, db.DbDetails{}, stats) + + // Enqueue with 1 second delay + entity1 := &mockWriteable{key: "test:1", writeType: "test", quality: 1} + q.Enqueue(entity1, true, 1*time.Second) + + q.mu.RLock() + entry := q.pending["test:1"] + q.mu.RUnlock() + + if entry.Delay != 1*time.Second { + t.Errorf("Expected delay of 1s, got %v", entry.Delay) + } + + // Enqueue same key with 0 delay (should reduce delay) + entity2 := &mockWriteable{key: "test:1", writeType: "test", quality: 2} + q.Enqueue(entity2, false, 0) + + q.mu.RLock() + entry = q.pending["test:1"] + q.mu.RUnlock() + + if entry.Delay != 0 { + t.Errorf("Expected delay reduced to 0, got %v", entry.Delay) + } +} + +func TestTokenBucketUnlimited(t *testing.T) { + tb := NewTokenBucket(0, 100) // 0 = unlimited + + if !tb.IsUnlimited() { + t.Error("Expected unlimited bucket") + } + + // Should always succeed + for i := 0; i < 1000; i++ { + if !tb.TryAcquire(1) { + t.Errorf("TryAcquire failed on iteration %d for unlimited bucket", i) + } + } +} + +func TestTokenBucketRateLimited(t *testing.T) { + tb := NewTokenBucket(10, 5) // 10/sec, burst of 5 + + // Should succeed for burst capacity + for i := 0; i < 5; i++ { + if !tb.TryAcquire(1) { + t.Errorf("TryAcquire should succeed for burst, failed on %d", i) + } + } + + // Should fail after burst exhausted + if tb.TryAcquire(1) { + t.Error("TryAcquire should fail after burst exhausted") + } + + // Wait for refill + time.Sleep(150 * time.Millisecond) // Should get ~1.5 tokens + + if !tb.TryAcquire(1) { + t.Error("TryAcquire should succeed after refill") + } +} + +func TestQueueWarmup(t *testing.T) { + stats := stats_collector.NewNoopStatsCollector() + cfg := QueueConfig{ + StartupDelaySeconds: 1, // 1 second delay + RateLimit: 0, + BurstCapacity: 100, + } + q := NewQueue(cfg, db.DbDetails{}, stats) + + if q.IsWarmupComplete() { + t.Error("Warmup should not be complete immediately") + } + + // Wait for warmup + time.Sleep(1100 * time.Millisecond) + + // Trigger warmup check + q.checkWarmup() + + if !q.IsWarmupComplete() { + t.Error("Warmup should be complete after delay") + } +} diff --git a/decoder/writebehind/ratelimit.go b/decoder/writebehind/ratelimit.go new file mode 100644 index 00000000..90d2822c --- /dev/null +++ b/decoder/writebehind/ratelimit.go @@ -0,0 +1,116 @@ +package writebehind + +import ( + "sync" + "time" +) + +// TokenBucket implements a token bucket rate limiter +type TokenBucket struct { + mu sync.Mutex + tokens float64 + maxTokens float64 + refillRate float64 // tokens per second + lastRefill time.Time + unlimited bool +} + +// NewTokenBucket creates a new token bucket rate limiter +// ratePerSecond is the sustained rate (tokens added per second) +// burstCapacity is the maximum burst size (bucket capacity) +// If ratePerSecond <= 0, the limiter is unlimited +func NewTokenBucket(ratePerSecond int, burstCapacity int) *TokenBucket { + if ratePerSecond <= 0 { + return &TokenBucket{ + unlimited: true, + } + } + + return &TokenBucket{ + tokens: float64(burstCapacity), // Start with full bucket + maxTokens: float64(burstCapacity), + refillRate: float64(ratePerSecond), + lastRefill: time.Now(), + unlimited: false, + } +} + +// refill adds tokens based on time elapsed since last refill +func (tb *TokenBucket) refill() { + now := time.Now() + elapsed := now.Sub(tb.lastRefill).Seconds() + tb.tokens += elapsed * tb.refillRate + if tb.tokens > tb.maxTokens { + tb.tokens = tb.maxTokens + } + tb.lastRefill = now +} + +// TryAcquire attempts to acquire n tokens without blocking +// Returns true if tokens were acquired, false otherwise +func (tb *TokenBucket) TryAcquire(n int) bool { + if tb.unlimited { + return true + } + + tb.mu.Lock() + defer tb.mu.Unlock() + + tb.refill() + + if tb.tokens >= float64(n) { + tb.tokens -= float64(n) + return true + } + return false +} + +// WaitAcquire blocks until n tokens are available, then acquires them +// Returns the time waited +func (tb *TokenBucket) WaitAcquire(n int) time.Duration { + if tb.unlimited { + return 0 + } + + start := time.Now() + + for { + tb.mu.Lock() + tb.refill() + + if tb.tokens >= float64(n) { + tb.tokens -= float64(n) + tb.mu.Unlock() + return time.Since(start) + } + + // Calculate time needed to get enough tokens + deficit := float64(n) - tb.tokens + waitTime := time.Duration(deficit / tb.refillRate * float64(time.Second)) + tb.mu.Unlock() + + // Sleep for the needed time (minimum 1ms to avoid busy loop) + if waitTime < time.Millisecond { + waitTime = time.Millisecond + } + time.Sleep(waitTime) + } +} + +// Available returns the current number of available tokens +func (tb *TokenBucket) Available() float64 { + if tb.unlimited { + return float64(^uint(0) >> 1) // Max float64 that fits in an int + } + + tb.mu.Lock() + defer tb.mu.Unlock() + + tb.refill() + return tb.tokens +} + +// IsUnlimited returns true if rate limiting is disabled +func (tb *TokenBucket) IsUnlimited() bool { + return tb.unlimited +} diff --git a/decoder/writebehind/s2cell_accumulator.go b/decoder/writebehind/s2cell_accumulator.go new file mode 100644 index 00000000..5fd556ee --- /dev/null +++ b/decoder/writebehind/s2cell_accumulator.go @@ -0,0 +1,192 @@ +package writebehind + +import ( + "context" + "sync" + "time" + + "golbat/db" + "golbat/stats_collector" + + log "github.com/sirupsen/logrus" +) + +// S2CellData holds the data needed for an S2Cell write +type S2CellData struct { + Id uint64 + Latitude float64 + Longitude float64 + Level int64 + Updated int64 +} + +// S2CellAccumulator collects S2Cell updates and writes them in batches +type S2CellAccumulator struct { + mu sync.Mutex + pending map[uint64]*S2CellData // Dedupe by cell ID + + warmupComplete bool + startTime time.Time + + config QueueConfig + db db.DbDetails + stats stats_collector.StatsCollector + + // Batch write function - injected to avoid circular dependency + batchWriter func(db.DbDetails, []*S2CellData) error +} + +// NewS2CellAccumulator creates a new S2Cell accumulator +func NewS2CellAccumulator(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.StatsCollector, batchWriter func(db.DbDetails, []*S2CellData) error) *S2CellAccumulator { + return &S2CellAccumulator{ + pending: make(map[uint64]*S2CellData), + warmupComplete: false, + startTime: time.Now(), + config: cfg, + db: dbDetails, + stats: stats, + batchWriter: batchWriter, + } +} + +// Add adds S2Cell data to the accumulator (deduplicates by ID) +func (a *S2CellAccumulator) Add(cells []*S2CellData) { + a.mu.Lock() + defer a.mu.Unlock() + + for _, cell := range cells { + if existing, ok := a.pending[cell.Id]; ok { + // Update existing - latest timestamp wins + if cell.Updated > existing.Updated { + a.pending[cell.Id] = cell + } + a.stats.IncWriteBehindSquashed("s2cell") + } else { + a.pending[cell.Id] = cell + } + } + + a.stats.SetWriteBehindQueueDepth("s2cell", float64(len(a.pending))) +} + +// Size returns the current number of pending cells +func (a *S2CellAccumulator) Size() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.pending) +} + +// IsWarmupComplete returns true if the warmup period has elapsed +func (a *S2CellAccumulator) IsWarmupComplete() bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.warmupComplete +} + +// checkWarmup checks if warmup period has elapsed and updates state +func (a *S2CellAccumulator) checkWarmup() bool { + if a.warmupComplete { + return true + } + + elapsed := time.Since(a.startTime) + if elapsed >= time.Duration(a.config.StartupDelaySeconds)*time.Second { + a.mu.Lock() + if !a.warmupComplete { + a.warmupComplete = true + queueSize := len(a.pending) + a.mu.Unlock() + log.Infof("S2Cell accumulator warmup complete, processing %d queued cells", queueSize) + return true + } + a.mu.Unlock() + return true + } + return false +} + +// Start begins the background processing loop +func (a *S2CellAccumulator) Start(ctx context.Context) { + go a.processLoop(ctx) +} + +// processLoop runs the background flush loop +func (a *S2CellAccumulator) processLoop(ctx context.Context) { + ticker := time.NewTicker(5 * time.Second) // Flush every 5 seconds + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + a.Flush() + log.Info("S2Cell accumulator stopped") + return + case <-ticker.C: + if a.checkWarmup() { + a.flushBatch() + } + } + } +} + +// flushBatch writes pending cells +func (a *S2CellAccumulator) flushBatch() { + a.mu.Lock() + if len(a.pending) == 0 { + a.mu.Unlock() + return + } + + // Take all pending cells + cells := make([]*S2CellData, 0, len(a.pending)) + for _, cell := range a.pending { + cells = append(cells, cell) + } + a.pending = make(map[uint64]*S2CellData) + a.mu.Unlock() + + batchSize := len(cells) + log.Debugf("S2Cell accumulator flushing batch of %d cells", batchSize) + + // Write batch + start := time.Now() + err := a.batchWriter(a.db, cells) + latency := time.Since(start).Seconds() + + if err != nil { + a.stats.IncWriteBehindErrors("s2cell") + log.Errorf("S2Cell batch write error: %v", err) + } else { + a.stats.IncWriteBehindWrites("s2cell") + } + + a.stats.ObserveWriteBehindLatency("s2cell", latency) + a.stats.SetWriteBehindQueueDepth("s2cell", float64(len(a.pending))) + a.stats.SetS2CellBatchSize(batchSize) +} + +// Flush writes all pending cells immediately (used during shutdown) +func (a *S2CellAccumulator) Flush() { + a.mu.Lock() + if len(a.pending) == 0 { + a.mu.Unlock() + return + } + + cells := make([]*S2CellData, 0, len(a.pending)) + for _, cell := range a.pending { + cells = append(cells, cell) + } + a.pending = make(map[uint64]*S2CellData) + a.mu.Unlock() + + log.Infof("S2Cell accumulator flushing %d cells", len(cells)) + + // Write without rate limiting during shutdown + err := a.batchWriter(a.db, cells) + if err != nil { + log.Errorf("S2Cell flush error: %v", err) + } + + log.Info("S2Cell accumulator flush complete") +} diff --git a/decoder/writebehind/types.go b/decoder/writebehind/types.go new file mode 100644 index 00000000..4369672f --- /dev/null +++ b/decoder/writebehind/types.go @@ -0,0 +1,37 @@ +package writebehind + +import ( + "time" + + "golbat/db" +) + +// Writeable is implemented by any entity that can be written through the queue +type Writeable interface { + // WriteKey returns a unique key for this entity (for squashing) + WriteKey() string + + // WriteType returns the entity type name (for metrics) + WriteType() string + + // WriteToDB performs the actual database write + // isNewRecord determines INSERT vs UPDATE + WriteToDB(db db.DbDetails, isNewRecord bool) error +} + +// QueueEntry represents a pending write in the queue +type QueueEntry struct { + Key string + Entity Writeable + QueuedAt time.Time + UpdatedAt time.Time + IsNewRecord bool // Track if this needs INSERT (preserved across updates) + Delay time.Duration // Minimum delay before writing (0 = immediate) +} + +// QueueConfig holds configuration for the write-behind queue +type QueueConfig struct { + StartupDelaySeconds int // Delay before processing starts (warmup period) + RateLimit int // Writes per second, 0 = unlimited + BurstCapacity int // Token bucket burst capacity +} diff --git a/main.go b/main.go index 5214c5e7..555ee8cd 100644 --- a/main.go +++ b/main.go @@ -207,7 +207,8 @@ func main() { _ = decoder.WatchMasterFileData() } decoder.LoadStatsGeofences() - decoder.InitPokemonPendingQueue(ctx, dbDetails, 30*time.Second, 5*time.Second) + decoder.InitWriteBehindQueue(ctx, dbDetails) + decoder.InitS2CellAccumulator(ctx, dbDetails) InitDeviceCache() wg.Add(1) @@ -243,7 +244,7 @@ func main() { } if cfg.Cleanup.Quests == true { - StartQuestExpiry(db) + StartQuestExpiry(dbDetails) } if cfg.Cleanup.Stats == true { @@ -394,7 +395,11 @@ func main() { log.Info("http server is shutdown, waiting for other go routines to exit...") wg.Wait() - log.Info("go routines have exited, flushing webhooks now...") + log.Info("go routines have exited, flushing write-behind queue...") + decoder.FlushS2CellAccumulator() + decoder.FlushWriteBehindQueue() + + log.Info("flushing webhooks now...") webhooksSender.Flush() log.Info("Golbat exiting!") diff --git a/stats.go b/stats.go index 1cf549f0..f3a2c85d 100644 --- a/stats.go +++ b/stats.go @@ -1,9 +1,11 @@ package main import ( + "context" "database/sql" "fmt" "golbat/config" + db2 "golbat/db" "golbat/decoder" "time" @@ -215,61 +217,23 @@ func StartTappableExpiry(db *sqlx.DB) { }() } -func StartQuestExpiry(db *sqlx.DB) { +func StartQuestExpiry(dbDetails db2.DbDetails) { ticker := time.NewTicker(time.Hour + 1*time.Minute) go func() { for { <-ticker.C start := time.Now() - var totalRows int64 = 0 - var result sql.Result - var err error - - result, err = db.Exec("UPDATE pokestop " + - "SET " + - "quest_type = NULL," + - "quest_timestamp = NULL," + - "quest_target = NULL," + - "quest_conditions = NULL," + - "quest_rewards = NULL," + - "quest_template = NULL," + - "quest_title = NULL " + - "WHERE quest_expiry < UNIX_TIMESTAMP();") - - if err != nil { - log.Errorf("DB - Cleanup of quest table error %s", err) - } else { - rows, _ := result.RowsAffected() - totalRows += rows - if rows > 0 { - decoder.ClearPokestopCache() - } - } - - result, err = db.Exec("UPDATE pokestop " + - "SET " + - "alternative_quest_type = NULL," + - "alternative_quest_timestamp = NULL," + - "alternative_quest_target = NULL," + - "alternative_quest_conditions = NULL," + - "alternative_quest_rewards = NULL," + - "alternative_quest_template = NULL," + - "alternative_quest_title = NULL " + - "WHERE alternative_quest_expiry < UNIX_TIMESTAMP();") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + count, err := decoder.ExpireQuests(ctx, dbDetails) + cancel() if err != nil { log.Errorf("DB - Cleanup of quest table error %s", err) - } else { - rows, _ := result.RowsAffected() - totalRows += rows - if rows > 0 { - decoder.ClearPokestopCache() - } } elapsed := time.Since(start) - log.Infof("DB - Cleanup of quest table took %s (%d quests)", elapsed, totalRows) + log.Infof("DB - Cleanup of quest table took %s (%d pokestops)", elapsed, count) } }() } diff --git a/stats_collector/noop.go b/stats_collector/noop.go index 1a1ff0fa..24bbf8d1 100644 --- a/stats_collector/noop.go +++ b/stats_collector/noop.go @@ -51,6 +51,15 @@ func (col *noopCollector) DecPokemons(bool, null.String) func (col *noopCollector) UpdateMaxBattleCount([]geo.AreaName, int64) {} func (col *noopCollector) IncFortChange(string) {} +// Write-behind queue metrics (noop) +func (col *noopCollector) SetWriteBehindQueueDepth(string, float64) {} +func (col *noopCollector) IncWriteBehindSquashed(string) {} +func (col *noopCollector) IncWriteBehindRateLimited(string) {} +func (col *noopCollector) IncWriteBehindErrors(string) {} +func (col *noopCollector) IncWriteBehindWrites(string) {} +func (col *noopCollector) ObserveWriteBehindLatency(string, float64) {} +func (col *noopCollector) SetS2CellBatchSize(int) {} + func NewNoopStatsCollector() StatsCollector { return &noopCollector{} } diff --git a/stats_collector/prometheus.go b/stats_collector/prometheus.go index 93547984..875e2810 100644 --- a/stats_collector/prometheus.go +++ b/stats_collector/prometheus.go @@ -335,6 +335,66 @@ var ( }, []string{"change_type"}, ) + + // Write-behind queue metrics + writeBehindQueueDepth = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: ns, + Name: "write_behind_queue_depth", + Help: "Current write-behind queue depth", + }, + []string{"entity_type"}, + ) + writeBehindSquashed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: ns, + Name: "write_behind_squashed_total", + Help: "Total number of updates squashed in write-behind queue", + }, + []string{"entity_type"}, + ) + writeBehindRateLimited = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: ns, + Name: "write_behind_rate_limited_total", + Help: "Total number of rate limit hits in write-behind queue", + }, + []string{"entity_type"}, + ) + writeBehindErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: ns, + Name: "write_behind_errors_total", + Help: "Total number of write errors in write-behind queue", + }, + []string{"entity_type"}, + ) + writeBehindWrites = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: ns, + Name: "write_behind_writes_total", + Help: "Total number of successful writes from write-behind queue", + }, + []string{"entity_type"}, + ) + writeBehindLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: ns, + Name: "write_behind_latency_seconds", + Help: "Write latency in seconds for write-behind queue", + Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, + }, + []string{"entity_type"}, + ) + + // S2Cell batch metrics + s2CellBatchSize = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: ns, + Name: "s2cell_batch_size", + Help: "Number of S2Cells written in the last batch flush", + }, + ) ) var _ StatsCollector = (*promCollector)(nil) @@ -596,6 +656,34 @@ func (col *promCollector) IncFortChange(changeType string) { fortChange.WithLabelValues(changeType).Inc() } +func (col *promCollector) SetWriteBehindQueueDepth(entityType string, depth float64) { + writeBehindQueueDepth.WithLabelValues(entityType).Set(depth) +} + +func (col *promCollector) IncWriteBehindSquashed(entityType string) { + writeBehindSquashed.WithLabelValues(entityType).Inc() +} + +func (col *promCollector) IncWriteBehindRateLimited(entityType string) { + writeBehindRateLimited.WithLabelValues(entityType).Inc() +} + +func (col *promCollector) IncWriteBehindErrors(entityType string) { + writeBehindErrors.WithLabelValues(entityType).Inc() +} + +func (col *promCollector) IncWriteBehindWrites(entityType string) { + writeBehindWrites.WithLabelValues(entityType).Inc() +} + +func (col *promCollector) ObserveWriteBehindLatency(entityType string, seconds float64) { + writeBehindLatency.WithLabelValues(entityType).Observe(seconds) +} + +func (col *promCollector) SetS2CellBatchSize(size int) { + s2CellBatchSize.Set(float64(size)) +} + func initPrometheus() { prometheus.MustRegister( rawRequests, decodeMethods, decodeFortDetails, decodeGetMapForts, decodeGetGymInfo, decodeEncounter, @@ -611,6 +699,10 @@ func initPrometheus() { duplicateEncounters, dbQueries, gyms, incidents, pokemons, lures, quests, raids, + + writeBehindQueueDepth, writeBehindSquashed, writeBehindRateLimited, + writeBehindErrors, writeBehindWrites, writeBehindLatency, + s2CellBatchSize, ) } diff --git a/stats_collector/stats_collector.go b/stats_collector/stats_collector.go index 9481c41a..deba00e5 100644 --- a/stats_collector/stats_collector.go +++ b/stats_collector/stats_collector.go @@ -50,6 +50,17 @@ type StatsCollector interface { DecPokemons(hasIv bool, seenType null.String) UpdateMaxBattleCount(areas []geo.AreaName, level int64) IncFortChange(changeType string) + + // Write-behind queue metrics + SetWriteBehindQueueDepth(entityType string, depth float64) + IncWriteBehindSquashed(entityType string) + IncWriteBehindRateLimited(entityType string) + IncWriteBehindErrors(entityType string) + IncWriteBehindWrites(entityType string) + ObserveWriteBehindLatency(entityType string, seconds float64) + + // S2Cell batch metrics + SetS2CellBatchSize(size int) } type Config interface { From f1b28894bb1734d288304175e93d7b8a7ef597d3 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 4 Feb 2026 13:20:01 +0000 Subject: [PATCH 42/78] Switch to a concurrent writer model --- config/config.go | 5 +- config/reader.go | 3 +- decoder/main.go | 20 +++--- decoder/writebehind/processor.go | 83 +++++++++++++-------- decoder/writebehind/queue.go | 77 ++++++-------------- decoder/writebehind/queue_test.go | 71 ++++-------------- decoder/writebehind/ratelimit.go | 116 ------------------------------ decoder/writebehind/types.go | 3 +- 8 files changed, 103 insertions(+), 275 deletions(-) delete mode 100644 decoder/writebehind/ratelimit.go diff --git a/config/config.go b/config/config.go index 95120c58..7d21e568 100644 --- a/config/config.go +++ b/config/config.go @@ -126,9 +126,8 @@ type tuning struct { ProfileRoutes bool `koanf:"profile_routes"` MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` ReduceUpdates bool `koanf:"reduce_updates"` - WriteBehindStartupDelay int `koanf:"write_behind_startup_delay"` // seconds, default: 120 - WriteBehindRateLimit int `koanf:"write_behind_rate_limit"` // writes/sec, 0=unlimited - WriteBehindBurstCapacity int `koanf:"write_behind_burst_capacity"` // default: 100 + WriteBehindStartupDelay int `koanf:"write_behind_startup_delay"` // seconds, default: 120 + WriteBehindWorkerCount int `koanf:"write_behind_worker_count"` // concurrent writers, default: 100 } type scanRule struct { diff --git a/config/reader.go b/config/reader.go index 1038475d..c77df9c3 100644 --- a/config/reader.go +++ b/config/reader.go @@ -56,8 +56,7 @@ func ReadConfig() (configDefinition, error) { MaxConcurrentProactiveIVSwitch: 6, ReduceUpdates: false, WriteBehindStartupDelay: 120, // 2 minutes - WriteBehindRateLimit: 0, // unlimited - WriteBehindBurstCapacity: 100, + WriteBehindWorkerCount: 50, // concurrent writers }, Weather: weather{ ProactiveIVSwitching: true, diff --git a/decoder/main.go b/decoder/main.go index ef42e94c..fa7e683f 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -297,14 +297,21 @@ func SetStatsCollector(collector stats_collector.StatsCollector) { func InitWriteBehindQueue(ctx context.Context, dbDetails db.DbDetails) { cfg := writebehind.QueueConfig{ StartupDelaySeconds: config.Config.Tuning.WriteBehindStartupDelay, - RateLimit: config.Config.Tuning.WriteBehindRateLimit, - BurstCapacity: config.Config.Tuning.WriteBehindBurstCapacity, + WorkerCount: config.Config.Tuning.WriteBehindWorkerCount, } writeBehindQueue = writebehind.NewQueue(cfg, dbDetails, statsCollector) - log.Infof("Write-behind queue initialized: startup_delay=%ds, rate_limit=%d/s, burst=%d", - cfg.StartupDelaySeconds, cfg.RateLimit, cfg.BurstCapacity) + log.Infof("Write-behind queue initialized: startup_delay=%ds, workers=%d", + cfg.StartupDelaySeconds, cfg.WorkerCount) + + // Warn if worker count exceeds half of database pool size + maxPool := config.Config.Database.MaxPool + if cfg.WorkerCount > maxPool/2 { + log.Warnf("Write-behind worker count (%d) exceeds half of database pool size (%d). "+ + "Consider increasing database.max_pool or reducing tuning.write_behind_worker_count", + cfg.WorkerCount, maxPool) + } // Start the processing loop in a goroutine go writeBehindQueue.ProcessLoop(ctx) @@ -327,14 +334,11 @@ func FlushWriteBehindQueue() { func InitS2CellAccumulator(ctx context.Context, dbDetails db.DbDetails) { cfg := writebehind.QueueConfig{ StartupDelaySeconds: config.Config.Tuning.WriteBehindStartupDelay, - RateLimit: config.Config.Tuning.WriteBehindRateLimit, - BurstCapacity: config.Config.Tuning.WriteBehindBurstCapacity, } s2CellAccumulator = writebehind.NewS2CellAccumulator(cfg, dbDetails, statsCollector, s2CellBatchWrite) - log.Infof("S2Cell accumulator initialized: startup_delay=%ds, rate_limit=%d/s, burst=%d", - cfg.StartupDelaySeconds, cfg.RateLimit, cfg.BurstCapacity) + log.Infof("S2Cell accumulator initialized: startup_delay=%ds", cfg.StartupDelaySeconds) // Start the processing loop s2CellAccumulator.Start(ctx) diff --git a/decoder/writebehind/processor.go b/decoder/writebehind/processor.go index 60e094c7..4658c121 100644 --- a/decoder/writebehind/processor.go +++ b/decoder/writebehind/processor.go @@ -8,20 +8,40 @@ import ( ) const ( - // processingInterval is how often the processor checks for work - processingInterval = 100 * time.Millisecond - - // batchSize is the maximum number of entries to process per tick - batchSize = 50 + // dispatchInterval is how often the dispatcher checks for ready items + dispatchInterval = 100 * time.Millisecond // statusLogInterval is how often to log queue status statusLogInterval = 30 * time.Second ) -// ProcessLoop runs the main processing loop for the queue +// ProcessLoop starts the dispatcher and workers // This should be called in a goroutine func (q *Queue) ProcessLoop(ctx context.Context) { - ticker := time.NewTicker(processingInterval) + // Start worker goroutines + for i := 0; i < q.workerCount; i++ { + go q.worker(ctx, i) + } + + // Run dispatcher in this goroutine + q.dispatcher(ctx) +} + +// worker reads from the work channel and writes entries to DB +func (q *Queue) worker(ctx context.Context, id int) { + for { + select { + case <-ctx.Done(): + return + case entry := <-q.workChan: + q.writeEntry(entry) + } + } +} + +// dispatcher moves ready entries from pending map to work channel +func (q *Queue) dispatcher(ctx context.Context) { + ticker := time.NewTicker(dispatchInterval) defer ticker.Stop() statusTicker := time.NewTicker(statusLogInterval) @@ -30,45 +50,48 @@ func (q *Queue) ProcessLoop(ctx context.Context) { for { select { case <-ctx.Done(): - log.Info("Write-behind processor shutting down, flushing queue...") + log.Info("Write-behind dispatcher shutting down, flushing queue...") q.Flush() return case <-statusTicker.C: queueSize := q.Size() - log.Infof("Write-behind queue length: %d", queueSize) + channelLen := len(q.workChan) + log.Infof("Write-behind queue: %d pending, %d in channel", queueSize, channelLen) case <-ticker.C: if q.checkWarmup() { - q.processBatch(ctx) + q.dispatchReady() } } } } -// processBatch processes a batch of entries from the queue -func (q *Queue) processBatch(ctx context.Context) { - entries := q.getReadyEntries(batchSize) - if len(entries) == 0 { +// dispatchReady moves entries that are ready (delay expired) to the work channel +func (q *Queue) dispatchReady() { + q.mu.Lock() + defer q.mu.Unlock() + + if len(q.pending) == 0 { return } - for _, entry := range entries { - // Check for context cancellation - select { - case <-ctx.Done(): - return - default: - } + now := time.Now() + dispatched := 0 - // Apply rate limiting - if !q.rateLimiter.TryAcquire(1) { - q.stats.IncWriteBehindRateLimited(entry.Entity.WriteType()) - log.Debugf("Write-behind rate limited for %s", entry.Key) - waitTime := q.rateLimiter.WaitAcquire(1) - if waitTime > time.Second { - log.Warnf("Write-behind rate limited for %s, waited %v", entry.Key, waitTime) - } + for key, entry := range q.pending { + // Check if delay has elapsed + if now.Sub(entry.QueuedAt) < entry.Delay { + continue } - q.writeEntry(entry) + // Try to send to work channel (non-blocking) + select { + case q.workChan <- entry: + delete(q.pending, key) + dispatched++ + default: + // Channel full, stop dispatching this tick + // Workers will drain it and we'll dispatch more next tick + return + } } } diff --git a/decoder/writebehind/queue.go b/decoder/writebehind/queue.go index 8e679eb5..cf985068 100644 --- a/decoder/writebehind/queue.go +++ b/decoder/writebehind/queue.go @@ -12,11 +12,12 @@ import ( // Queue is the write-behind queue that buffers database writes type Queue struct { - mu sync.RWMutex - pending map[string]*QueueEntry // key -> entry for O(1) lookup - orderedKeys []string // FIFO order for processing + mu sync.Mutex + pending map[string]*QueueEntry // key -> entry for squashing - rateLimiter *TokenBucket + workChan chan *QueueEntry // buffered channel for workers + + workerCount int warmupComplete bool startTime time.Time @@ -27,10 +28,14 @@ type Queue struct { // NewQueue creates a new write-behind queue func NewQueue(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.StatsCollector) *Queue { + if cfg.WorkerCount <= 0 { + cfg.WorkerCount = 50 // default + } + return &Queue{ pending: make(map[string]*QueueEntry), - orderedKeys: make([]string, 0, 1024), - rateLimiter: NewTokenBucket(cfg.RateLimit, cfg.BurstCapacity), + workChan: make(chan *QueueEntry, cfg.WorkerCount*10), // buffer 10x worker count + workerCount: cfg.WorkerCount, warmupComplete: false, startTime: time.Now(), config: cfg, @@ -39,8 +44,9 @@ func NewQueue(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.Sta } } -// Enqueue adds or updates an entity write with a specified delay +// Enqueue adds or updates an entity write // If an entry already exists for the same key: +// - Entity is replaced with the newer one // - IsNewRecord is preserved if either is true (INSERT takes priority) // - Delay is updated to the minimum of existing and new delay (0 means immediate) // - QueuedAt is preserved (for total time tracking) @@ -71,23 +77,22 @@ func (q *Queue) Enqueue(entity Writeable, isNewRecord bool, delay time.Duration) IsNewRecord: isNewRecord, Delay: delay, } - q.orderedKeys = append(q.orderedKeys, key) } q.stats.SetWriteBehindQueueDepth(entity.WriteType(), float64(len(q.pending))) } -// Size returns the current queue size +// Size returns the current pending queue size func (q *Queue) Size() int { - q.mu.RLock() - defer q.mu.RUnlock() + q.mu.Lock() + defer q.mu.Unlock() return len(q.pending) } // IsWarmupComplete returns true if the warmup period has elapsed func (q *Queue) IsWarmupComplete() bool { - q.mu.RLock() - defer q.mu.RUnlock() + q.mu.Lock() + defer q.mu.Unlock() return q.warmupComplete } @@ -104,7 +109,7 @@ func (q *Queue) checkWarmup() bool { q.warmupComplete = true queueSize := len(q.pending) q.mu.Unlock() - log.Infof("Write-behind warmup complete, processing %d queued writes", queueSize) + log.Infof("Write-behind warmup complete, processing %d queued writes with %d workers", queueSize, q.workerCount) return true } q.mu.Unlock() @@ -113,47 +118,6 @@ func (q *Queue) checkWarmup() bool { return false } -// getReadyEntries returns entries that are ready to be written -// An entry is ready if its delay has elapsed since QueuedAt -func (q *Queue) getReadyEntries(maxCount int) []*QueueEntry { - q.mu.Lock() - defer q.mu.Unlock() - - if len(q.orderedKeys) == 0 { - return nil - } - - ready := make([]*QueueEntry, 0, maxCount) - remainingKeys := make([]string, 0, len(q.orderedKeys)) - now := time.Now() - - for _, key := range q.orderedKeys { - if len(ready) >= maxCount { - remainingKeys = append(remainingKeys, key) - continue - } - - entry, ok := q.pending[key] - if !ok { - continue - } - - // Check if delay has elapsed - if now.Sub(entry.QueuedAt) < entry.Delay { - remainingKeys = append(remainingKeys, key) - continue - } - - // Entry is ready - ready = append(ready, entry) - delete(q.pending, key) - } - - q.orderedKeys = remainingKeys - - return ready -} - // Flush writes all pending entries immediately (used during shutdown) func (q *Queue) Flush() { q.mu.Lock() @@ -162,7 +126,6 @@ func (q *Queue) Flush() { entries = append(entries, entry) } q.pending = make(map[string]*QueueEntry) - q.orderedKeys = q.orderedKeys[:0] q.mu.Unlock() if len(entries) == 0 { @@ -171,7 +134,7 @@ func (q *Queue) Flush() { log.Infof("Write-behind flushing %d entries", len(entries)) - // Write all entries without rate limiting + // Write all entries directly (bypass channel during shutdown) for _, entry := range entries { q.writeEntry(entry) } diff --git a/decoder/writebehind/queue_test.go b/decoder/writebehind/queue_test.go index 05012ca8..0038c2a3 100644 --- a/decoder/writebehind/queue_test.go +++ b/decoder/writebehind/queue_test.go @@ -26,9 +26,8 @@ func (m *mockWriteable) WriteToDB(db db.DbDetails, isNewRecord bool) error { func TestQueueEnqueue(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() cfg := QueueConfig{ - StartupDelaySeconds: 0, // No delay for tests - RateLimit: 0, // Unlimited - BurstCapacity: 100, + StartupDelaySeconds: 0, // No delay for tests + WorkerCount: 10, // 10 workers for tests } q := NewQueue(cfg, db.DbDetails{}, stats) @@ -44,8 +43,7 @@ func TestQueueSquashing(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() cfg := QueueConfig{ StartupDelaySeconds: 0, - RateLimit: 0, - BurstCapacity: 100, + WorkerCount: 10, } q := NewQueue(cfg, db.DbDetails{}, stats) @@ -63,9 +61,9 @@ func TestQueueSquashing(t *testing.T) { } // The entry should use the newer entity (replaces old) - q.mu.RLock() + q.mu.Lock() entry := q.pending["test:1"] - q.mu.RUnlock() + q.mu.Unlock() if entry.Entity.(*mockWriteable).quality != 2 { t.Errorf("Expected entity quality 2 (newer), got %d", entry.Entity.(*mockWriteable).quality) @@ -81,8 +79,7 @@ func TestQueueNewRecordPreservation(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() cfg := QueueConfig{ StartupDelaySeconds: 0, - RateLimit: 0, - BurstCapacity: 100, + WorkerCount: 10, } q := NewQueue(cfg, db.DbDetails{}, stats) @@ -94,9 +91,9 @@ func TestQueueNewRecordPreservation(t *testing.T) { entity2 := &mockWriteable{key: "test:1", writeType: "test", quality: 2} q.Enqueue(entity2, false, 0) - q.mu.RLock() + q.mu.Lock() entry := q.pending["test:1"] - q.mu.RUnlock() + q.mu.Unlock() if !entry.IsNewRecord { t.Error("IsNewRecord should be preserved as true when first entry was new") @@ -107,8 +104,7 @@ func TestQueueDelayHandling(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() cfg := QueueConfig{ StartupDelaySeconds: 0, - RateLimit: 0, - BurstCapacity: 100, + WorkerCount: 10, } q := NewQueue(cfg, db.DbDetails{}, stats) @@ -116,9 +112,9 @@ func TestQueueDelayHandling(t *testing.T) { entity1 := &mockWriteable{key: "test:1", writeType: "test", quality: 1} q.Enqueue(entity1, true, 1*time.Second) - q.mu.RLock() + q.mu.Lock() entry := q.pending["test:1"] - q.mu.RUnlock() + q.mu.Unlock() if entry.Delay != 1*time.Second { t.Errorf("Expected delay of 1s, got %v", entry.Delay) @@ -128,59 +124,20 @@ func TestQueueDelayHandling(t *testing.T) { entity2 := &mockWriteable{key: "test:1", writeType: "test", quality: 2} q.Enqueue(entity2, false, 0) - q.mu.RLock() + q.mu.Lock() entry = q.pending["test:1"] - q.mu.RUnlock() + q.mu.Unlock() if entry.Delay != 0 { t.Errorf("Expected delay reduced to 0, got %v", entry.Delay) } } -func TestTokenBucketUnlimited(t *testing.T) { - tb := NewTokenBucket(0, 100) // 0 = unlimited - - if !tb.IsUnlimited() { - t.Error("Expected unlimited bucket") - } - - // Should always succeed - for i := 0; i < 1000; i++ { - if !tb.TryAcquire(1) { - t.Errorf("TryAcquire failed on iteration %d for unlimited bucket", i) - } - } -} - -func TestTokenBucketRateLimited(t *testing.T) { - tb := NewTokenBucket(10, 5) // 10/sec, burst of 5 - - // Should succeed for burst capacity - for i := 0; i < 5; i++ { - if !tb.TryAcquire(1) { - t.Errorf("TryAcquire should succeed for burst, failed on %d", i) - } - } - - // Should fail after burst exhausted - if tb.TryAcquire(1) { - t.Error("TryAcquire should fail after burst exhausted") - } - - // Wait for refill - time.Sleep(150 * time.Millisecond) // Should get ~1.5 tokens - - if !tb.TryAcquire(1) { - t.Error("TryAcquire should succeed after refill") - } -} - func TestQueueWarmup(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() cfg := QueueConfig{ StartupDelaySeconds: 1, // 1 second delay - RateLimit: 0, - BurstCapacity: 100, + WorkerCount: 10, } q := NewQueue(cfg, db.DbDetails{}, stats) diff --git a/decoder/writebehind/ratelimit.go b/decoder/writebehind/ratelimit.go deleted file mode 100644 index 90d2822c..00000000 --- a/decoder/writebehind/ratelimit.go +++ /dev/null @@ -1,116 +0,0 @@ -package writebehind - -import ( - "sync" - "time" -) - -// TokenBucket implements a token bucket rate limiter -type TokenBucket struct { - mu sync.Mutex - tokens float64 - maxTokens float64 - refillRate float64 // tokens per second - lastRefill time.Time - unlimited bool -} - -// NewTokenBucket creates a new token bucket rate limiter -// ratePerSecond is the sustained rate (tokens added per second) -// burstCapacity is the maximum burst size (bucket capacity) -// If ratePerSecond <= 0, the limiter is unlimited -func NewTokenBucket(ratePerSecond int, burstCapacity int) *TokenBucket { - if ratePerSecond <= 0 { - return &TokenBucket{ - unlimited: true, - } - } - - return &TokenBucket{ - tokens: float64(burstCapacity), // Start with full bucket - maxTokens: float64(burstCapacity), - refillRate: float64(ratePerSecond), - lastRefill: time.Now(), - unlimited: false, - } -} - -// refill adds tokens based on time elapsed since last refill -func (tb *TokenBucket) refill() { - now := time.Now() - elapsed := now.Sub(tb.lastRefill).Seconds() - tb.tokens += elapsed * tb.refillRate - if tb.tokens > tb.maxTokens { - tb.tokens = tb.maxTokens - } - tb.lastRefill = now -} - -// TryAcquire attempts to acquire n tokens without blocking -// Returns true if tokens were acquired, false otherwise -func (tb *TokenBucket) TryAcquire(n int) bool { - if tb.unlimited { - return true - } - - tb.mu.Lock() - defer tb.mu.Unlock() - - tb.refill() - - if tb.tokens >= float64(n) { - tb.tokens -= float64(n) - return true - } - return false -} - -// WaitAcquire blocks until n tokens are available, then acquires them -// Returns the time waited -func (tb *TokenBucket) WaitAcquire(n int) time.Duration { - if tb.unlimited { - return 0 - } - - start := time.Now() - - for { - tb.mu.Lock() - tb.refill() - - if tb.tokens >= float64(n) { - tb.tokens -= float64(n) - tb.mu.Unlock() - return time.Since(start) - } - - // Calculate time needed to get enough tokens - deficit := float64(n) - tb.tokens - waitTime := time.Duration(deficit / tb.refillRate * float64(time.Second)) - tb.mu.Unlock() - - // Sleep for the needed time (minimum 1ms to avoid busy loop) - if waitTime < time.Millisecond { - waitTime = time.Millisecond - } - time.Sleep(waitTime) - } -} - -// Available returns the current number of available tokens -func (tb *TokenBucket) Available() float64 { - if tb.unlimited { - return float64(^uint(0) >> 1) // Max float64 that fits in an int - } - - tb.mu.Lock() - defer tb.mu.Unlock() - - tb.refill() - return tb.tokens -} - -// IsUnlimited returns true if rate limiting is disabled -func (tb *TokenBucket) IsUnlimited() bool { - return tb.unlimited -} diff --git a/decoder/writebehind/types.go b/decoder/writebehind/types.go index 4369672f..840a2995 100644 --- a/decoder/writebehind/types.go +++ b/decoder/writebehind/types.go @@ -32,6 +32,5 @@ type QueueEntry struct { // QueueConfig holds configuration for the write-behind queue type QueueConfig struct { StartupDelaySeconds int // Delay before processing starts (warmup period) - RateLimit int // Writes per second, 0 = unlimited - BurstCapacity int // Token bucket burst capacity + WorkerCount int // Number of concurrent write workers (default 100) } From 9c6159c1c3f6102b4fb007e7c594473a94d119af Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 4 Feb 2026 16:16:30 +0000 Subject: [PATCH 43/78] Add stats to log entry --- decoder/writebehind/processor.go | 8 ++-- decoder/writebehind/queue.go | 65 +++++++++++++++++++++++++++++--- decoder/writebehind/types.go | 1 + 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/decoder/writebehind/processor.go b/decoder/writebehind/processor.go index 4658c121..436c1ded 100644 --- a/decoder/writebehind/processor.go +++ b/decoder/writebehind/processor.go @@ -56,7 +56,9 @@ func (q *Queue) dispatcher(ctx context.Context) { case <-statusTicker.C: queueSize := q.Size() channelLen := len(q.workChan) - log.Infof("Write-behind queue: %d pending, %d in channel", queueSize, channelLen) + avgWriteTime, avgLatency, writeCount := q.GetAndResetMetrics() + log.Infof("Write-behind queue: %d pending, %d in channel, %d writes (avg write: %.1fms, avg latency: %.1fms)", + queueSize, channelLen, writeCount, avgWriteTime, avgLatency) case <-ticker.C: if q.checkWarmup() { q.dispatchReady() @@ -78,8 +80,8 @@ func (q *Queue) dispatchReady() { dispatched := 0 for key, entry := range q.pending { - // Check if delay has elapsed - if now.Sub(entry.QueuedAt) < entry.Delay { + // Check if delay has elapsed (entry is ready when now >= ReadyAt) + if now.Before(entry.ReadyAt) { continue } diff --git a/decoder/writebehind/queue.go b/decoder/writebehind/queue.go index cf985068..72869bd3 100644 --- a/decoder/writebehind/queue.go +++ b/decoder/writebehind/queue.go @@ -21,6 +21,13 @@ type Queue struct { warmupComplete bool startTime time.Time + // Metrics tracking (protected by metricsMu) + metricsMu sync.Mutex + totalWriteTime float64 // sum of write durations in seconds + writeCount int64 // number of writes completed + totalLatency float64 // sum of latencies (ready to complete) in seconds + latencyCount int64 // number of latency samples + config QueueConfig db db.DbDetails stats stats_collector.StatsCollector @@ -50,30 +57,40 @@ func NewQueue(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.Sta // - IsNewRecord is preserved if either is true (INSERT takes priority) // - Delay is updated to the minimum of existing and new delay (0 means immediate) // - QueuedAt is preserved (for total time tracking) +// - ReadyAt is updated if the new delay makes the entry ready earlier func (q *Queue) Enqueue(entity Writeable, isNewRecord bool, delay time.Duration) { key := entity.WriteKey() q.mu.Lock() defer q.mu.Unlock() + now := time.Now() + if existing, ok := q.pending[key]; ok { // Update existing entry with newer entity existing.Entity = entity - existing.UpdatedAt = time.Now() + existing.UpdatedAt = now // Preserve INSERT status existing.IsNewRecord = existing.IsNewRecord || isNewRecord // Use minimum delay (0 means write immediately) if delay < existing.Delay { existing.Delay = delay + // Update ReadyAt if this squash makes it ready earlier + newReadyAt := now.Add(delay) + if newReadyAt.Before(existing.ReadyAt) { + existing.ReadyAt = newReadyAt + } } q.stats.IncWriteBehindSquashed(entity.WriteType()) } else { - // New entry + // New entry - ReadyAt is when the entry becomes eligible for dispatch + readyAt := now.Add(delay) q.pending[key] = &QueueEntry{ Key: key, Entity: entity, - QueuedAt: time.Now(), - UpdatedAt: time.Now(), + QueuedAt: now, + UpdatedAt: now, + ReadyAt: readyAt, IsNewRecord: isNewRecord, Delay: delay, } @@ -89,6 +106,31 @@ func (q *Queue) Size() int { return len(q.pending) } +// GetAndResetMetrics returns average write time and latency, then resets counters +// Returns (avgWriteTime, avgLatency, count) - times in milliseconds +func (q *Queue) GetAndResetMetrics() (float64, float64, int64) { + q.metricsMu.Lock() + defer q.metricsMu.Unlock() + + var avgWriteTime, avgLatency float64 + count := q.writeCount + + if q.writeCount > 0 { + avgWriteTime = (q.totalWriteTime / float64(q.writeCount)) * 1000 // convert to ms + } + if q.latencyCount > 0 { + avgLatency = (q.totalLatency / float64(q.latencyCount)) * 1000 // convert to ms + } + + // Reset counters + q.totalWriteTime = 0 + q.writeCount = 0 + q.totalLatency = 0 + q.latencyCount = 0 + + return avgWriteTime, avgLatency, count +} + // IsWarmupComplete returns true if the warmup period has elapsed func (q *Queue) IsWarmupComplete() bool { q.mu.Lock() @@ -147,7 +189,7 @@ func (q *Queue) writeEntry(entry *QueueEntry) { start := time.Now() err := entry.Entity.WriteToDB(q.db, entry.IsNewRecord) - latency := time.Since(start).Seconds() + writeTime := time.Since(start).Seconds() if err != nil { q.stats.IncWriteBehindErrors(entry.Entity.WriteType()) @@ -156,5 +198,16 @@ func (q *Queue) writeEntry(entry *QueueEntry) { q.stats.IncWriteBehindWrites(entry.Entity.WriteType()) } - q.stats.ObserveWriteBehindLatency(entry.Entity.WriteType(), latency) + q.stats.ObserveWriteBehindLatency(entry.Entity.WriteType(), writeTime) + + // Track metrics for status logging + // Latency is from when entry became ready (ReadyAt) to write completion + latency := time.Since(entry.ReadyAt).Seconds() + + q.metricsMu.Lock() + q.totalWriteTime += writeTime + q.writeCount++ + q.totalLatency += latency + q.latencyCount++ + q.metricsMu.Unlock() } diff --git a/decoder/writebehind/types.go b/decoder/writebehind/types.go index 840a2995..4030d736 100644 --- a/decoder/writebehind/types.go +++ b/decoder/writebehind/types.go @@ -25,6 +25,7 @@ type QueueEntry struct { Entity Writeable QueuedAt time.Time UpdatedAt time.Time + ReadyAt time.Time // When the entry became ready to write (set by dispatcher) IsNewRecord bool // Track if this needs INSERT (preserved across updates) Delay time.Duration // Minimum delay before writing (0 = immediate) } From 75f9c51ef8aac2d35f8d54572cf74ca251164742 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 4 Feb 2026 17:56:49 +0000 Subject: [PATCH 44/78] RaidSeed/QuestSeed as internal fields --- decoder/gym.go | 15 +++++++++++++++ decoder/gym_decode.go | 2 ++ decoder/gym_state.go | 35 ++++++++++++++++++++++------------- decoder/pokestop.go | 35 +++++++++++++++++++++++++++++++++++ decoder/pokestop_decode.go | 4 ++++ decoder/pokestop_state.go | 30 ++++++++++++++++++++---------- 6 files changed, 98 insertions(+), 23 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index 902cb15b..2236ff67 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -54,6 +54,9 @@ type Gym struct { Defenders null.String `db:"defenders"` Rsvps null.String `db:"rsvps"` + // Memory-only fields (not persisted to DB) + RaidSeed null.String `db:"-"` // Raid seed (memory only, sent in webhook) + dirty bool `db:"-"` // Not persisted - tracks if object needs saving (to db) internalDirty bool `db:"-"` // Not persisted - tracks if object needs saving (in memory only) newRecord bool `db:"-"` // Not persisted - tracks if this is a new record @@ -569,6 +572,18 @@ func (gym *Gym) SetRsvps(v null.String) { } } +// SetRaidSeed sets the raid seed (memory only, not saved to DB) +func (gym *Gym) SetRaidSeed(v null.String) { + if gym.RaidSeed != v { + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidSeed:%s->%s", FormatNull(gym.RaidSeed), FormatNull(v))) + } + gym.RaidSeed = v + // Do not set dirty, as this doesn't trigger a DB update + gym.internalDirty = true + } +} + func (gym *Gym) SetUpdated(v int64) { if gym.Updated != v { if dbDebugEnabled { diff --git a/decoder/gym_decode.go b/decoder/gym_decode.go index 3bec72aa..d30397d4 100644 --- a/decoder/gym_decode.go +++ b/decoder/gym_decode.go @@ -4,6 +4,7 @@ import ( "cmp" "encoding/json" "slices" + "strconv" "strings" "time" @@ -123,6 +124,7 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64 if fortData.RaidInfo != nil { gym.SetRaidEndTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000)) gym.SetRaidSpawnTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000)) + gym.SetRaidSeed(null.StringFrom(strconv.FormatInt(fortData.RaidInfo.RaidSeed, 10))) raidBattleTimestamp := int64(fortData.RaidInfo.RaidBattleMs) / 1000 if gym.RaidBattleTimestamp.ValueOrZero() != raidBattleTimestamp { diff --git a/decoder/gym_state.go b/decoder/gym_state.go index 2e927a59..d966c652 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -7,13 +7,14 @@ import ( "errors" "time" + "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + "golbat/config" "golbat/db" "golbat/geo" "golbat/webhooks" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" ) // gymSelectColumns defines the columns for gym queries. @@ -225,6 +226,7 @@ type RaidWebhook struct { PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` ArScanEligible int64 `json:"ar_scan_eligible"` Rsvps json.RawMessage `json:"rsvps"` + RaidSeed null.String `json:"raid_seed"` } func createGymFortWebhooks(gym *Gym) { @@ -326,6 +328,7 @@ func createGymWebhooks(gym *Gym, areas []geo.AreaName) { PowerUpEndTimestamp: gym.PowerUpEndTimestamp.ValueOrZero(), ArScanEligible: gym.ArScanEligible.ValueOrZero(), Rsvps: rsvps, + RaidSeed: gym.RaidSeed, } webhooksSender.AddMessage(webhooks.Raid, raidHook, areas) @@ -350,19 +353,25 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { // Debug logging before queueing if dbDebugEnabled { - if isNewRecord { - dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) - } else if gym.IsDirty() { - dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) + if gym.IsDirty() { + if isNewRecord { + dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) + } else { + dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) + } + } else { + dbDebugLog("MEMORY", "Gym", gym.Id, gym.changedFields) } } - // Queue the write through the write-behind system - if writeBehindQueue != nil && gym.IsDirty() { - writeBehindQueue.Enqueue(gym, isNewRecord, 0) - } else if gym.IsDirty() { - // Fallback to direct write if queue not initialized - _ = gymWriteDB(db, gym, isNewRecord) + if gym.IsDirty() { + // Queue the write through the write-behind system + if writeBehindQueue != nil { + writeBehindQueue.Enqueue(gym, isNewRecord, 0) + } else { + // Fallback to direct write if queue not initialized + _ = gymWriteDB(db, gym, isNewRecord) + } } if config.Config.FortInMemory { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index e77f75e0..7769ebde 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -65,7 +65,12 @@ type Pokestop struct { ShowcaseExpiry null.Int `db:"showcase_expiry"` ShowcaseRankings null.String `db:"showcase_rankings"` + // Memory-only fields (not persisted to DB) + QuestSeed null.Int `db:"-"` // Quest seed for AR quest (memory only, sent in webhook) + AlternativeQuestSeed null.Int `db:"-"` // Quest seed for non-AR quest (memory only, sent in webhook) + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + internalDirty bool `db:"-"` // Not persisted - tracks if object needs saving (in memory only) newRecord bool `db:"-"` // Not persisted - tracks if this is a new record changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) @@ -129,9 +134,15 @@ func (p *Pokestop) IsDirty() bool { return p.dirty } +// IsInternalDirty returns true if any field has been modified for in-memory +func (p *Pokestop) IsInternalDirty() bool { + return p.internalDirty +} + // ClearDirty resets the dirty flag (call after saving to DB) func (p *Pokestop) ClearDirty() { p.dirty = false + p.internalDirty = false } // IsNewRecord returns true if this is a new record (not yet in DB) @@ -697,3 +708,27 @@ func (p *Pokestop) SetUpdated(v int64) { p.dirty = true } } + +// SetQuestSeed sets the quest seed (memory only, not saved to DB) +func (p *Pokestop) SetQuestSeed(v null.Int) { + if p.QuestSeed != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestSeed:%s->%s", FormatNull(p.QuestSeed), FormatNull(v))) + } + p.QuestSeed = v + // Do not set dirty, as this doesn't trigger a DB update + p.internalDirty = true + } +} + +// SetAlternativeQuestSeed sets the alternative quest seed (memory only, not saved to DB) +func (p *Pokestop) SetAlternativeQuestSeed(v null.Int) { + if p.AlternativeQuestSeed != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestSeed:%s->%s", FormatNull(p.AlternativeQuestSeed), FormatNull(v))) + } + p.AlternativeQuestSeed = v + // Do not set dirty, as this doesn't trigger a DB update + p.internalDirty = true + } +} diff --git a/decoder/pokestop_decode.go b/decoder/pokestop_decode.go index 86746bd9..4b6f7722 100644 --- a/decoder/pokestop_decode.go +++ b/decoder/pokestop_decode.go @@ -382,6 +382,8 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu questExpiry = null.IntFrom(time.Now().Unix() + 24*60*60) // Set expiry to 24 hours from now } + questSeed := null.IntFrom(questData.QuestSeed) + if !haveAr { stop.SetAlternativeQuestType(null.IntFrom(questType)) stop.SetAlternativeQuestTarget(null.IntFrom(questTarget)) @@ -396,6 +398,7 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu stop.SetAlternativeQuestRewardAmount(rewardAmount) stop.SetAlternativeQuestPokemonId(rewardPokemonId) stop.SetAlternativeQuestPokemonFormId(rewardPokemonFormId) + stop.SetAlternativeQuestSeed(questSeed) } else { stop.SetQuestType(null.IntFrom(questType)) stop.SetQuestTarget(null.IntFrom(questTarget)) @@ -410,6 +413,7 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu stop.SetQuestRewardAmount(rewardAmount) stop.SetQuestPokemonId(rewardPokemonId) stop.SetQuestPokemonFormId(rewardPokemonFormId) + stop.SetQuestSeed(questSeed) } return questTitle diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index 28047201..f79d232f 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -148,6 +148,7 @@ type QuestWebhook struct { ArScanEligible int64 `json:"ar_scan_eligible"` PokestopUrl string `json:"pokestop_url"` WithAr bool `json:"with_ar"` + QuestSeed null.Int `json:"quest_seed"` } type PokestopWebhook struct { @@ -217,6 +218,7 @@ func createPokestopWebhooks(stop *Pokestop) { ArScanEligible: stop.ArScanEligible.ValueOrZero(), PokestopUrl: stop.Url.ValueOrZero(), WithAr: false, + QuestSeed: stop.AlternativeQuestSeed, } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } @@ -237,6 +239,7 @@ func createPokestopWebhooks(stop *Pokestop) { ArScanEligible: stop.ArScanEligible.ValueOrZero(), PokestopUrl: stop.Url.ValueOrZero(), WithAr: true, + QuestSeed: stop.QuestSeed, } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } @@ -276,10 +279,10 @@ func createPokestopWebhooks(stop *Pokestop) { func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { now := time.Now().Unix() - if !pokestop.IsNewRecord() && !pokestop.IsDirty() { + if !pokestop.IsNewRecord() && !pokestop.IsDirty() && !pokestop.IsInternalDirty() { // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. if pokestop.Updated > now-GetUpdateThreshold(900) { - // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again + // if a pokestop is unchanged and was seen recently, skip saving return } } @@ -290,19 +293,26 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop // Debug logging happens here, before queueing if dbDebugEnabled { - if isNewRecord { - dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) + if pokestop.IsDirty() { + if isNewRecord { + dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) + } else { + dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) + } } else { - dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) + dbDebugLog("MEMORY", "Pokestop", pokestop.Id, pokestop.changedFields) } } // Queue the write through the write-behind system (no delay for pokestops) - if writeBehindQueue != nil { - writeBehindQueue.Enqueue(pokestop, isNewRecord, 0) - } else { - // Fallback to direct write if queue not initialized - _ = pokestopWriteDB(db, pokestop, isNewRecord) + // Only queue if dirty (not just internalDirty) + if pokestop.IsDirty() { + if writeBehindQueue != nil { + writeBehindQueue.Enqueue(pokestop, isNewRecord, 0) + } else { + // Fallback to direct write if queue not initialized + _ = pokestopWriteDB(db, pokestop, isNewRecord) + } } if dbDebugEnabled { From 8407bb19f140be48cda376d005fe7c47eeb2e068 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 4 Feb 2026 20:30:52 +0000 Subject: [PATCH 45/78] Amend default in comment --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 7d21e568..ea13a777 100644 --- a/config/config.go +++ b/config/config.go @@ -127,7 +127,7 @@ type tuning struct { MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` ReduceUpdates bool `koanf:"reduce_updates"` WriteBehindStartupDelay int `koanf:"write_behind_startup_delay"` // seconds, default: 120 - WriteBehindWorkerCount int `koanf:"write_behind_worker_count"` // concurrent writers, default: 100 + WriteBehindWorkerCount int `koanf:"write_behind_worker_count"` // concurrent writers, default: 50 } type scanRule struct { From f89d4495c6c8e6ee97bb8c5f798d62c10ffe3331 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 4 Feb 2026 21:25:39 +0000 Subject: [PATCH 46/78] Fort tracker goes through write behind --- config.toml.example | 23 +++++++++++----------- db/gym.go | 19 ------------------- db/pokestop.go | 14 -------------- decoder/fort_tracker.go | 42 +++++++++++++++++++++++++---------------- 4 files changed, 38 insertions(+), 60 deletions(-) delete mode 100644 db/gym.go diff --git a/config.toml.example b/config.toml.example index ed23dabf..629aea9d 100644 --- a/config.toml.example +++ b/config.toml.example @@ -3,18 +3,11 @@ port = 9001 # Listening port for golbat raw_bearer = "" # Raw bearer (password) required api_secret = "golbat" # Golbat secret required on api calls (blank for none) -# When enabled, reduce_updates will make fort update debounce windows much longer -# to reduce database churn. Specifically, gym/pokestop/station debounce will be -# extended from 15 minutes (900s) to 12 hours (43200s) and spawnpoint last_seen -# will be updated every 12 hours instead of the default 6 hours. -reduce_updates = false - pokemon_memory_only = false # Use in-memory storage for pokemon only -# Fort loading options: -# preload_forts: Pre-load all forts into cache on startup (faster initial responses) +# preload: Pre-load objects into cache on startup (faster initial responses, but increases startup time) # fort_in_memory: Keep forts in memory with rtree for spatial lookups -preload_forts = false +preload = false fort_in_memory = false [koji] @@ -75,8 +68,16 @@ url = "http://localhost:4201" [tuning] max_pokemon_distance = 100 # Maximum distance in kilometers for searching pokemon max_pokemon_results = 3000 # Maximum number of pokemon to return -extended_timeout = false -profile_routes = false +extended_timeout = false # Extend timeouts for processing, hopefully not needed +profile_routes = false # Turn on debugging endpoints + +# When enabled, reduce_updates will make fort update debounce windows much longer +# to reduce database churn. Specifically, gym/pokestop/station debounce will be +# extended from 15 minutes (900s) to 12 hours (43200s) and spawnpoint last_seen +# will be updated every 12 hours instead of the default 6 hours. +reduce_updates = false +write_behind_startup_delay = 120 +write_behind_worker_count = 50 [weather] proactive_iv_switching = true # Enable proactive IV switching upon weather changes (default: true) diff --git a/db/gym.go b/db/gym.go deleted file mode 100644 index 1e5377fd..00000000 --- a/db/gym.go +++ /dev/null @@ -1,19 +0,0 @@ -package db - -import ( - "context" - - "github.com/jmoiron/sqlx" -) - -func ClearOldGyms(ctx context.Context, db DbDetails, gymIds []string) error { - query, args, _ := sqlx.In("UPDATE gym SET deleted = 1 WHERE id IN (?);", gymIds) - query = db.GeneralDb.Rebind(query) - - _, err := db.GeneralDb.ExecContext(ctx, query, args...) - statsCollector.IncDbQuery("clear old-gyms", err) - if err != nil { - return err - } - return nil -} diff --git a/db/pokestop.go b/db/pokestop.go index 897289a1..35276b8f 100644 --- a/db/pokestop.go +++ b/db/pokestop.go @@ -1,10 +1,8 @@ package db import ( - "context" "database/sql" - "github.com/jmoiron/sqlx" "github.com/paulmach/orb/geojson" ) @@ -48,18 +46,6 @@ func GetPokestopPositions(db DbDetails, fence *geojson.Feature) ([]QuestLocation return areas, nil } -func ClearOldPokestops(ctx context.Context, db DbDetails, stopIds []string) error { - query, args, _ := sqlx.In("UPDATE pokestop SET deleted = 1 WHERE id IN (?);", stopIds) - query = db.GeneralDb.Rebind(query) - - _, err := db.GeneralDb.ExecContext(ctx, query, args...) - statsCollector.IncDbQuery("clear old-pokestops", err) - if err != nil { - return err - } - return nil -} - func GetQuestStatus(db DbDetails, fence *geojson.Feature) (QuestStatus, error) { bbox := fence.Geometry.Bound() status := QuestStatus{} diff --git a/decoder/fort_tracker.go b/decoder/fort_tracker.go index f543fbc5..5094c62a 100644 --- a/decoder/fort_tracker.go +++ b/decoder/fort_tracker.go @@ -432,17 +432,22 @@ func GetFortTracker() *FortTracker { // clearGymWithLock marks a gym as deleted while holding the object-level mutex func clearGymWithLock(ctx context.Context, dbDetails db.DbDetails, gymId string, cellId uint64, removeFromTracker bool) { - // Lock the gym if it exists in cache - gym, unlock, _ := PeekGymRecord(gymId) - if gym != nil { - defer unlock() + // Load gym through cache (will load from DB if not cached) + gym, unlock, err := getGymRecordForUpdate(ctx, dbDetails, gymId) + if err != nil { + log.Errorf("FortTracker: failed to load gym %s - %s", gymId, err) + return } - - gymCache.Delete(gymId) - if err := db.ClearOldGyms(ctx, dbDetails, []string{gymId}); err != nil { - log.Errorf("FortTracker: failed to clear gym %s - %s", gymId, err) + if gym == nil { + log.Warnf("FortTracker: gym %s not found in cache or database", gymId) return } + defer unlock() + + // Mark as deleted and save through write-behind queue + gym.SetDeleted(true) + saveGymRecord(ctx, dbDetails, gym) + if removeFromTracker { fortTracker.RemoveFort(gymId) log.Infof("FortTracker: removed gym in cell %d: %s", cellId, gymId) @@ -456,17 +461,22 @@ func clearGymWithLock(ctx context.Context, dbDetails db.DbDetails, gymId string, // clearPokestopWithLock marks a pokestop as deleted while holding the object-level mutex func clearPokestopWithLock(ctx context.Context, dbDetails db.DbDetails, stopId string, cellId uint64, removeFromTracker bool) { - // Lock the pokestop if it exists in cache - pokestop, unlock, _ := PeekPokestopRecord(stopId) - if pokestop != nil { - defer unlock() + // Load pokestop through cache (will load from DB if not cached) + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, dbDetails, stopId) + if err != nil { + log.Errorf("FortTracker: failed to load pokestop %s - %s", stopId, err) + return } - - pokestopCache.Delete(stopId) - if err := db.ClearOldPokestops(ctx, dbDetails, []string{stopId}); err != nil { - log.Errorf("FortTracker: failed to clear pokestop %s - %s", stopId, err) + if pokestop == nil { + log.Warnf("FortTracker: pokestop %s not found in cache or database", stopId) return } + defer unlock() + + // Mark as deleted and save through write-behind queue + pokestop.SetDeleted(true) + savePokestopRecord(ctx, dbDetails, pokestop) + if removeFromTracker { fortTracker.RemoveFort(stopId) log.Infof("FortTracker: removed pokestop in cell %d: %s", cellId, stopId) From 2c0ade64080d0718d2369c117fb9c2f9d8fde9e6 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 5 Feb 2026 18:52:03 +0000 Subject: [PATCH 47/78] implementation of batch writer --- config/config.go | 2 + config/reader.go | 2 + decoder/api_pokemon_common.go | 3 +- decoder/api_pokemon_scan_v2.go | 2 +- decoder/api_pokemon_scan_v3.go | 2 +- decoder/main.go | 9 +- decoder/pokemon.go | 2 +- decoder/pokemonRtree.go | 8 +- decoder/pokemon_decode.go | 6 +- decoder/pokemon_state.go | 18 +- decoder/stats.go | 8 +- decoder/uint64str.go | 56 +++++ decoder/uint64str_test.go | 140 +++++++++++++ decoder/writebehind/batch.go | 169 +++++++++++++++ decoder/writebehind/processor.go | 10 +- decoder/writebehind/queue.go | 65 +++++- decoder/writebehind/types.go | 6 +- decoder/writebehind_batch.go | 343 +++++++++++++++++++++++++++++++ 18 files changed, 813 insertions(+), 38 deletions(-) create mode 100644 decoder/uint64str.go create mode 100644 decoder/uint64str_test.go create mode 100644 decoder/writebehind/batch.go create mode 100644 decoder/writebehind_batch.go diff --git a/config/config.go b/config/config.go index ea13a777..b481c87b 100644 --- a/config/config.go +++ b/config/config.go @@ -128,6 +128,8 @@ type tuning struct { ReduceUpdates bool `koanf:"reduce_updates"` WriteBehindStartupDelay int `koanf:"write_behind_startup_delay"` // seconds, default: 120 WriteBehindWorkerCount int `koanf:"write_behind_worker_count"` // concurrent writers, default: 50 + WriteBehindBatchSize int `koanf:"write_behind_batch_size"` // entries per batch, default: 50 + WriteBehindBatchTimeoutMs int `koanf:"write_behind_batch_timeout"` // max wait for batch in ms, default: 100 } type scanRule struct { diff --git a/config/reader.go b/config/reader.go index c77df9c3..d391185f 100644 --- a/config/reader.go +++ b/config/reader.go @@ -57,6 +57,8 @@ func ReadConfig() (configDefinition, error) { ReduceUpdates: false, WriteBehindStartupDelay: 120, // 2 minutes WriteBehindWorkerCount: 50, // concurrent writers + WriteBehindBatchSize: 50, // entries per batch + WriteBehindBatchTimeoutMs: 100, // ms to wait for batch to fill }, Weather: weather{ ProactiveIVSwitching: true, diff --git a/decoder/api_pokemon_common.go b/decoder/api_pokemon_common.go index c9eeca16..04199013 100644 --- a/decoder/api_pokemon_common.go +++ b/decoder/api_pokemon_common.go @@ -2,7 +2,6 @@ package decoder import ( "math" - "strconv" "time" "golbat/config" @@ -71,7 +70,7 @@ type ApiPokemonResult struct { func buildApiPokemonResult(pokemon *Pokemon) ApiPokemonResult { return ApiPokemonResult{ - Id: strconv.FormatUint(pokemon.Id, 10), + Id: pokemon.Id.String(), PokestopId: pokemon.PokestopId, SpawnId: pokemon.SpawnId, Lat: pokemon.Lat, diff --git a/decoder/api_pokemon_scan_v2.go b/decoder/api_pokemon_scan_v2.go index ae029c90..4603c036 100644 --- a/decoder/api_pokemon_scan_v2.go +++ b/decoder/api_pokemon_scan_v2.go @@ -187,7 +187,7 @@ func GrpcGetPokemonInArea2(retrieveParameters *pb.PokemonScanRequest) []*pb.Poke if pokemon != nil { if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { apiPokemon := pb.PokemonDetails{ - Id: pokemon.Id, + Id: uint64(pokemon.Id), PokestopId: pokemon.PokestopId.Ptr(), SpawnId: pokemon.SpawnId.Ptr(), Lat: pokemon.Lat, diff --git a/decoder/api_pokemon_scan_v3.go b/decoder/api_pokemon_scan_v3.go index 6cb5e220..a53b7aeb 100644 --- a/decoder/api_pokemon_scan_v3.go +++ b/decoder/api_pokemon_scan_v3.go @@ -204,7 +204,7 @@ func GrpcGetPokemonInArea3(retrieveParameters *pb.PokemonScanRequestV3) ([]*pb.P if pokemon != nil { if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { apiPokemon := pb.PokemonDetails{ - Id: pokemon.Id, + Id: uint64(pokemon.Id), PokestopId: pokemon.PokestopId.Ptr(), SpawnId: pokemon.SpawnId.Ptr(), Lat: pokemon.Lat, diff --git a/decoder/main.go b/decoder/main.go index fa7e683f..c77c6bc9 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -298,12 +298,17 @@ func InitWriteBehindQueue(ctx context.Context, dbDetails db.DbDetails) { cfg := writebehind.QueueConfig{ StartupDelaySeconds: config.Config.Tuning.WriteBehindStartupDelay, WorkerCount: config.Config.Tuning.WriteBehindWorkerCount, + BatchSize: config.Config.Tuning.WriteBehindBatchSize, + BatchTimeout: time.Duration(config.Config.Tuning.WriteBehindBatchTimeoutMs) * time.Millisecond, } writeBehindQueue = writebehind.NewQueue(cfg, dbDetails, statsCollector) - log.Infof("Write-behind queue initialized: startup_delay=%ds, workers=%d", - cfg.StartupDelaySeconds, cfg.WorkerCount) + // Register batch writers for each entity type + RegisterBatchWriters(writeBehindQueue) + + log.Infof("Write-behind queue initialized: startup_delay=%ds, workers=%d, batch_size=%d, batch_timeout=%dms", + cfg.StartupDelaySeconds, cfg.WorkerCount, cfg.BatchSize, cfg.BatchTimeout.Milliseconds()) // Warn if worker count exceeds half of database pool size maxPool := config.Config.Database.MaxPool diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 56cf62b1..ebb1ad7b 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -20,7 +20,7 @@ import ( type Pokemon struct { mu sync.Mutex `db:"-"` // Object-level mutex - Id uint64 `db:"id"` + Id Uint64Str `db:"id"` PokestopId null.String `db:"pokestop_id"` SpawnId null.Int `db:"spawn_id"` Lat float64 `db:"lat"` diff --git a/decoder/pokemonRtree.go b/decoder/pokemonRtree.go index 16dff824..20a18d51 100644 --- a/decoder/pokemonRtree.go +++ b/decoder/pokemonRtree.go @@ -53,13 +53,13 @@ func initPokemonRtree() { // Set up OnEviction callback on all shards pokemonCache.OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, *Pokemon]) { pokemon := v.Value() - removePokemonFromTree(pokemon.Id, pokemon.Lat, pokemon.Lon) + removePokemonFromTree(uint64(pokemon.Id), pokemon.Lat, pokemon.Lon) // Rely on the pokemon pvp lookup caches to remove themselves rather than trying to synchronise }) } func pokemonRtreeUpdatePokemonOnGet(pokemon *Pokemon) { - pokemonId := pokemon.Id + pokemonId := uint64(pokemon.Id) _, inMap := pokemonLookupCache.Load(pokemonId) @@ -78,7 +78,7 @@ func valueOrMinus1(n null.Int) int { } func updatePokemonLookup(pokemon *Pokemon, changePvp bool, pvpResults map[string][]gohbem.PokemonEntry) { - pokemonId := pokemon.Id + pokemonId := uint64(pokemon.Id) pokemonLookupCacheItem, _ := pokemonLookupCache.Load(pokemonId) @@ -151,7 +151,7 @@ func calculatePokemonPvpLookup(pokemon *Pokemon, pvpResults map[string][]gohbem. } func addPokemonToTree(pokemon *Pokemon) { - pokemonId := pokemon.Id + pokemonId := uint64(pokemon.Id) pokemonTreeMutex.Lock() pokemonTree.Insert([2]float64{pokemon.Lon, pokemon.Lat}, [2]float64{pokemon.Lon, pokemon.Lat}, pokemonId) diff --git a/decoder/pokemon_decode.go b/decoder/pokemon_decode.go index e29de264..5a53ddd1 100644 --- a/decoder/pokemon_decode.go +++ b/decoder/pokemon_decode.go @@ -77,7 +77,7 @@ func (pokemon *Pokemon) remainingDuration(now int64) time.Duration { } func (pokemon *Pokemon) addWildPokemon(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, timestampMs int64, trustworthyTimestamp bool) { - if wildPokemon.EncounterId != pokemon.Id { + if wildPokemon.EncounterId != uint64(pokemon.Id) { panic("Unmatched EncounterId") } pokemon.SetLat(wildPokemon.Latitude) @@ -160,7 +160,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP pokemon.SetIsEvent(0) - pokemon.Id = mapPokemon.EncounterId + pokemon.Id = Uint64Str(mapPokemon.EncounterId) spawnpointId := mapPokemon.SpawnpointId @@ -191,7 +191,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP pokemon.SetExpireTimestamp(null.IntFrom(mapPokemon.ExpirationTimeMs / 1000)) pokemon.SetExpireTimestampVerified(true) // if we have cached an encounter for this pokemon, update the TTL. - encounterCache.UpdateTTL(pokemon.Id, pokemon.remainingDuration(timestampMs/1000)) + encounterCache.UpdateTTL(uint64(pokemon.Id), pokemon.remainingDuration(timestampMs/1000)) } else { pokemon.SetExpireTimestampVerified(false) } diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index 44faac02..7de88dd8 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -104,7 +104,7 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { // Create new Pokemon atomically - function only called if key doesn't exist pokemonItem, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - return &Pokemon{Id: encounterId, newRecord: true} + return &Pokemon{Id: Uint64Str(encounterId), newRecord: true} }) pokemon := pokemonItem.Value() @@ -254,9 +254,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po // Debug logging happens here, before queueing if dbDebugEnabled { if isNewRecord { - dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + dbDebugLog("INSERT", "Pokemon", pokemon.Id.String(), pokemon.changedFields) } else { - dbDebugLog("UPDATE", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + dbDebugLog("UPDATE", "Pokemon", pokemon.Id.String(), pokemon.changedFields) } } @@ -277,7 +277,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } else { if dbDebugEnabled { - dbDebugLog("MEMORY", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + dbDebugLog("MEMORY", "Pokemon", pokemon.Id.String(), pokemon.changedFields) } } @@ -286,7 +286,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po addPokemonToTree(pokemon) } else if pokemon.Lat != pokemon.oldValues.Lat || pokemon.Lon != pokemon.oldValues.Lon { // Position changed - update R-tree by removing from old position and adding to new - removePokemonFromTree(pokemon.Id, pokemon.oldValues.Lat, pokemon.oldValues.Lon) + removePokemonFromTree(uint64(pokemon.Id), pokemon.oldValues.Lat, pokemon.oldValues.Lon) addPokemonToTree(pokemon) } @@ -308,7 +308,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache if db.UsePokemonCache { - pokemonCache.Set(pokemon.Id, pokemon, pokemon.remainingDuration(now)) + pokemonCache.Set(uint64(pokemon.Id), pokemon, pokemon.remainingDuration(now)) } } @@ -337,7 +337,7 @@ func pokemonWriteDB(db db.DbDetails, pokemon *Pokemon, isNewRecord bool) error { statsCollector.IncDbQuery("insert pokemon", err) if err != nil { log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) - pokemonCache.Delete(pokemon.Id) + pokemonCache.Delete(uint64(pokemon.Id)) return err } } else { @@ -387,7 +387,7 @@ func pokemonWriteDB(db db.DbDetails, pokemon *Pokemon, isNewRecord bool) error { statsCollector.IncDbQuery("update pokemon", err) if err != nil { log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) - pokemonCache.Delete(pokemon.Id) + pokemonCache.Delete(uint64(pokemon.Id)) return err } } @@ -467,7 +467,7 @@ func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemo SpawnpointId: spawnpointId, PokestopId: pokestopId, PokestopName: pokestopName, - EncounterId: strconv.FormatUint(pokemon.Id, 10), + EncounterId: pokemon.Id.String(), PokemonId: pokemon.PokemonId, Latitude: pokemon.Lat, Longitude: pokemon.Lon, diff --git a/decoder/stats.go b/decoder/stats.go index d5312bf9..b3f5be7a 100644 --- a/decoder/stats.go +++ b/decoder/stats.go @@ -173,7 +173,7 @@ func updateEncounterStats(pokemon *Pokemon) { username = "" } - encounterCacheVal := encounterCache.GetOrCreate(pokemon.Id) + encounterCacheVal := encounterCache.GetOrCreate(uint64(pokemon.Id)) isNewEncounter := encounterCacheVal.NumAccountsSeen() == 0 if encounterCacheVal.SetAccountSeen(pokemon.Username.ValueOrZero()) { @@ -188,7 +188,7 @@ func updateEncounterStats(pokemon *Pokemon) { statsCollector.IncDuplicateEncounters(false) } - encounterCache.Put(pokemon.Id, encounterCacheVal, pokemon.remainingDuration(time.Now().Unix())) + encounterCache.Put(uint64(pokemon.Id), encounterCacheVal, pokemon.remainingDuration(time.Now().Unix())) pokemonIdStr := strconv.Itoa(int(pokemon.PokemonId)) var formId int @@ -273,7 +273,7 @@ func updatePokemonStats(pokemon *Pokemon, areas []geo.AreaName, now int64) { populateEncounterCacheVal := func() { if encounterCacheVal == nil { - encounterCacheVal = encounterCache.GetOrCreate(pokemon.Id) + encounterCacheVal = encounterCache.GetOrCreate(uint64(pokemon.Id)) } } @@ -334,7 +334,7 @@ func updatePokemonStats(pokemon *Pokemon, areas []geo.AreaName, now int64) { // If we have a cache entry, it means we updated it. So now let's store it. if encounterCacheVal != nil { - encounterCache.Put(pokemon.Id, encounterCacheVal, pokemon.remainingDuration(now)) + encounterCache.Put(uint64(pokemon.Id), encounterCacheVal, pokemon.remainingDuration(now)) } if (currentSeenType == SeenType_Wild && oldSeenType == SeenType_Encounter) || diff --git a/decoder/uint64str.go b/decoder/uint64str.go new file mode 100644 index 00000000..62ef3c8f --- /dev/null +++ b/decoder/uint64str.go @@ -0,0 +1,56 @@ +package decoder + +import ( + "database/sql/driver" + "fmt" + "strconv" +) + +// Uint64Str is a uint64 that serializes to/from database as a string. +// This is useful for columns that store large integers as VARCHAR. +type Uint64Str uint64 + +// Value implements driver.Valuer - outputs as string for database writes +func (u Uint64Str) Value() (driver.Value, error) { + return strconv.FormatUint(uint64(u), 10), nil +} + +// Scan implements sql.Scanner - reads string from database into uint64 +func (u *Uint64Str) Scan(src interface{}) error { + if src == nil { + *u = 0 + return nil + } + + switch v := src.(type) { + case string: + parsed, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return fmt.Errorf("Uint64Str.Scan: cannot parse %q: %w", v, err) + } + *u = Uint64Str(parsed) + case []byte: + parsed, err := strconv.ParseUint(string(v), 10, 64) + if err != nil { + return fmt.Errorf("Uint64Str.Scan: cannot parse %q: %w", string(v), err) + } + *u = Uint64Str(parsed) + case int64: + *u = Uint64Str(v) + case uint64: + *u = Uint64Str(v) + default: + return fmt.Errorf("Uint64Str.Scan: unsupported type %T", src) + } + return nil +} + +// Uint64 returns the underlying uint64 value +func (u Uint64Str) Uint64() uint64 { + return uint64(u) +} + +// String returns the string representation +func (u Uint64Str) String() string { + return strconv.FormatUint(uint64(u), 10) +} diff --git a/decoder/uint64str_test.go b/decoder/uint64str_test.go new file mode 100644 index 00000000..3fa134f3 --- /dev/null +++ b/decoder/uint64str_test.go @@ -0,0 +1,140 @@ +package decoder + +import ( + "testing" + + "github.com/jmoiron/sqlx" +) + +func TestUint64StrWithSqlxNamed(t *testing.T) { + // Simple struct with Uint64Str field + type TestRecord struct { + Id Uint64Str `db:"id"` + Name string `db:"name"` + } + + records := []TestRecord{ + {Id: Uint64Str(12345678901234567890), Name: "first"}, + {Id: Uint64Str(9876543210987654321), Name: "second"}, + {Id: Uint64Str(1111111111111111111), Name: "third"}, + } + + query := `INSERT INTO test (id, name) VALUES (:id, :name)` + + // Call sqlx.Named with slice of records + expandedQuery, args, err := sqlx.Named(query, records) + if err != nil { + t.Fatalf("sqlx.Named failed: %v", err) + } + + // Verify query was expanded for 3 records (sqlx uses no space after comma) + expectedQuery := `INSERT INTO test (id, name) VALUES (?, ?),(?, ?),(?, ?)` + if expandedQuery != expectedQuery { + t.Errorf("Query mismatch:\nexpected: %s\ngot: %s", expectedQuery, expandedQuery) + } + + // Verify we have 6 args (2 per record * 3 records) + if len(args) != 6 { + t.Fatalf("Expected 6 args, got %d", len(args)) + } + + // sqlx.Named extracts the Uint64Str values as-is; the database driver + // will call Value() when executing to convert to string. + // Here we verify the Uint64Str values are present and correct. + expectedIds := []Uint64Str{ + Uint64Str(12345678901234567890), + Uint64Str(9876543210987654321), + Uint64Str(1111111111111111111), + } + expectedNames := []string{"first", "second", "third"} + + for i := 0; i < 3; i++ { + idArg, ok := args[i*2].(Uint64Str) + if !ok { + t.Errorf("Arg[%d] is not Uint64Str: got %T", i*2, args[i*2]) + continue + } + if idArg != expectedIds[i] { + t.Errorf("Arg[%d] ID mismatch: expected %d, got %d", i*2, expectedIds[i], idArg) + } + + nameArg, ok := args[i*2+1].(string) + if !ok { + t.Errorf("Arg[%d] is not string: got %T", i*2+1, args[i*2+1]) + continue + } + if nameArg != expectedNames[i] { + t.Errorf("Arg[%d] name mismatch: expected %s, got %s", i*2+1, expectedNames[i], nameArg) + } + } + + // Verify that Value() produces strings (this is what the DB driver will call) + for i, id := range expectedIds { + val, err := id.Value() + if err != nil { + t.Errorf("Value() error for record %d: %v", i, err) + continue + } + strVal, ok := val.(string) + if !ok { + t.Errorf("Value() for record %d returned %T, expected string", i, val) + continue + } + t.Logf("Record %d: Uint64Str(%d) -> Value() -> %q", i, id, strVal) + } + + t.Logf("Expanded query: %s", expandedQuery) + t.Logf("Args types: %T, %T, %T, %T, %T, %T", args[0], args[1], args[2], args[3], args[4], args[5]) +} + +func TestUint64StrValuer(t *testing.T) { + tests := []struct { + input Uint64Str + expected string + }{ + {Uint64Str(0), "0"}, + {Uint64Str(12345), "12345"}, + {Uint64Str(18446744073709551615), "18446744073709551615"}, // max uint64 + } + + for _, tc := range tests { + val, err := tc.input.Value() + if err != nil { + t.Errorf("Value() error for %d: %v", tc.input, err) + continue + } + strVal, ok := val.(string) + if !ok { + t.Errorf("Value() returned %T, expected string", val) + continue + } + if strVal != tc.expected { + t.Errorf("Value() = %q, expected %q", strVal, tc.expected) + } + } +} + +func TestUint64StrScanner(t *testing.T) { + tests := []struct { + input interface{} + expected Uint64Str + }{ + {"12345", Uint64Str(12345)}, + {[]byte("67890"), Uint64Str(67890)}, + {int64(99999), Uint64Str(99999)}, + {uint64(88888), Uint64Str(88888)}, + {nil, Uint64Str(0)}, + } + + for _, tc := range tests { + var u Uint64Str + err := u.Scan(tc.input) + if err != nil { + t.Errorf("Scan(%v) error: %v", tc.input, err) + continue + } + if u != tc.expected { + t.Errorf("Scan(%v) = %d, expected %d", tc.input, u, tc.expected) + } + } +} diff --git a/decoder/writebehind/batch.go b/decoder/writebehind/batch.go new file mode 100644 index 00000000..260c3054 --- /dev/null +++ b/decoder/writebehind/batch.go @@ -0,0 +1,169 @@ +package writebehind + +import ( + "context" + "sync" + "time" + + "github.com/jmoiron/sqlx" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/stats_collector" +) + +// BatchWriter handles batched writes for a specific table type +type BatchWriter struct { + mu sync.Mutex + entries []*QueueEntry + timer *time.Timer + batchSize int + timeout time.Duration + flushFunc func(ctx context.Context, db db.DbDetails, entries []*QueueEntry) error + db db.DbDetails + stats stats_collector.StatsCollector + tableType string + queue *Queue // Reference to parent queue for metrics +} + +// BatchWriterConfig holds configuration for a batch writer +type BatchWriterConfig struct { + BatchSize int + Timeout time.Duration + TableType string + FlushFunc func(ctx context.Context, db db.DbDetails, entries []*QueueEntry) error + Db db.DbDetails + Stats stats_collector.StatsCollector + Queue *Queue +} + +// NewBatchWriter creates a new batch writer for a table type +func NewBatchWriter(cfg BatchWriterConfig) *BatchWriter { + return &BatchWriter{ + entries: make([]*QueueEntry, 0, cfg.BatchSize), + batchSize: cfg.BatchSize, + timeout: cfg.Timeout, + flushFunc: cfg.FlushFunc, + db: cfg.Db, + stats: cfg.Stats, + tableType: cfg.TableType, + queue: cfg.Queue, + } +} + +// Add adds an entry to the batch, flushing if batch is full +func (bw *BatchWriter) Add(entry *QueueEntry) { + bw.mu.Lock() + defer bw.mu.Unlock() + + bw.entries = append(bw.entries, entry) + + if len(bw.entries) >= bw.batchSize { + bw.flushLocked() + } else if bw.timer == nil { + // Start timeout for partial batch + bw.timer = time.AfterFunc(bw.timeout, func() { + bw.mu.Lock() + defer bw.mu.Unlock() + if len(bw.entries) > 0 { + bw.flushLocked() + } + }) + } +} + +// flushLocked flushes the current batch (must be called with lock held) +func (bw *BatchWriter) flushLocked() { + if bw.timer != nil { + bw.timer.Stop() + bw.timer = nil + } + + if len(bw.entries) == 0 { + return + } + + // Take ownership of entries slice + entries := bw.entries + bw.entries = make([]*QueueEntry, 0, bw.batchSize) + + // Release lock before doing I/O + bw.mu.Unlock() + + // Execute batch write + start := time.Now() + ctx := context.Background() + err := bw.flushFunc(ctx, bw.db, entries) + writeTime := time.Since(start).Seconds() + + if err != nil { + bw.stats.IncWriteBehindErrors(bw.tableType) + log.Errorf("Write-behind batch error for %s (%d entries): %v", bw.tableType, len(entries), err) + } else { + // Increment write count by number of entries in batch + for range entries { + bw.stats.IncWriteBehindWrites(bw.tableType) + } + log.Debugf("Write-behind batch wrote %d %s entries in %.1fms", len(entries), bw.tableType, writeTime*1000) + } + + // Track metrics on the parent queue + bw.queue.metricsMu.Lock() + bw.queue.totalWriteTime += writeTime + bw.queue.writeCount += int64(len(entries)) + for _, entry := range entries { + latency := time.Since(entry.ReadyAt).Seconds() + bw.queue.totalLatency += latency + bw.queue.latencyCount++ + } + bw.queue.metricsMu.Unlock() + + // Re-acquire lock (caller expects it held) + bw.mu.Lock() +} + +// Flush forces a flush of any pending entries +func (bw *BatchWriter) Flush() { + bw.mu.Lock() + defer bw.mu.Unlock() + if len(bw.entries) > 0 { + bw.flushLocked() + } +} + +// Size returns number of pending entries +func (bw *BatchWriter) Size() int { + bw.mu.Lock() + defer bw.mu.Unlock() + return len(bw.entries) +} + +// ExecuteBatchUpsert builds and executes a batch upsert using sqlx.Named +// lockFunc should lock all entities and return an unlock function +// The query should use :field placeholders matching the struct's db tags +func ExecuteBatchUpsert( + ctx context.Context, + dbConn *sqlx.DB, + query string, + entities interface{}, + lockFunc func() func(), +) error { + // Lock all entities to read their values + unlock := lockFunc() + + // Generate SQL and args while holding locks + expandedQuery, args, err := sqlx.Named(query, entities) + + // Release locks - args now contains the values + unlock() + + if err != nil { + return err + } + + expandedQuery = dbConn.Rebind(expandedQuery) + + // Execute without locks held + _, err = dbConn.ExecContext(ctx, expandedQuery, args...) + return err +} diff --git a/decoder/writebehind/processor.go b/decoder/writebehind/processor.go index 436c1ded..9848b81a 100644 --- a/decoder/writebehind/processor.go +++ b/decoder/writebehind/processor.go @@ -27,14 +27,20 @@ func (q *Queue) ProcessLoop(ctx context.Context) { q.dispatcher(ctx) } -// worker reads from the work channel and writes entries to DB +// worker reads from the work channel and routes to batch writers func (q *Queue) worker(ctx context.Context, id int) { for { select { case <-ctx.Done(): return case entry := <-q.workChan: - q.writeEntry(entry) + // Route to batch writer if registered, otherwise write directly + tableType := entry.Entity.WriteType() + if bw := q.getBatchWriter(tableType); bw != nil { + bw.Add(entry) + } else { + q.writeEntry(entry) + } } } } diff --git a/decoder/writebehind/queue.go b/decoder/writebehind/queue.go index 72869bd3..84bab4f5 100644 --- a/decoder/writebehind/queue.go +++ b/decoder/writebehind/queue.go @@ -1,6 +1,7 @@ package writebehind import ( + "context" "sync" "time" @@ -15,7 +16,11 @@ type Queue struct { mu sync.Mutex pending map[string]*QueueEntry // key -> entry for squashing - workChan chan *QueueEntry // buffered channel for workers + workChan chan *QueueEntry // buffered channel for dispatcher + + // Batch writers per table type + batchWriters map[string]*BatchWriter + batchWritersMu sync.RWMutex workerCount int warmupComplete bool @@ -38,10 +43,17 @@ func NewQueue(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.Sta if cfg.WorkerCount <= 0 { cfg.WorkerCount = 50 // default } + if cfg.BatchSize <= 0 { + cfg.BatchSize = 50 // default + } + if cfg.BatchTimeout <= 0 { + cfg.BatchTimeout = 100 * time.Millisecond // default + } return &Queue{ pending: make(map[string]*QueueEntry), workChan: make(chan *QueueEntry, cfg.WorkerCount*10), // buffer 10x worker count + batchWriters: make(map[string]*BatchWriter), workerCount: cfg.WorkerCount, warmupComplete: false, startTime: time.Now(), @@ -51,6 +63,29 @@ func NewQueue(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.Sta } } +// RegisterBatchWriter registers a batch writer for a specific table type +func (q *Queue) RegisterBatchWriter(tableType string, flushFunc func(ctx context.Context, db db.DbDetails, entries []*QueueEntry) error) { + q.batchWritersMu.Lock() + defer q.batchWritersMu.Unlock() + + q.batchWriters[tableType] = NewBatchWriter(BatchWriterConfig{ + BatchSize: q.config.BatchSize, + Timeout: q.config.BatchTimeout, + TableType: tableType, + FlushFunc: flushFunc, + Db: q.db, + Stats: q.stats, + Queue: q, // Pass queue reference for metrics + }) +} + +// getBatchWriter returns the batch writer for a table type, or nil if not registered +func (q *Queue) getBatchWriter(tableType string) *BatchWriter { + q.batchWritersMu.RLock() + defer q.batchWritersMu.RUnlock() + return q.batchWriters[tableType] +} + // Enqueue adds or updates an entity write // If an entry already exists for the same key: // - Entity is replaced with the newer one @@ -171,15 +206,31 @@ func (q *Queue) Flush() { q.mu.Unlock() if len(entries) == 0 { - return + log.Info("Write-behind flush: no pending entries") + } else { + log.Infof("Write-behind flushing %d pending entries", len(entries)) + + // Route entries to batch writers or write directly + for _, entry := range entries { + tableType := entry.Entity.WriteType() + if bw := q.getBatchWriter(tableType); bw != nil { + bw.Add(entry) + } else { + q.writeEntry(entry) + } + } } - log.Infof("Write-behind flushing %d entries", len(entries)) - - // Write all entries directly (bypass channel during shutdown) - for _, entry := range entries { - q.writeEntry(entry) + // Flush all batch writers + q.batchWritersMu.RLock() + for tableType, bw := range q.batchWriters { + size := bw.Size() + if size > 0 { + log.Infof("Write-behind flushing %d %s batch entries", size, tableType) + } + bw.Flush() } + q.batchWritersMu.RUnlock() log.Info("Write-behind flush complete") } diff --git a/decoder/writebehind/types.go b/decoder/writebehind/types.go index 4030d736..5f8c5f04 100644 --- a/decoder/writebehind/types.go +++ b/decoder/writebehind/types.go @@ -32,6 +32,8 @@ type QueueEntry struct { // QueueConfig holds configuration for the write-behind queue type QueueConfig struct { - StartupDelaySeconds int // Delay before processing starts (warmup period) - WorkerCount int // Number of concurrent write workers (default 100) + StartupDelaySeconds int // Delay before processing starts (warmup period) + WorkerCount int // Number of concurrent write workers (default 50) + BatchSize int // Number of entries per batch (default 50) + BatchTimeout time.Duration // Max time to wait for a full batch (default 100ms) } diff --git a/decoder/writebehind_batch.go b/decoder/writebehind_batch.go new file mode 100644 index 00000000..e7139183 --- /dev/null +++ b/decoder/writebehind_batch.go @@ -0,0 +1,343 @@ +package decoder + +import ( + "context" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/decoder/writebehind" +) + +// RegisterBatchWriters registers all batch writers with the write-behind queue +func RegisterBatchWriters(queue *writebehind.Queue) { + queue.RegisterBatchWriter("pokestop", flushPokestopBatch) + queue.RegisterBatchWriter("gym", flushGymBatch) + queue.RegisterBatchWriter("pokemon", flushPokemonBatch) + queue.RegisterBatchWriter("spawnpoint", flushSpawnpointBatch) + + log.Info("Write-behind batch writers registered for: pokestop, gym, pokemon, spawnpoint") +} + +// flushPokestopBatch writes a batch of pokestops using INSERT ... ON DUPLICATE KEY UPDATE +func flushPokestopBatch(ctx context.Context, dbDetails db.DbDetails, entries []*writebehind.QueueEntry) error { + pokestops := make([]*Pokestop, len(entries)) + for i, e := range entries { + pokestops[i] = e.Entity.(*Pokestop) + } + + return writebehind.ExecuteBatchUpsert( + ctx, + dbDetails.GeneralDb, + pokestopBatchUpsertQuery, + pokestops, + func() func() { + // Lock all pokestops + for _, p := range pokestops { + p.Lock() + } + // Return unlock function + return func() { + for _, p := range pokestops { + p.Unlock() + } + } + }, + ) +} + +// flushGymBatch writes a batch of gyms using INSERT ... ON DUPLICATE KEY UPDATE +func flushGymBatch(ctx context.Context, dbDetails db.DbDetails, entries []*writebehind.QueueEntry) error { + gyms := make([]*Gym, len(entries)) + for i, e := range entries { + gyms[i] = e.Entity.(*Gym) + } + + return writebehind.ExecuteBatchUpsert( + ctx, + dbDetails.GeneralDb, + gymBatchUpsertQuery, + gyms, + func() func() { + for _, g := range gyms { + g.Lock() + } + return func() { + for _, g := range gyms { + g.Unlock() + } + } + }, + ) +} + +// flushPokemonBatch writes a batch of pokemon using INSERT ... ON DUPLICATE KEY UPDATE +func flushPokemonBatch(ctx context.Context, dbDetails db.DbDetails, entries []*writebehind.QueueEntry) error { + pokemon := make([]*Pokemon, len(entries)) + for i, e := range entries { + pokemon[i] = e.Entity.(*Pokemon) + } + + return writebehind.ExecuteBatchUpsert( + ctx, + dbDetails.PokemonDb, + pokemonBatchUpsertQuery, + pokemon, + func() func() { + for _, p := range pokemon { + p.Lock() + } + return func() { + for _, p := range pokemon { + p.Unlock() + } + } + }, + ) +} + +// flushSpawnpointBatch writes a batch of spawnpoints using INSERT ... ON DUPLICATE KEY UPDATE +func flushSpawnpointBatch(ctx context.Context, dbDetails db.DbDetails, entries []*writebehind.QueueEntry) error { + spawnpoints := make([]*Spawnpoint, len(entries)) + for i, e := range entries { + spawnpoints[i] = e.Entity.(*Spawnpoint) + } + + return writebehind.ExecuteBatchUpsert( + ctx, + dbDetails.GeneralDb, + spawnpointBatchUpsertQuery, + spawnpoints, + func() func() { + for _, s := range spawnpoints { + s.Lock() + } + return func() { + for _, s := range spawnpoints { + s.Unlock() + } + } + }, + ) +} + +// Batch upsert queries - using INSERT ... ON DUPLICATE KEY UPDATE +// This eliminates need to track isNewRecord + +const pokestopBatchUpsertQuery = ` +INSERT INTO pokestop ( + id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, + quest_type, quest_timestamp, quest_target, quest_conditions, quest_rewards, + quest_template, quest_title, quest_expiry, quest_reward_type, quest_item_id, + quest_reward_amount, quest_pokemon_id, quest_pokemon_form_id, + alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, + alternative_quest_conditions, alternative_quest_rewards, alternative_quest_template, + alternative_quest_title, alternative_quest_expiry, alternative_quest_reward_type, + alternative_quest_item_id, alternative_quest_reward_amount, + alternative_quest_pokemon_id, alternative_quest_pokemon_form_id, + cell_id, lure_id, deleted, sponsor_id, partner_id, ar_scan_eligible, + power_up_points, power_up_level, power_up_end_timestamp, updated, first_seen_timestamp, + description, showcase_focus, showcase_pokemon_id, showcase_pokemon_form_id, + showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings +) +VALUES ( + :id, :lat, :lon, :name, :url, :enabled, :lure_expire_timestamp, :last_modified_timestamp, + :quest_type, :quest_timestamp, :quest_target, :quest_conditions, :quest_rewards, + :quest_template, :quest_title, :quest_expiry, :quest_reward_type, :quest_item_id, + :quest_reward_amount, :quest_pokemon_id, :quest_pokemon_form_id, + :alternative_quest_type, :alternative_quest_timestamp, :alternative_quest_target, + :alternative_quest_conditions, :alternative_quest_rewards, :alternative_quest_template, + :alternative_quest_title, :alternative_quest_expiry, :alternative_quest_reward_type, + :alternative_quest_item_id, :alternative_quest_reward_amount, + :alternative_quest_pokemon_id, :alternative_quest_pokemon_form_id, + :cell_id, :lure_id, :deleted, :sponsor_id, :partner_id, :ar_scan_eligible, + :power_up_points, :power_up_level, :power_up_end_timestamp, :updated, UNIX_TIMESTAMP(), + :description, :showcase_focus, :showcase_pokemon_id, :showcase_pokemon_form_id, + :showcase_pokemon_type_id, :showcase_ranking_standard, :showcase_expiry, :showcase_rankings +) +ON DUPLICATE KEY UPDATE + lat = VALUES(lat), + lon = VALUES(lon), + name = VALUES(name), + url = VALUES(url), + enabled = VALUES(enabled), + lure_expire_timestamp = VALUES(lure_expire_timestamp), + last_modified_timestamp = VALUES(last_modified_timestamp), + quest_type = VALUES(quest_type), + quest_timestamp = VALUES(quest_timestamp), + quest_target = VALUES(quest_target), + quest_conditions = VALUES(quest_conditions), + quest_rewards = VALUES(quest_rewards), + quest_template = VALUES(quest_template), + quest_title = VALUES(quest_title), + quest_expiry = VALUES(quest_expiry), + quest_reward_type = VALUES(quest_reward_type), + quest_item_id = VALUES(quest_item_id), + quest_reward_amount = VALUES(quest_reward_amount), + quest_pokemon_id = VALUES(quest_pokemon_id), + quest_pokemon_form_id = VALUES(quest_pokemon_form_id), + alternative_quest_type = VALUES(alternative_quest_type), + alternative_quest_timestamp = VALUES(alternative_quest_timestamp), + alternative_quest_target = VALUES(alternative_quest_target), + alternative_quest_conditions = VALUES(alternative_quest_conditions), + alternative_quest_rewards = VALUES(alternative_quest_rewards), + alternative_quest_template = VALUES(alternative_quest_template), + alternative_quest_title = VALUES(alternative_quest_title), + alternative_quest_expiry = VALUES(alternative_quest_expiry), + alternative_quest_reward_type = VALUES(alternative_quest_reward_type), + alternative_quest_item_id = VALUES(alternative_quest_item_id), + alternative_quest_reward_amount = VALUES(alternative_quest_reward_amount), + alternative_quest_pokemon_id = VALUES(alternative_quest_pokemon_id), + alternative_quest_pokemon_form_id = VALUES(alternative_quest_pokemon_form_id), + cell_id = VALUES(cell_id), + lure_id = VALUES(lure_id), + deleted = VALUES(deleted), + sponsor_id = VALUES(sponsor_id), + partner_id = VALUES(partner_id), + ar_scan_eligible = VALUES(ar_scan_eligible), + power_up_points = VALUES(power_up_points), + power_up_level = VALUES(power_up_level), + power_up_end_timestamp = VALUES(power_up_end_timestamp), + updated = VALUES(updated), + description = VALUES(description), + showcase_focus = VALUES(showcase_focus), + showcase_pokemon_id = VALUES(showcase_pokemon_id), + showcase_pokemon_form_id = VALUES(showcase_pokemon_form_id), + showcase_pokemon_type_id = VALUES(showcase_pokemon_type_id), + showcase_ranking_standard = VALUES(showcase_ranking_standard), + showcase_expiry = VALUES(showcase_expiry), + showcase_rankings = VALUES(showcase_rankings) +` + +const gymBatchUpsertQuery = ` +INSERT INTO gym ( + id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, + raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, + guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, + raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, + raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, + raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, + raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, + raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, + power_up_end_timestamp, description, rsvps, defenders +) +VALUES ( + :id, :lat, :lon, :name, :url, :last_modified_timestamp, :raid_end_timestamp, + :raid_spawn_timestamp, :raid_battle_timestamp, :updated, :raid_pokemon_id, + :guarding_pokemon_id, :guarding_pokemon_display, :available_slots, :team_id, + :raid_level, :enabled, :ex_raid_eligible, :in_battle, :raid_pokemon_move_1, + :raid_pokemon_move_2, :raid_pokemon_form, :raid_pokemon_alignment, :raid_pokemon_cp, + :raid_is_exclusive, :cell_id, :deleted, :total_cp, UNIX_TIMESTAMP(), + :raid_pokemon_gender, :sponsor_id, :partner_id, :raid_pokemon_costume, + :raid_pokemon_evolution, :ar_scan_eligible, :power_up_level, :power_up_points, + :power_up_end_timestamp, :description, :rsvps, :defenders +) +ON DUPLICATE KEY UPDATE + lat = VALUES(lat), + lon = VALUES(lon), + name = VALUES(name), + url = VALUES(url), + last_modified_timestamp = VALUES(last_modified_timestamp), + raid_end_timestamp = VALUES(raid_end_timestamp), + raid_spawn_timestamp = VALUES(raid_spawn_timestamp), + raid_battle_timestamp = VALUES(raid_battle_timestamp), + updated = VALUES(updated), + raid_pokemon_id = VALUES(raid_pokemon_id), + guarding_pokemon_id = VALUES(guarding_pokemon_id), + guarding_pokemon_display = VALUES(guarding_pokemon_display), + available_slots = VALUES(available_slots), + team_id = VALUES(team_id), + raid_level = VALUES(raid_level), + enabled = VALUES(enabled), + ex_raid_eligible = VALUES(ex_raid_eligible), + in_battle = VALUES(in_battle), + raid_pokemon_move_1 = VALUES(raid_pokemon_move_1), + raid_pokemon_move_2 = VALUES(raid_pokemon_move_2), + raid_pokemon_form = VALUES(raid_pokemon_form), + raid_pokemon_alignment = VALUES(raid_pokemon_alignment), + raid_pokemon_cp = VALUES(raid_pokemon_cp), + raid_is_exclusive = VALUES(raid_is_exclusive), + cell_id = VALUES(cell_id), + deleted = VALUES(deleted), + total_cp = VALUES(total_cp), + raid_pokemon_gender = VALUES(raid_pokemon_gender), + sponsor_id = VALUES(sponsor_id), + partner_id = VALUES(partner_id), + raid_pokemon_costume = VALUES(raid_pokemon_costume), + raid_pokemon_evolution = VALUES(raid_pokemon_evolution), + ar_scan_eligible = VALUES(ar_scan_eligible), + power_up_level = VALUES(power_up_level), + power_up_points = VALUES(power_up_points), + power_up_end_timestamp = VALUES(power_up_end_timestamp), + description = VALUES(description), + rsvps = VALUES(rsvps), + defenders = VALUES(defenders) +` + +const pokemonBatchUpsertQuery = ` +INSERT INTO pokemon ( + id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, + golbat_internal, iv, move_1, move_2, gender, form, cp, level, strong, weather, + costume, weight, height, size, display_pokemon_id, is_ditto, pokestop_id, + updated, first_seen_timestamp, changed, cell_id, expire_timestamp_verified, + shiny, username, pvp, is_event, seen_type +) +VALUES ( + :id, :pokemon_id, :lat, :lon, :spawn_id, :expire_timestamp, :atk_iv, :def_iv, :sta_iv, + :golbat_internal, :iv, :move_1, :move_2, :gender, :form, :cp, :level, :strong, :weather, + :costume, :weight, :height, :size, :display_pokemon_id, :is_ditto, :pokestop_id, + :updated, :first_seen_timestamp, :changed, :cell_id, :expire_timestamp_verified, + :shiny, :username, :pvp, :is_event, :seen_type +) +ON DUPLICATE KEY UPDATE + pokemon_id = VALUES(pokemon_id), + lat = VALUES(lat), + lon = VALUES(lon), + spawn_id = VALUES(spawn_id), + expire_timestamp = VALUES(expire_timestamp), + atk_iv = VALUES(atk_iv), + def_iv = VALUES(def_iv), + sta_iv = VALUES(sta_iv), + golbat_internal = VALUES(golbat_internal), + iv = VALUES(iv), + move_1 = VALUES(move_1), + move_2 = VALUES(move_2), + gender = VALUES(gender), + form = VALUES(form), + cp = VALUES(cp), + level = VALUES(level), + strong = VALUES(strong), + weather = VALUES(weather), + costume = VALUES(costume), + weight = VALUES(weight), + height = VALUES(height), + size = VALUES(size), + display_pokemon_id = VALUES(display_pokemon_id), + is_ditto = VALUES(is_ditto), + pokestop_id = VALUES(pokestop_id), + updated = VALUES(updated), + first_seen_timestamp = VALUES(first_seen_timestamp), + changed = VALUES(changed), + cell_id = VALUES(cell_id), + expire_timestamp_verified = VALUES(expire_timestamp_verified), + shiny = VALUES(shiny), + username = VALUES(username), + pvp = VALUES(pvp), + is_event = VALUES(is_event), + seen_type = VALUES(seen_type) +` + +const spawnpointBatchUpsertQuery = ` +INSERT INTO spawnpoint ( + id, lat, lon, updated, despawn_sec +) +VALUES ( + :id, :lat, :lon, :updated, :last_seen, :despawn_sec +) +ON DUPLICATE KEY UPDATE + lat = VALUES(lat), + lon = VALUES(lon), + updated = VALUES(updated), + last_seen=VALUES(last_seen), + despawn_sec = VALUES(despawn_sec) +` From 314cc37161a763d9c68db58caa228c26d59ff290 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 5 Feb 2026 18:59:19 +0000 Subject: [PATCH 48/78] Fix spawnpoint upsert query --- decoder/writebehind_batch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decoder/writebehind_batch.go b/decoder/writebehind_batch.go index e7139183..b0effbc7 100644 --- a/decoder/writebehind_batch.go +++ b/decoder/writebehind_batch.go @@ -329,7 +329,7 @@ ON DUPLICATE KEY UPDATE const spawnpointBatchUpsertQuery = ` INSERT INTO spawnpoint ( - id, lat, lon, updated, despawn_sec + id, lat, lon, updated, last_seen, despawn_sec ) VALUES ( :id, :lat, :lon, :updated, :last_seen, :despawn_sec From 1d6681239d44231f6d617202b4efcf38ba69c4f6 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 5 Feb 2026 19:36:53 +0000 Subject: [PATCH 49/78] Improve logging --- decoder/writebehind/batch.go | 25 +++++--- decoder/writebehind/processor.go | 8 ++- decoder/writebehind/queue.go | 100 ++++++++++++++++++++--------- stats_collector/noop.go | 17 +++-- stats_collector/prometheus.go | 39 +++++++++++ stats_collector/stats_collector.go | 3 + 6 files changed, 144 insertions(+), 48 deletions(-) diff --git a/decoder/writebehind/batch.go b/decoder/writebehind/batch.go index 260c3054..3333322d 100644 --- a/decoder/writebehind/batch.go +++ b/decoder/writebehind/batch.go @@ -94,27 +94,36 @@ func (bw *BatchWriter) flushLocked() { start := time.Now() ctx := context.Background() err := bw.flushFunc(ctx, bw.db, entries) - writeTime := time.Since(start).Seconds() + batchTime := time.Since(start).Seconds() + entryCount := len(entries) if err != nil { bw.stats.IncWriteBehindErrors(bw.tableType) - log.Errorf("Write-behind batch error for %s (%d entries): %v", bw.tableType, len(entries), err) + log.Errorf("Write-behind batch error for %s (%d entries): %v", bw.tableType, entryCount, err) } else { // Increment write count by number of entries in batch for range entries { bw.stats.IncWriteBehindWrites(bw.tableType) } - log.Debugf("Write-behind batch wrote %d %s entries in %.1fms", len(entries), bw.tableType, writeTime*1000) + // Record batch metrics in Prometheus + bw.stats.IncWriteBehindBatches(bw.tableType) + bw.stats.ObserveWriteBehindBatchSize(bw.tableType, float64(entryCount)) + bw.stats.ObserveWriteBehindBatchTime(bw.tableType, batchTime) + + log.Debugf("Write-behind batch wrote %d %s entries in %.1fms", entryCount, bw.tableType, batchTime*1000) } - // Track metrics on the parent queue + // Track batch metrics on the parent queue bw.queue.metricsMu.Lock() - bw.queue.totalWriteTime += writeTime - bw.queue.writeCount += int64(len(entries)) + bw.queue.batchCount++ + bw.queue.batchEntryCount += int64(entryCount) + bw.queue.batchWriteTime += batchTime for _, entry := range entries { latency := time.Since(entry.ReadyAt).Seconds() - bw.queue.totalLatency += latency - bw.queue.latencyCount++ + bw.queue.batchLatency += latency + bw.queue.batchLatencyCount++ + // Also record per-entry latency in Prometheus + bw.stats.ObserveWriteBehindLatency(bw.tableType, latency) } bw.queue.metricsMu.Unlock() diff --git a/decoder/writebehind/processor.go b/decoder/writebehind/processor.go index 9848b81a..ee7b66e1 100644 --- a/decoder/writebehind/processor.go +++ b/decoder/writebehind/processor.go @@ -62,9 +62,11 @@ func (q *Queue) dispatcher(ctx context.Context) { case <-statusTicker.C: queueSize := q.Size() channelLen := len(q.workChan) - avgWriteTime, avgLatency, writeCount := q.GetAndResetMetrics() - log.Infof("Write-behind queue: %d pending, %d in channel, %d writes (avg write: %.1fms, avg latency: %.1fms)", - queueSize, channelLen, writeCount, avgWriteTime, avgLatency) + metrics := q.GetAndResetMetrics() + log.Infof("Write-behind queue: %d pending, %d in channel | single: %d entries (avg write: %.1fms, avg latency: %.1fms) | batch: %d entries in %d batches (avg write: %.1fms, avg latency: %.1fms)", + queueSize, channelLen, + metrics.SingleEntryCount, metrics.SingleAvgWriteMs, metrics.SingleAvgLatencyMs, + metrics.BatchEntryCount, metrics.BatchCount, metrics.BatchAvgWriteMs, metrics.BatchAvgLatencyMs) case <-ticker.C: if q.checkWarmup() { q.dispatchReady() diff --git a/decoder/writebehind/queue.go b/decoder/writebehind/queue.go index 84bab4f5..b5cad8fb 100644 --- a/decoder/writebehind/queue.go +++ b/decoder/writebehind/queue.go @@ -27,11 +27,18 @@ type Queue struct { startTime time.Time // Metrics tracking (protected by metricsMu) - metricsMu sync.Mutex - totalWriteTime float64 // sum of write durations in seconds - writeCount int64 // number of writes completed - totalLatency float64 // sum of latencies (ready to complete) in seconds - latencyCount int64 // number of latency samples + metricsMu sync.Mutex + // Single write metrics + singleWriteCount int64 // number of single entries written + singleWriteTime float64 // sum of single write durations in seconds + singleLatency float64 // sum of single write latencies in seconds + singleLatencyCount int64 // number of single latency samples + // Batch write metrics + batchCount int64 // number of batches executed + batchEntryCount int64 // total entries written via batches + batchWriteTime float64 // sum of batch execution times in seconds + batchLatency float64 // sum of batch entry latencies in seconds + batchLatencyCount int64 // number of batch latency samples config QueueConfig db db.DbDetails @@ -141,29 +148,62 @@ func (q *Queue) Size() int { return len(q.pending) } -// GetAndResetMetrics returns average write time and latency, then resets counters -// Returns (avgWriteTime, avgLatency, count) - times in milliseconds -func (q *Queue) GetAndResetMetrics() (float64, float64, int64) { +// WriteMetrics holds the metrics returned by GetAndResetMetrics +type WriteMetrics struct { + // Single write metrics + SingleEntryCount int64 // number of entries written via single writes + SingleAvgWriteMs float64 // average write time per single entry in milliseconds + SingleAvgLatencyMs float64 // average latency per single entry in milliseconds + // Batch write metrics + BatchCount int64 // number of batches executed + BatchEntryCount int64 // total entries written via batches + BatchAvgWriteMs float64 // average time per batch in milliseconds + BatchAvgLatencyMs float64 // average latency per batch entry in milliseconds +} + +// GetAndResetMetrics returns write metrics then resets counters +func (q *Queue) GetAndResetMetrics() WriteMetrics { q.metricsMu.Lock() defer q.metricsMu.Unlock() - var avgWriteTime, avgLatency float64 - count := q.writeCount + var singleAvgWrite, singleAvgLatency float64 + var batchAvgWrite, batchAvgLatency float64 - if q.writeCount > 0 { - avgWriteTime = (q.totalWriteTime / float64(q.writeCount)) * 1000 // convert to ms + if q.singleWriteCount > 0 { + singleAvgWrite = (q.singleWriteTime / float64(q.singleWriteCount)) * 1000 + } + if q.singleLatencyCount > 0 { + singleAvgLatency = (q.singleLatency / float64(q.singleLatencyCount)) * 1000 + } + if q.batchCount > 0 { + batchAvgWrite = (q.batchWriteTime / float64(q.batchCount)) * 1000 } - if q.latencyCount > 0 { - avgLatency = (q.totalLatency / float64(q.latencyCount)) * 1000 // convert to ms + if q.batchLatencyCount > 0 { + batchAvgLatency = (q.batchLatency / float64(q.batchLatencyCount)) * 1000 } - // Reset counters - q.totalWriteTime = 0 - q.writeCount = 0 - q.totalLatency = 0 - q.latencyCount = 0 + metrics := WriteMetrics{ + SingleEntryCount: q.singleWriteCount, + SingleAvgWriteMs: singleAvgWrite, + SingleAvgLatencyMs: singleAvgLatency, + BatchCount: q.batchCount, + BatchEntryCount: q.batchEntryCount, + BatchAvgWriteMs: batchAvgWrite, + BatchAvgLatencyMs: batchAvgLatency, + } - return avgWriteTime, avgLatency, count + // Reset counters + q.singleWriteCount = 0 + q.singleWriteTime = 0 + q.singleLatency = 0 + q.singleLatencyCount = 0 + q.batchCount = 0 + q.batchEntryCount = 0 + q.batchWriteTime = 0 + q.batchLatency = 0 + q.batchLatencyCount = 0 + + return metrics } // IsWarmupComplete returns true if the warmup period has elapsed @@ -235,30 +275,30 @@ func (q *Queue) Flush() { log.Info("Write-behind flush complete") } -// writeEntry performs the actual database write for an entry +// writeEntry performs the actual database write for an entry (single write path) func (q *Queue) writeEntry(entry *QueueEntry) { start := time.Now() err := entry.Entity.WriteToDB(q.db, entry.IsNewRecord) writeTime := time.Since(start).Seconds() + entityType := entry.Entity.WriteType() if err != nil { - q.stats.IncWriteBehindErrors(entry.Entity.WriteType()) + q.stats.IncWriteBehindErrors(entityType) log.Errorf("Write-behind error for %s: %v", entry.Key, err) } else { - q.stats.IncWriteBehindWrites(entry.Entity.WriteType()) + q.stats.IncWriteBehindWrites(entityType) } - q.stats.ObserveWriteBehindLatency(entry.Entity.WriteType(), writeTime) - - // Track metrics for status logging // Latency is from when entry became ready (ReadyAt) to write completion latency := time.Since(entry.ReadyAt).Seconds() + q.stats.ObserveWriteBehindLatency(entityType, latency) + // Track single write metrics for status logging q.metricsMu.Lock() - q.totalWriteTime += writeTime - q.writeCount++ - q.totalLatency += latency - q.latencyCount++ + q.singleWriteCount++ + q.singleWriteTime += writeTime + q.singleLatency += latency + q.singleLatencyCount++ q.metricsMu.Unlock() } diff --git a/stats_collector/noop.go b/stats_collector/noop.go index 24bbf8d1..36f5d506 100644 --- a/stats_collector/noop.go +++ b/stats_collector/noop.go @@ -52,13 +52,16 @@ func (col *noopCollector) UpdateMaxBattleCount([]geo.AreaName, int64) func (col *noopCollector) IncFortChange(string) {} // Write-behind queue metrics (noop) -func (col *noopCollector) SetWriteBehindQueueDepth(string, float64) {} -func (col *noopCollector) IncWriteBehindSquashed(string) {} -func (col *noopCollector) IncWriteBehindRateLimited(string) {} -func (col *noopCollector) IncWriteBehindErrors(string) {} -func (col *noopCollector) IncWriteBehindWrites(string) {} -func (col *noopCollector) ObserveWriteBehindLatency(string, float64) {} -func (col *noopCollector) SetS2CellBatchSize(int) {} +func (col *noopCollector) SetWriteBehindQueueDepth(string, float64) {} +func (col *noopCollector) IncWriteBehindSquashed(string) {} +func (col *noopCollector) IncWriteBehindRateLimited(string) {} +func (col *noopCollector) IncWriteBehindErrors(string) {} +func (col *noopCollector) IncWriteBehindWrites(string) {} +func (col *noopCollector) ObserveWriteBehindLatency(string, float64) {} +func (col *noopCollector) IncWriteBehindBatches(string) {} +func (col *noopCollector) ObserveWriteBehindBatchSize(string, float64) {} +func (col *noopCollector) ObserveWriteBehindBatchTime(string, float64) {} +func (col *noopCollector) SetS2CellBatchSize(int) {} func NewNoopStatsCollector() StatsCollector { return &noopCollector{} diff --git a/stats_collector/prometheus.go b/stats_collector/prometheus.go index 875e2810..c6c7269c 100644 --- a/stats_collector/prometheus.go +++ b/stats_collector/prometheus.go @@ -386,6 +386,32 @@ var ( }, []string{"entity_type"}, ) + writeBehindBatches = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: ns, + Name: "write_behind_batches_total", + Help: "Total number of batches written by write-behind queue", + }, + []string{"entity_type"}, + ) + writeBehindBatchSize = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: ns, + Name: "write_behind_batch_size", + Help: "Number of entries per batch write", + Buckets: []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000}, + }, + []string{"entity_type"}, + ) + writeBehindBatchTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: ns, + Name: "write_behind_batch_time_seconds", + Help: "Time to execute a batch write in seconds", + Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, + }, + []string{"entity_type"}, + ) // S2Cell batch metrics s2CellBatchSize = prometheus.NewGauge( @@ -680,6 +706,18 @@ func (col *promCollector) ObserveWriteBehindLatency(entityType string, seconds f writeBehindLatency.WithLabelValues(entityType).Observe(seconds) } +func (col *promCollector) IncWriteBehindBatches(entityType string) { + writeBehindBatches.WithLabelValues(entityType).Inc() +} + +func (col *promCollector) ObserveWriteBehindBatchSize(entityType string, size float64) { + writeBehindBatchSize.WithLabelValues(entityType).Observe(size) +} + +func (col *promCollector) ObserveWriteBehindBatchTime(entityType string, seconds float64) { + writeBehindBatchTime.WithLabelValues(entityType).Observe(seconds) +} + func (col *promCollector) SetS2CellBatchSize(size int) { s2CellBatchSize.Set(float64(size)) } @@ -702,6 +740,7 @@ func initPrometheus() { writeBehindQueueDepth, writeBehindSquashed, writeBehindRateLimited, writeBehindErrors, writeBehindWrites, writeBehindLatency, + writeBehindBatches, writeBehindBatchSize, writeBehindBatchTime, s2CellBatchSize, ) } diff --git a/stats_collector/stats_collector.go b/stats_collector/stats_collector.go index deba00e5..2d8336e0 100644 --- a/stats_collector/stats_collector.go +++ b/stats_collector/stats_collector.go @@ -58,6 +58,9 @@ type StatsCollector interface { IncWriteBehindErrors(entityType string) IncWriteBehindWrites(entityType string) ObserveWriteBehindLatency(entityType string, seconds float64) + IncWriteBehindBatches(entityType string) + ObserveWriteBehindBatchSize(entityType string, size float64) + ObserveWriteBehindBatchTime(entityType string, seconds float64) // S2Cell batch metrics SetS2CellBatchSize(size int) From dce8b1dad5ad0379e65dedd1f8f954af45a75413 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 5 Feb 2026 21:33:28 +0000 Subject: [PATCH 50/78] Update spawnpoint through write behind --- decoder/spawnpoint.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 0883360d..866bdafa 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -276,6 +276,7 @@ func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon spawnpointUpdate(ctx, db, spawnpoint) } else { spawnpointSeen(ctx, db, spawnpoint) + spawnpointUpdate(ctx, db, spawnpoint) } unlock() } @@ -351,15 +352,14 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoint // update at least every 6 hours (21600s). If reduce_updates is enabled, use 12 hours. if now-spawnpoint.LastSeen > GetUpdateThreshold(21600) { spawnpoint.SetLastSeen(now) - - _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ - "SET last_seen=? "+ - "WHERE id = ? ", now, spawnpoint.Id) - statsCollector.IncDbQuery("update spawnpoint", err) - if err != nil { - log.Printf("Error updating spawnpoint last seen %s", err) - return - } + //_, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ + // "SET last_seen=? "+ + // "WHERE id = ? ", now, spawnpoint.Id) + //statsCollector.IncDbQuery("update spawnpoint", err) + //if err != nil { + // log.Printf("Error updating spawnpoint last seen %s", err) + // return + //} // Cache already contains a pointer, no need to update } } From 895be01dafc78eca028953f4c486ceb0716351a9 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 5 Feb 2026 22:00:35 +0000 Subject: [PATCH 51/78] Remove debug log --- decoder/writebehind/batch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decoder/writebehind/batch.go b/decoder/writebehind/batch.go index 3333322d..c552f344 100644 --- a/decoder/writebehind/batch.go +++ b/decoder/writebehind/batch.go @@ -110,7 +110,7 @@ func (bw *BatchWriter) flushLocked() { bw.stats.ObserveWriteBehindBatchSize(bw.tableType, float64(entryCount)) bw.stats.ObserveWriteBehindBatchTime(bw.tableType, batchTime) - log.Debugf("Write-behind batch wrote %d %s entries in %.1fms", entryCount, bw.tableType, batchTime*1000) + //log.Debugf("Write-behind batch wrote %d %s entries in %.1fms", entryCount, bw.tableType, batchTime*1000) } // Track batch metrics on the parent queue From 675fc053bf3ecf333b297d32699794ba9120aa2f Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 6 Feb 2026 09:49:50 +0000 Subject: [PATCH 52/78] Avoid deadlock --- decoder/writebehind/batch.go | 6 ++++++ decoder/writebehind_batch.go | 26 ++++++++++++++++---------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/decoder/writebehind/batch.go b/decoder/writebehind/batch.go index c552f344..4c841001 100644 --- a/decoder/writebehind/batch.go +++ b/decoder/writebehind/batch.go @@ -2,6 +2,7 @@ package writebehind import ( "context" + "sort" "sync" "time" @@ -87,6 +88,11 @@ func (bw *BatchWriter) flushLocked() { entries := bw.entries bw.entries = make([]*QueueEntry, 0, bw.batchSize) + // Sort entries by key to ensure consistent lock ordering and prevent deadlocks + sort.Slice(entries, func(i, j int) bool { + return entries[i].Key < entries[j].Key + }) + // Release lock before doing I/O bw.mu.Unlock() diff --git a/decoder/writebehind_batch.go b/decoder/writebehind_batch.go index b0effbc7..c9dce2de 100644 --- a/decoder/writebehind_batch.go +++ b/decoder/writebehind_batch.go @@ -32,14 +32,14 @@ func flushPokestopBatch(ctx context.Context, dbDetails db.DbDetails, entries []* pokestopBatchUpsertQuery, pokestops, func() func() { - // Lock all pokestops + // Lock all pokestops in sorted order for _, p := range pokestops { p.Lock() } - // Return unlock function + // Return unlock function (unlock in reverse order) return func() { - for _, p := range pokestops { - p.Unlock() + for i := len(pokestops) - 1; i >= 0; i-- { + pokestops[i].Unlock() } } }, @@ -59,12 +59,14 @@ func flushGymBatch(ctx context.Context, dbDetails db.DbDetails, entries []*write gymBatchUpsertQuery, gyms, func() func() { + // Lock all gyms in sorted order for _, g := range gyms { g.Lock() } + // Return unlock function (unlock in reverse order) return func() { - for _, g := range gyms { - g.Unlock() + for i := len(gyms) - 1; i >= 0; i-- { + gyms[i].Unlock() } } }, @@ -84,12 +86,14 @@ func flushPokemonBatch(ctx context.Context, dbDetails db.DbDetails, entries []*w pokemonBatchUpsertQuery, pokemon, func() func() { + // Lock all pokemon in sorted order for _, p := range pokemon { p.Lock() } + // Return unlock function (unlock in reverse order) return func() { - for _, p := range pokemon { - p.Unlock() + for i := len(pokemon) - 1; i >= 0; i-- { + pokemon[i].Unlock() } } }, @@ -109,12 +113,14 @@ func flushSpawnpointBatch(ctx context.Context, dbDetails db.DbDetails, entries [ spawnpointBatchUpsertQuery, spawnpoints, func() func() { + // Lock all spawnpoints in sorted order for _, s := range spawnpoints { s.Lock() } + // Return unlock function (unlock in reverse order) return func() { - for _, s := range spawnpoints { - s.Unlock() + for i := len(spawnpoints) - 1; i >= 0; i-- { + spawnpoints[i].Unlock() } } }, From bea539ed6d29198a731043f374cd6d5b7d20c8bc Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 6 Feb 2026 13:16:02 +0000 Subject: [PATCH 53/78] Ensure we can collect blocked and mutex details --- main.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 555ee8cd..068c6968 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "net/http/pprof" + "runtime" "sync" "time" _ "time/tzdata" @@ -337,7 +338,7 @@ func main() { pprof.Index(c.Writer, c.Request) }) pprofGroup.GET("/block", func(c *gin.Context) { - pprof.Index(c.Writer, c.Request) + pprof.Handler("block").ServeHTTP(c.Writer, c.Request) }) pprofGroup.GET("/mutex", func(c *gin.Context) { pprof.Index(c.Writer, c.Request) @@ -351,6 +352,11 @@ func main() { pprofGroup.GET("/symbol", func(c *gin.Context) { pprof.Symbol(c.Writer, c.Request) }) + pprofGroup.GET("/goroutine", func(c *gin.Context) { + pprof.Handler("goroutine").ServeHTTP(c.Writer, c.Request) + }) + runtime.SetBlockProfileRate(1) + runtime.SetMutexProfileFraction(1) } srv := &http.Server{ From a4af10d8b95b5b70880d27a9682b1366dbbe53b9 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 6 Feb 2026 16:14:56 +0000 Subject: [PATCH 54/78] Potentially save deadlock on gym/pokestop interaction --- decoder/fort.go | 118 ++++++++++++++++++++++++++------------ decoder/gmo_decode.go | 50 ++++++++++++---- decoder/gym_state.go | 17 ++++++ decoder/pokestop_state.go | 17 ++++++ 4 files changed, 154 insertions(+), 48 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index 4cc34417..17e20212 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -5,6 +5,7 @@ import ( "net/url" "strings" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" @@ -219,58 +220,103 @@ func UpdateFortRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetail return status, output } -// copySharedFieldsFrom copies shared fields from a pokestop to a gym during conversion -func (gym *Gym) copySharedFieldsFrom(pokestop *Pokestop) { - if pokestop.Name.Valid && !gym.Name.Valid { - gym.SetName(pokestop.Name) +// SharedFortFields holds fields shared between gyms and pokestops for safe cross-entity copying. +// This allows copying data without holding locks on both entities simultaneously. +type SharedFortFields struct { + Name null.String + Url null.String + Description null.String + PartnerId null.String + ArScanEligible null.Int64 + PowerUpLevel null.Int64 + PowerUpPoints null.Int64 + PowerUpEndTimestamp null.Int64 +} + +// GetSharedFields returns a copy of shared fields from a Gym. +// Safe to call while holding the gym lock. +func (gym *Gym) GetSharedFields() SharedFortFields { + return SharedFortFields{ + Name: gym.Name, + Url: gym.Url, + Description: gym.Description, + PartnerId: gym.PartnerId, + ArScanEligible: gym.ArScanEligible, + PowerUpLevel: gym.PowerUpLevel, + PowerUpPoints: gym.PowerUpPoints, + PowerUpEndTimestamp: gym.PowerUpEndTimestamp, + } +} + +// GetSharedFields returns a copy of shared fields from a Pokestop. +// Safe to call while holding the pokestop lock. +func (stop *Pokestop) GetSharedFields() SharedFortFields { + return SharedFortFields{ + Name: stop.Name, + Url: stop.Url, + Description: stop.Description, + PartnerId: stop.PartnerId, + ArScanEligible: stop.ArScanEligible, + PowerUpLevel: stop.PowerUpLevel, + PowerUpPoints: stop.PowerUpPoints, + PowerUpEndTimestamp: stop.PowerUpEndTimestamp, + } +} + +// ApplySharedFields applies shared fields to a Gym if not already set. +// Safe to call while holding only the gym lock. +func (gym *Gym) ApplySharedFields(fields SharedFortFields) { + if fields.Name.Valid && !gym.Name.Valid { + gym.SetName(fields.Name) } - if pokestop.Url.Valid && !gym.Url.Valid { - gym.SetUrl(pokestop.Url) + if fields.Url.Valid && !gym.Url.Valid { + gym.SetUrl(fields.Url) } - if pokestop.Description.Valid && !gym.Description.Valid { - gym.SetDescription(pokestop.Description) + if fields.Description.Valid && !gym.Description.Valid { + gym.SetDescription(fields.Description) } - if pokestop.PartnerId.Valid && !gym.PartnerId.Valid { - gym.SetPartnerId(pokestop.PartnerId) + if fields.PartnerId.Valid && !gym.PartnerId.Valid { + gym.SetPartnerId(fields.PartnerId) } - if pokestop.ArScanEligible.Valid && !gym.ArScanEligible.Valid { - gym.SetArScanEligible(pokestop.ArScanEligible) + if fields.ArScanEligible.Valid && !gym.ArScanEligible.Valid { + gym.SetArScanEligible(fields.ArScanEligible) } - if pokestop.PowerUpLevel.Valid && !gym.PowerUpLevel.Valid { - gym.SetPowerUpLevel(pokestop.PowerUpLevel) + if fields.PowerUpLevel.Valid && !gym.PowerUpLevel.Valid { + gym.SetPowerUpLevel(fields.PowerUpLevel) } - if pokestop.PowerUpPoints.Valid && !gym.PowerUpPoints.Valid { - gym.SetPowerUpPoints(pokestop.PowerUpPoints) + if fields.PowerUpPoints.Valid && !gym.PowerUpPoints.Valid { + gym.SetPowerUpPoints(fields.PowerUpPoints) } - if pokestop.PowerUpEndTimestamp.Valid && !gym.PowerUpEndTimestamp.Valid { - gym.SetPowerUpEndTimestamp(pokestop.PowerUpEndTimestamp) + if fields.PowerUpEndTimestamp.Valid && !gym.PowerUpEndTimestamp.Valid { + gym.SetPowerUpEndTimestamp(fields.PowerUpEndTimestamp) } } -// copySharedFieldsFrom copies shared fields from a gym to a pokestop during conversion -func (stop *Pokestop) copySharedFieldsFrom(gym *Gym) { - if gym.Name.Valid && !stop.Name.Valid { - stop.SetName(gym.Name) +// ApplySharedFields applies shared fields to a Pokestop if not already set. +// Safe to call while holding only the pokestop lock. +func (stop *Pokestop) ApplySharedFields(fields SharedFortFields) { + if fields.Name.Valid && !stop.Name.Valid { + stop.SetName(fields.Name) } - if gym.Url.Valid && !stop.Url.Valid { - stop.SetUrl(gym.Url) + if fields.Url.Valid && !stop.Url.Valid { + stop.SetUrl(fields.Url) } - if gym.Description.Valid && !stop.Description.Valid { - stop.SetDescription(gym.Description) + if fields.Description.Valid && !stop.Description.Valid { + stop.SetDescription(fields.Description) } - if gym.PartnerId.Valid && !stop.PartnerId.Valid { - stop.SetPartnerId(gym.PartnerId) + if fields.PartnerId.Valid && !stop.PartnerId.Valid { + stop.SetPartnerId(fields.PartnerId) } - if gym.ArScanEligible.Valid && !stop.ArScanEligible.Valid { - stop.SetArScanEligible(gym.ArScanEligible) + if fields.ArScanEligible.Valid && !stop.ArScanEligible.Valid { + stop.SetArScanEligible(fields.ArScanEligible) } - if gym.PowerUpLevel.Valid && !stop.PowerUpLevel.Valid { - stop.SetPowerUpLevel(gym.PowerUpLevel) + if fields.PowerUpLevel.Valid && !stop.PowerUpLevel.Valid { + stop.SetPowerUpLevel(fields.PowerUpLevel) } - if gym.PowerUpPoints.Valid && !stop.PowerUpPoints.Valid { - stop.SetPowerUpPoints(gym.PowerUpPoints) + if fields.PowerUpPoints.Valid && !stop.PowerUpPoints.Valid { + stop.SetPowerUpPoints(fields.PowerUpPoints) } - if gym.PowerUpEndTimestamp.Valid && !stop.PowerUpEndTimestamp.Valid { - stop.SetPowerUpEndTimestamp(gym.PowerUpEndTimestamp) + if fields.PowerUpEndTimestamp.Valid && !stop.PowerUpEndTimestamp.Valid { + stop.SetPowerUpEndTimestamp(fields.PowerUpEndTimestamp) } } diff --git a/decoder/gmo_decode.go b/decoder/gmo_decode.go index 44b633e1..2bcdf1f9 100644 --- a/decoder/gmo_decode.go +++ b/decoder/gmo_decode.go @@ -28,19 +28,32 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa } pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) + isNewRecord := pokestop.IsNewRecord() - // If this is a new pokestop, check if it was converted from a gym and copy shared fields - if pokestop.IsNewRecord() { + savePokestopRecord(ctx, db, pokestop) + unlock() + + // If this was a new pokestop, check if it was converted from a gym and copy shared fields. + // To avoid deadlock, we do this after releasing the pokestop lock. + if isNewRecord && DoesGymExist(ctx, db, fortId) { + // Get shared fields from gym (with gym lock only) gym, gymUnlock, _ := GetGymRecordReadOnly(ctx, db, fortId) if gym != nil { - pokestop.copySharedFieldsFrom(gym) + sharedFields := gym.GetSharedFields() gymUnlock() + + // Re-acquire pokestop lock to apply shared fields + pokestop, unlock, err = getPokestopRecordForUpdate(ctx, db, fortId) + if err != nil { + log.Errorf("getPokestopRecordForUpdate (shared fields): %s", err) + } else if pokestop != nil { + pokestop.ApplySharedFields(sharedFields) + savePokestopRecord(ctx, db, pokestop) + unlock() + } } } - savePokestopRecord(ctx, db, pokestop) - unlock() - incidents := fort.Data.PokestopDisplays if incidents == nil && fort.Data.PokestopDisplay != nil { incidents = []*pogo.PokestopIncidentDisplayProto{fort.Data.PokestopDisplay} @@ -68,18 +81,31 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa } gym.updateGymFromFort(fort.Data, fort.Cell) + isNewRecord := gym.IsNewRecord() - // If this is a new gym, check if it was converted from a pokestop and copy shared fields - if gym.IsNewRecord() { + saveGymRecord(ctx, db, gym) + gymUnlock() + + // If this was a new gym, check if it was converted from a pokestop and copy shared fields. + // To avoid deadlock, we do this after releasing the gym lock. + if isNewRecord && DoesPokestopExist(ctx, db, fortId) { + // Get shared fields from pokestop (with pokestop lock only) pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, fortId) if pokestop != nil { - gym.copySharedFieldsFrom(pokestop) + sharedFields := pokestop.GetSharedFields() unlock() + + // Re-acquire gym lock to apply shared fields + gym, gymUnlock, err = getGymRecordForUpdate(ctx, db, fortId) + if err != nil { + log.Errorf("getGymRecordForUpdate (shared fields): %s", err) + } else if gym != nil { + gym.ApplySharedFields(sharedFields) + saveGymRecord(ctx, db, gym) + gymUnlock() + } } } - - saveGymRecord(ctx, db, gym) - gymUnlock() } } } diff --git a/decoder/gym_state.go b/decoder/gym_state.go index d966c652..83ea0c60 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -33,6 +33,23 @@ func loadGymFromDatabase(ctx context.Context, db db.DbDetails, fortId string, gy return err } +// DoesGymExist checks if a gym exists in cache or database without acquiring a lock. +// This is useful for checking if a fort was converted from a gym before doing cross-entity updates. +func DoesGymExist(ctx context.Context, db db.DbDetails, fortId string) bool { + // Check cache first (fast path) + if item := gymCache.Get(fortId); item != nil { + return true + } + + // Check database + var exists bool + err := db.GeneralDb.GetContext(ctx, &exists, "SELECT EXISTS(SELECT 1 FROM gym WHERE id = ?)", fortId) + if err != nil { + return false + } + return exists +} + // PeekGymRecord - cache-only lookup, no DB fallback, returns locked. // Caller MUST call returned unlock function if non-nil. func PeekGymRecord(fortId string) (*Gym, func(), error) { diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index f79d232f..9e0be50f 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -51,6 +51,23 @@ func PeekPokestopRecord(fortId string) (*Pokestop, func(), error) { return nil, nil, nil } +// DoesPokestopExist checks if a pokestop exists in cache or database without acquiring a lock. +// This is useful for checking if a fort was converted from a pokestop before doing cross-entity updates. +func DoesPokestopExist(ctx context.Context, db db.DbDetails, fortId string) bool { + // Check cache first (fast path) + if item := pokestopCache.Get(fortId); item != nil { + return true + } + + // Check database + var exists bool + err := db.GeneralDb.GetContext(ctx, &exists, "SELECT EXISTS(SELECT 1 FROM pokestop WHERE id = ?)", fortId) + if err != nil { + return false + } + return exists +} + // getPokestopRecordReadOnly acquires lock but does NOT take snapshot. // Use for read-only checks. Will cause a backing database lookup. // Caller MUST call returned unlock function if non-nil. From eb238e2e9530f8f22d2bb6ef6be420b3c989bcbe Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 6 Feb 2026 17:00:23 +0000 Subject: [PATCH 55/78] Introduce a deadlock detector --- decoder/gym.go | 4 ++-- decoder/pokemon.go | 4 ++-- decoder/pokestop.go | 4 ++-- decoder/spawnpoint.go | 4 ++-- go.mod | 2 ++ go.sum | 4 ++++ 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index 2236ff67..c47790b4 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -2,15 +2,15 @@ package decoder import ( "fmt" - "sync" "github.com/guregu/null/v6" + "github.com/sasha-s/go-deadlock" ) // Gym struct. // REMINDER! Keep hasChangesGym updated after making changes type Gym struct { - mu sync.Mutex `db:"-"` // Object-level mutex + mu deadlock.Mutex `db:"-"` // Object-level mutex Id string `db:"id"` Lat float64 `db:"lat"` diff --git a/decoder/pokemon.go b/decoder/pokemon.go index ebb1ad7b..01de200e 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -2,11 +2,11 @@ package decoder import ( "fmt" - "sync" "golbat/grpc" "github.com/guregu/null/v6" + "github.com/sasha-s/go-deadlock" ) // Pokemon struct. @@ -18,7 +18,7 @@ import ( // // FirstSeenTimestamp: This field is used in IsNewRecord. It should only be set in savePokemonRecord. type Pokemon struct { - mu sync.Mutex `db:"-"` // Object-level mutex + mu deadlock.Mutex `db:"-"` // Object-level mutex Id Uint64Str `db:"id"` PokestopId null.String `db:"pokestop_id"` diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 7769ebde..9292784e 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -2,14 +2,14 @@ package decoder import ( "fmt" - "sync" "github.com/guregu/null/v6" + "github.com/sasha-s/go-deadlock" ) // Pokestop struct. type Pokestop struct { - mu sync.Mutex `db:"-"` // Object-level mutex + mu deadlock.Mutex `db:"-"` // Object-level mutex Id string `db:"id"` Lat float64 `db:"lat"` diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 866bdafa..85818e8c 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "strconv" - "sync" "time" "golbat/db" @@ -14,6 +13,7 @@ import ( "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" + "github.com/sasha-s/go-deadlock" log "github.com/sirupsen/logrus" ) @@ -24,7 +24,7 @@ const spawnpointSelectColumns = `id, lat, lon, updated, last_seen, despawn_sec` // Spawnpoint struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Spawnpoint struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + mu deadlock.Mutex `db:"-" json:"-"` // Object-level mutex Id int64 `db:"id"` Lat float64 `db:"lat"` diff --git a/go.mod b/go.mod index c3b5b86d..6ba08f3b 100644 --- a/go.mod +++ b/go.mod @@ -66,12 +66,14 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect github.com/ringsaturn/tzf-rel-lite v0.0.2025-c // indirect + github.com/sasha-s/go-deadlock v0.3.6 // indirect github.com/tidwall/geoindex v1.7.0 // indirect github.com/tidwall/geojson v1.4.6 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 282db98a..dd75feb2 100644 --- a/go.sum +++ b/go.sum @@ -174,6 +174,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe h1:vHpqOnPlnkba8iSxU4j/CvDSS9J4+F4473esQsYLGoE= +github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -205,6 +207,8 @@ github.com/ringsaturn/tzf-rel-lite v0.0.2025-c h1:CUs4l73ApN87MhlAhp1UtcRe3E5UFM github.com/ringsaturn/tzf-rel-lite v0.0.2025-c/go.mod h1:SyVF6OU+Le0vKajtTA7PvYabdYCJsDlmplHuXeCZDrw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sasha-s/go-deadlock v0.3.6 h1:TR7sfOnZ7x00tWPfD397Peodt57KzMDo+9Ae9rMiUmw= +github.com/sasha-s/go-deadlock v0.3.6/go.mod h1:CUqNyyvMxTyjFqDT7MRg9mb4Dv/btmGTqSR+rky/UXo= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 339c0bf86ed707762cb5ce928a6672a5ceff2d29 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 6 Feb 2026 17:33:14 +0000 Subject: [PATCH 56/78] Squash also in batch writer --- decoder/writebehind/batch.go | 40 +++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/decoder/writebehind/batch.go b/decoder/writebehind/batch.go index 4c841001..bea7e0a3 100644 --- a/decoder/writebehind/batch.go +++ b/decoder/writebehind/batch.go @@ -16,7 +16,7 @@ import ( // BatchWriter handles batched writes for a specific table type type BatchWriter struct { mu sync.Mutex - entries []*QueueEntry + pending map[string]*QueueEntry // key -> entry for deduplication timer *time.Timer batchSize int timeout time.Duration @@ -41,7 +41,7 @@ type BatchWriterConfig struct { // NewBatchWriter creates a new batch writer for a table type func NewBatchWriter(cfg BatchWriterConfig) *BatchWriter { return &BatchWriter{ - entries: make([]*QueueEntry, 0, cfg.BatchSize), + pending: make(map[string]*QueueEntry), batchSize: cfg.BatchSize, timeout: cfg.Timeout, flushFunc: cfg.FlushFunc, @@ -52,21 +52,34 @@ func NewBatchWriter(cfg BatchWriterConfig) *BatchWriter { } } -// Add adds an entry to the batch, flushing if batch is full +// Add adds an entry to the batch, flushing if batch is full. +// Deduplicates by key - if the same key is added twice, the newer entry replaces the older one. func (bw *BatchWriter) Add(entry *QueueEntry) { bw.mu.Lock() defer bw.mu.Unlock() - bw.entries = append(bw.entries, entry) + if existing, ok := bw.pending[entry.Key]; ok { + // Deduplicate: replace with newer entry, preserve IsNewRecord if either is true + entry.IsNewRecord = entry.IsNewRecord || existing.IsNewRecord + // Keep the earlier QueuedAt for latency tracking + if existing.QueuedAt.Before(entry.QueuedAt) { + entry.QueuedAt = existing.QueuedAt + } + if existing.ReadyAt.Before(entry.ReadyAt) { + entry.ReadyAt = existing.ReadyAt + } + bw.queue.stats.IncWriteBehindSquashed(bw.tableType) + } + bw.pending[entry.Key] = entry - if len(bw.entries) >= bw.batchSize { + if len(bw.pending) >= bw.batchSize { bw.flushLocked() } else if bw.timer == nil { // Start timeout for partial batch bw.timer = time.AfterFunc(bw.timeout, func() { bw.mu.Lock() defer bw.mu.Unlock() - if len(bw.entries) > 0 { + if len(bw.pending) > 0 { bw.flushLocked() } }) @@ -80,13 +93,16 @@ func (bw *BatchWriter) flushLocked() { bw.timer = nil } - if len(bw.entries) == 0 { + if len(bw.pending) == 0 { return } - // Take ownership of entries slice - entries := bw.entries - bw.entries = make([]*QueueEntry, 0, bw.batchSize) + // Convert map to slice and sort by key for consistent lock ordering + entries := make([]*QueueEntry, 0, len(bw.pending)) + for _, entry := range bw.pending { + entries = append(entries, entry) + } + bw.pending = make(map[string]*QueueEntry) // Sort entries by key to ensure consistent lock ordering and prevent deadlocks sort.Slice(entries, func(i, j int) bool { @@ -141,7 +157,7 @@ func (bw *BatchWriter) flushLocked() { func (bw *BatchWriter) Flush() { bw.mu.Lock() defer bw.mu.Unlock() - if len(bw.entries) > 0 { + if len(bw.pending) > 0 { bw.flushLocked() } } @@ -150,7 +166,7 @@ func (bw *BatchWriter) Flush() { func (bw *BatchWriter) Size() int { bw.mu.Lock() defer bw.mu.Unlock() - return len(bw.entries) + return len(bw.pending) } // ExecuteBatchUpsert builds and executes a batch upsert using sqlx.Named From 4defe14092ba38e556be81a852d217b0ca61ee32 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 16:23:07 +0000 Subject: [PATCH 57/78] Refactor to use data copies in the queue to avoid deadlocking --- decoder/gym.go | 63 +-- decoder/gym_state.go | 8 +- decoder/gym_writeable.go | 27 -- decoder/incident.go | 16 +- decoder/incident_state.go | 8 +- decoder/incident_writeable.go | 27 -- decoder/main.go | 67 +-- decoder/pokemon.go | 30 +- decoder/pokemon_decode.go | 2 +- decoder/pokemon_state.go | 8 +- decoder/pokemon_writeable.go | 29 -- decoder/pokestop.go | 14 +- decoder/pokestop_state.go | 8 +- decoder/pokestop_writeable.go | 27 -- decoder/routes.go | 16 +- decoder/routes_state.go | 8 +- decoder/routes_writeable.go | 27 -- decoder/s2cell.go | 70 +--- decoder/spawnpoint.go | 24 +- decoder/spawnpoint_writeable.go | 29 -- decoder/station.go | 16 +- decoder/station_state.go | 8 +- decoder/station_writeable.go | 27 -- decoder/tappable.go | 16 +- decoder/tappable_state.go | 8 +- decoder/tappable_writeable.go | 29 -- decoder/writebehind/batch.go | 200 --------- decoder/writebehind/limiter.go | 42 ++ decoder/writebehind/manager.go | 155 +++++++ decoder/writebehind/processor.go | 107 ----- decoder/writebehind/queue.go | 304 -------------- decoder/writebehind/queue_test.go | 175 +++++--- decoder/writebehind/s2cell_accumulator.go | 192 --------- decoder/writebehind/typed_queue.go | 400 ++++++++++++++++++ decoder/writebehind/types.go | 28 +- decoder/writebehind_batch.go | 472 +++++++++++++++++----- main.go | 2 - webhooks/webhooks_test.go | 2 +- 38 files changed, 1228 insertions(+), 1463 deletions(-) delete mode 100644 decoder/gym_writeable.go delete mode 100644 decoder/incident_writeable.go delete mode 100644 decoder/pokemon_writeable.go delete mode 100644 decoder/pokestop_writeable.go delete mode 100644 decoder/routes_writeable.go delete mode 100644 decoder/spawnpoint_writeable.go delete mode 100644 decoder/station_writeable.go delete mode 100644 decoder/tappable_writeable.go delete mode 100644 decoder/writebehind/batch.go create mode 100644 decoder/writebehind/limiter.go create mode 100644 decoder/writebehind/manager.go delete mode 100644 decoder/writebehind/processor.go delete mode 100644 decoder/writebehind/queue.go delete mode 100644 decoder/writebehind/s2cell_accumulator.go create mode 100644 decoder/writebehind/typed_queue.go diff --git a/decoder/gym.go b/decoder/gym.go index c47790b4..32aaa49b 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -7,11 +7,9 @@ import ( "github.com/sasha-s/go-deadlock" ) -// Gym struct. -// REMINDER! Keep hasChangesGym updated after making changes -type Gym struct { - mu deadlock.Mutex `db:"-"` // Object-level mutex - +// GymData contains all database-persisted fields for a Gym. +// This struct is copyable and used for write-behind queue snapshots. +type GymData struct { Id string `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` @@ -53,6 +51,14 @@ type Gym struct { Description null.String `db:"description"` Defenders null.String `db:"defenders"` Rsvps null.String `db:"rsvps"` +} + +// Gym struct. +// REMINDER! Keep hasChangesGym updated after making changes +type Gym struct { + mu deadlock.Mutex `db:"-"` // Object-level mutex + + GymData // Embedded data fields (all db columns) // Memory-only fields (not persisted to DB) RaidSeed null.String `db:"-"` // Raid seed (memory only, sent in webhook) @@ -81,53 +87,6 @@ type GymOldValues struct { InBattle null.Int } -//`id` varchar(35) NOT NULL, -//`lat` double(18,14) NOT NULL, -//`lon` double(18,14) NOT NULL, -//`name` varchar(128) DEFAULT NULL, -//`url` varchar(200) DEFAULT NULL, -//`last_modified_timestamp` int unsigned DEFAULT NULL, -//`raid_end_timestamp` int unsigned DEFAULT NULL, -//`raid_spawn_timestamp` int unsigned DEFAULT NULL, -//`raid_battle_timestamp` int unsigned DEFAULT NULL, -//`updated` int unsigned NOT NULL, -//`raid_pokemon_id` smallint unsigned DEFAULT NULL, -//`guarding_pokemon_id` smallint unsigned DEFAULT NULL, -//`available_slots` smallint unsigned DEFAULT NULL, -//`availble_slots` smallint unsigned GENERATED ALWAYS AS (`available_slots`) VIRTUAL, -//`team_id` tinyint unsigned DEFAULT NULL, -//`raid_level` tinyint unsigned DEFAULT NULL, -//`enabled` tinyint unsigned DEFAULT NULL, -//`ex_raid_eligible` tinyint unsigned DEFAULT NULL, -//`in_battle` tinyint unsigned DEFAULT NULL, -//`raid_pokemon_move_1` smallint unsigned DEFAULT NULL, -//`raid_pokemon_move_2` smallint unsigned DEFAULT NULL, -//`raid_pokemon_form` smallint unsigned DEFAULT NULL, -//`raid_pokemon_cp` int unsigned DEFAULT NULL, -//`raid_is_exclusive` tinyint unsigned DEFAULT NULL, -//`cell_id` bigint unsigned DEFAULT NULL, -//`deleted` tinyint unsigned NOT NULL DEFAULT '0', -//`total_cp` int unsigned DEFAULT NULL, -//`first_seen_timestamp` int unsigned NOT NULL, -//`raid_pokemon_gender` tinyint unsigned DEFAULT NULL, -//`sponsor_id` smallint unsigned DEFAULT NULL, -//`partner_id` varchar(35) DEFAULT NULL, -//`raid_pokemon_costume` smallint unsigned DEFAULT NULL, -//`raid_pokemon_evolution` tinyint unsigned DEFAULT NULL, -//`ar_scan_eligible` tinyint unsigned DEFAULT NULL, -//`power_up_level` smallint unsigned DEFAULT NULL, -//`power_up_points` int unsigned DEFAULT NULL, -//`power_up_end_timestamp` int unsigned DEFAULT NULL, - -// -//SELECT CONCAT("'", GROUP_CONCAT(column_name ORDER BY ordinal_position SEPARATOR "', '"), "'") AS columns -//FROM information_schema.columns -//WHERE table_schema = 'db_name' AND table_name = 'tbl_name' -// -//SELECT CONCAT("'", GROUP_CONCAT(column_name ORDER BY ordinal_position SEPARATOR "', '"), " = ", "'") AS columns -//FROM information_schema.columns -//WHERE table_schema = 'db_name' AND table_name = 'tbl_name' - // IsDirty returns true if any field has been modified func (gym *Gym) IsDirty() bool { return gym.dirty diff --git a/decoder/gym_state.go b/decoder/gym_state.go index 83ea0c60..f9170dbd 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -114,7 +114,7 @@ func getGymRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { // Create new Gym atomically - function only called if key doesn't exist gymItem, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { - return &Gym{Id: fortId, newRecord: true} + return &Gym{GymData: GymData{Id: fortId}, newRecord: true} }) gym := gymItem.Value() @@ -382,9 +382,9 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { } if gym.IsDirty() { - // Queue the write through the write-behind system - if writeBehindQueue != nil { - writeBehindQueue.Enqueue(gym, isNewRecord, 0) + // Queue the write through the typed write-behind queue + if gymQueue != nil { + gymQueue.Enqueue(gym.GymData, isNewRecord, 0) } else { // Fallback to direct write if queue not initialized _ = gymWriteDB(db, gym, isNewRecord) diff --git a/decoder/gym_writeable.go b/decoder/gym_writeable.go deleted file mode 100644 index 91f16e1b..00000000 --- a/decoder/gym_writeable.go +++ /dev/null @@ -1,27 +0,0 @@ -package decoder - -import ( - "golbat/db" - "golbat/decoder/writebehind" -) - -// Ensure Gym implements Writeable -var _ writebehind.Writeable = (*Gym)(nil) - -// WriteKey returns a unique key for this Gym (for squashing) -func (g *Gym) WriteKey() string { - return "gym:" + g.Id -} - -// WriteType returns the entity type name (for metrics) -func (g *Gym) WriteType() string { - return "gym" -} - -// WriteToDB performs the actual database write for this Gym -// This delegates to the shared direct write function -func (g *Gym) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { - g.Lock() - defer g.Unlock() - return gymWriteDB(dbDetails, g, isNewRecord) -} diff --git a/decoder/incident.go b/decoder/incident.go index c30e135d..a0cbd035 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -7,11 +7,9 @@ import ( "github.com/guregu/null/v6" ) -// Incident struct. -// REMINDER! Dirty flag pattern - use setter methods to modify fields -type Incident struct { - mu sync.Mutex `db:"-"` // Object-level mutex - +// IncidentData contains all database-persisted fields for Incident. +// This struct is embedded in Incident and can be safely copied for write-behind queueing. +type IncidentData struct { Id string `db:"id"` PokestopId string `db:"pokestop_id"` StartTime int64 `db:"start"` @@ -27,6 +25,14 @@ type Incident struct { Slot2Form null.Int `db:"slot_2_form"` Slot3PokemonId null.Int `db:"slot_3_pokemon_id"` Slot3Form null.Int `db:"slot_3_form"` +} + +// Incident struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields +type Incident struct { + mu sync.Mutex `db:"-"` // Object-level mutex + + IncidentData // Embedded data fields - can be copied for write-behind queue dirty bool `db:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-"` // Not persisted - tracks if this is a new record diff --git a/decoder/incident_state.go b/decoder/incident_state.go index c56b43a7..9d6766cd 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -83,7 +83,7 @@ func getIncidentRecordForUpdate(ctx context.Context, db db.DbDetails, incidentId func getOrCreateIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string, pokestopId string) (*Incident, func(), error) { // Create new Incident atomically - function only called if key doesn't exist incidentItem, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { - return &Incident{Id: incidentId, PokestopId: pokestopId, newRecord: true} + return &Incident{IncidentData: IncidentData{Id: incidentId, PokestopId: pokestopId}, newRecord: true} }) incident := incidentItem.Value() @@ -128,9 +128,9 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident } } - // Queue the write through the write-behind system - if writeBehindQueue != nil { - writeBehindQueue.Enqueue(incident, isNewRecord, 0) + // Queue the write through the typed write-behind queue + if incidentQueue != nil { + incidentQueue.Enqueue(incident.IncidentData, isNewRecord, 0) } else { // Fallback to direct write if queue not initialized _ = incidentWriteDB(db, incident, isNewRecord) diff --git a/decoder/incident_writeable.go b/decoder/incident_writeable.go deleted file mode 100644 index 9924d7f6..00000000 --- a/decoder/incident_writeable.go +++ /dev/null @@ -1,27 +0,0 @@ -package decoder - -import ( - "golbat/db" - "golbat/decoder/writebehind" -) - -// Ensure Incident implements Writeable -var _ writebehind.Writeable = (*Incident)(nil) - -// WriteKey returns a unique key for this Incident (for squashing) -func (i *Incident) WriteKey() string { - return "incident:" + i.Id -} - -// WriteType returns the entity type name (for metrics) -func (i *Incident) WriteType() string { - return "incident" -} - -// WriteToDB performs the actual database write for this Incident -// This delegates to the shared direct write function -func (i *Incident) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { - i.Lock() - defer i.Unlock() - return incidentWriteDB(dbDetails, i, isNewRecord) -} diff --git a/decoder/main.go b/decoder/main.go index c77c6bc9..d163ddaf 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -14,7 +14,6 @@ import ( "golbat/config" "golbat/db" - "golbat/decoder/writebehind" "golbat/geo" "golbat/pogo" "golbat/stats_collector" @@ -73,12 +72,6 @@ var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto var ProactiveIVSwitchSem chan bool -// writeBehindQueue is the global write-behind queue for database writes -var writeBehindQueue *writebehind.Queue - -// s2CellAccumulator is the global S2Cell batch accumulator -var s2CellAccumulator *writebehind.S2CellAccumulator - var ohbem *gohbem.Ohbem func init() { @@ -292,68 +285,16 @@ func SetStatsCollector(collector stats_collector.StatsCollector) { statsCollector = collector } -// InitWriteBehindQueue initializes the write-behind queue +// InitWriteBehindQueue initializes the typed write-behind queues // Should be called after SetStatsCollector func InitWriteBehindQueue(ctx context.Context, dbDetails db.DbDetails) { - cfg := writebehind.QueueConfig{ - StartupDelaySeconds: config.Config.Tuning.WriteBehindStartupDelay, - WorkerCount: config.Config.Tuning.WriteBehindWorkerCount, - BatchSize: config.Config.Tuning.WriteBehindBatchSize, - BatchTimeout: time.Duration(config.Config.Tuning.WriteBehindBatchTimeoutMs) * time.Millisecond, - } - - writeBehindQueue = writebehind.NewQueue(cfg, dbDetails, statsCollector) - - // Register batch writers for each entity type - RegisterBatchWriters(writeBehindQueue) - - log.Infof("Write-behind queue initialized: startup_delay=%ds, workers=%d, batch_size=%d, batch_timeout=%dms", - cfg.StartupDelaySeconds, cfg.WorkerCount, cfg.BatchSize, cfg.BatchTimeout.Milliseconds()) - - // Warn if worker count exceeds half of database pool size - maxPool := config.Config.Database.MaxPool - if cfg.WorkerCount > maxPool/2 { - log.Warnf("Write-behind worker count (%d) exceeds half of database pool size (%d). "+ - "Consider increasing database.max_pool or reducing tuning.write_behind_worker_count", - cfg.WorkerCount, maxPool) - } - - // Start the processing loop in a goroutine - go writeBehindQueue.ProcessLoop(ctx) -} - -// GetWriteBehindQueue returns the global write-behind queue -func GetWriteBehindQueue() *writebehind.Queue { - return writeBehindQueue + // Use the new typed queue system + InitTypedQueues(ctx, dbDetails, statsCollector) } // FlushWriteBehindQueue flushes all pending writes (for shutdown) func FlushWriteBehindQueue() { - if writeBehindQueue != nil { - writeBehindQueue.Flush() - } -} - -// InitS2CellAccumulator initializes the S2Cell batch accumulator -// Should be called after SetStatsCollector -func InitS2CellAccumulator(ctx context.Context, dbDetails db.DbDetails) { - cfg := writebehind.QueueConfig{ - StartupDelaySeconds: config.Config.Tuning.WriteBehindStartupDelay, - } - - s2CellAccumulator = writebehind.NewS2CellAccumulator(cfg, dbDetails, statsCollector, s2CellBatchWrite) - - log.Infof("S2Cell accumulator initialized: startup_delay=%ds", cfg.StartupDelaySeconds) - - // Start the processing loop - s2CellAccumulator.Start(ctx) -} - -// FlushS2CellAccumulator flushes all pending S2Cell writes (for shutdown) -func FlushS2CellAccumulator() { - if s2CellAccumulator != nil { - s2CellAccumulator.Flush() - } + FlushTypedQueues() } // GetUpdateThreshold returns the number of seconds that should be used as a diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 01de200e..a2b99ceb 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -9,17 +9,9 @@ import ( "github.com/sasha-s/go-deadlock" ) -// Pokemon struct. -// REMINDER! Keep hasChangesPokemon updated after making changes -// -// AtkIv/DefIv/StaIv: Should not be set directly. Use calculateIv -// -// GolbatInternal: internal data not exposed to frontend/users -// -// FirstSeenTimestamp: This field is used in IsNewRecord. It should only be set in savePokemonRecord. -type Pokemon struct { - mu deadlock.Mutex `db:"-"` // Object-level mutex - +// PokemonData contains all database-persisted fields for Pokemon. +// This struct is embedded in Pokemon and can be safely copied for write-behind queueing. +type PokemonData struct { Id Uint64Str `db:"id"` PokestopId null.String `db:"pokestop_id"` SpawnId null.Int `db:"spawn_id"` @@ -59,8 +51,22 @@ type Pokemon struct { Capture3 null.Float `db:"capture_3"` Pvp null.String `db:"pvp"` IsEvent int8 `db:"is_event"` +} + +// Pokemon struct. +// REMINDER! Keep hasChangesPokemon updated after making changes +// +// AtkIv/DefIv/StaIv: Should not be set directly. Use calculateIv +// +// GolbatInternal: internal data not exposed to frontend/users +// +// FirstSeenTimestamp: This field is used in IsNewRecord. It should only be set in savePokemonRecord. +type Pokemon struct { + mu deadlock.Mutex `db:"-"` // Object-level mutex + + PokemonData // Embedded data fields - can be copied for write-behind queue - internal grpc.PokemonInternal + internal grpc.PokemonInternal `db:"-"` // Memory-only internal state dirty bool `db:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-"` diff --git a/decoder/pokemon_decode.go b/decoder/pokemon_decode.go index 5a53ddd1..b98e4df4 100644 --- a/decoder/pokemon_decode.go +++ b/decoder/pokemon_decode.go @@ -257,7 +257,7 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n pokemon.SetLat(lat) pokemon.SetLon(lon) } else { - midpoint := s2.LatLngFromPoint(s2.Point{s2.PointFromLatLng(s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon)). + midpoint := s2.LatLngFromPoint(s2.Point{Vector: s2.PointFromLatLng(s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon)). Add(s2.PointFromLatLng(s2.LatLngFromDegrees(lat, lon)).Vector)}) pokemon.SetLat(midpoint.Lat.Degrees()) pokemon.SetLon(midpoint.Lng.Degrees()) diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index 7de88dd8..1c5ebd81 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -104,7 +104,7 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { // Create new Pokemon atomically - function only called if key doesn't exist pokemonItem, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - return &Pokemon{Id: Uint64Str(encounterId), newRecord: true} + return &Pokemon{PokemonData: PokemonData{Id: Uint64Str(encounterId)}, newRecord: true} }) pokemon := pokemonItem.Value() @@ -260,8 +260,8 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } - // Queue the write through the write-behind system - if writeBehindQueue != nil { + // Queue the write through the typed write-behind queue + if pokemonQueue != nil { // Determine delay based on seen type // Wild/nearby Pokemon wait for potential encounter data, encounter writes immediately delay := time.Duration(0) @@ -270,7 +270,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po seenType == SeenType_Cell || seenType == SeenType_NearbyStop { delay = wildPokemonDelay } - writeBehindQueue.Enqueue(pokemon, isNewRecord, delay) + pokemonQueue.Enqueue(pokemon.PokemonData, isNewRecord, delay) } else { // Fallback to direct write if queue not initialized _ = pokemonWriteDB(db, pokemon, isNewRecord) diff --git a/decoder/pokemon_writeable.go b/decoder/pokemon_writeable.go deleted file mode 100644 index 87f9f8c3..00000000 --- a/decoder/pokemon_writeable.go +++ /dev/null @@ -1,29 +0,0 @@ -package decoder - -import ( - "fmt" - - "golbat/db" - "golbat/decoder/writebehind" -) - -// Ensure Pokemon implements Writeable -var _ writebehind.Writeable = (*Pokemon)(nil) - -// WriteKey returns a unique key for this Pokemon (for squashing) -func (pokemon *Pokemon) WriteKey() string { - return fmt.Sprintf("pokemon:%d", pokemon.Id) -} - -// WriteType returns the entity type name (for metrics) -func (pokemon *Pokemon) WriteType() string { - return "pokemon" -} - -// WriteToDB performs the actual database write for this Pokemon -// This delegates to the shared direct write function -func (pokemon *Pokemon) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { - pokemon.Lock() - defer pokemon.Unlock() - return pokemonWriteDB(dbDetails, pokemon, isNewRecord) -} diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 9292784e..db8567e8 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -7,10 +7,9 @@ import ( "github.com/sasha-s/go-deadlock" ) -// Pokestop struct. -type Pokestop struct { - mu deadlock.Mutex `db:"-"` // Object-level mutex - +// PokestopData contains all database-persisted fields for Pokestop. +// This struct is embedded in Pokestop and can be safely copied for write-behind queueing. +type PokestopData struct { Id string `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` @@ -64,6 +63,13 @@ type Pokestop struct { ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard"` ShowcaseExpiry null.Int `db:"showcase_expiry"` ShowcaseRankings null.String `db:"showcase_rankings"` +} + +// Pokestop struct. +type Pokestop struct { + mu deadlock.Mutex `db:"-"` // Object-level mutex + + PokestopData // Embedded data fields - can be copied for write-behind queue // Memory-only fields (not persisted to DB) QuestSeed null.Int `db:"-"` // Quest seed for AR quest (memory only, sent in webhook) diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index 9e0be50f..9321ac5c 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -122,7 +122,7 @@ func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId str func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { // Create new Pokestop atomically - function only called if key doesn't exist pokestopItem, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { - return &Pokestop{Id: fortId, newRecord: true} + return &Pokestop{PokestopData: PokestopData{Id: fortId}, newRecord: true} }) pokestop := pokestopItem.Value() @@ -321,11 +321,11 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } } - // Queue the write through the write-behind system (no delay for pokestops) + // Queue the write through the typed write-behind queue (no delay for pokestops) // Only queue if dirty (not just internalDirty) if pokestop.IsDirty() { - if writeBehindQueue != nil { - writeBehindQueue.Enqueue(pokestop, isNewRecord, 0) + if pokestopQueue != nil { + pokestopQueue.Enqueue(pokestop.PokestopData, isNewRecord, 0) } else { // Fallback to direct write if queue not initialized _ = pokestopWriteDB(db, pokestop, isNewRecord) diff --git a/decoder/pokestop_writeable.go b/decoder/pokestop_writeable.go deleted file mode 100644 index 34a3fa4f..00000000 --- a/decoder/pokestop_writeable.go +++ /dev/null @@ -1,27 +0,0 @@ -package decoder - -import ( - "golbat/db" - "golbat/decoder/writebehind" -) - -// Ensure Pokestop implements Writeable -var _ writebehind.Writeable = (*Pokestop)(nil) - -// WriteKey returns a unique key for this Pokestop (for squashing) -func (p *Pokestop) WriteKey() string { - return "pokestop:" + p.Id -} - -// WriteType returns the entity type name (for metrics) -func (p *Pokestop) WriteType() string { - return "pokestop" -} - -// WriteToDB performs the actual database write for this Pokestop -// This delegates to the shared direct write function -func (p *Pokestop) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { - p.Lock() - defer p.Unlock() - return pokestopWriteDB(dbDetails, p, isNewRecord) -} diff --git a/decoder/routes.go b/decoder/routes.go index 567a7adb..bedc33e4 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -7,11 +7,9 @@ import ( "github.com/guregu/null/v6" ) -// Route struct. -// REMINDER! Dirty flag pattern - use setter methods to modify fields -type Route struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - +// RouteData contains all database-persisted fields for Route. +// This struct is embedded in Route and can be safely copied for write-behind queueing. +type RouteData struct { Id string `db:"id"` Name string `db:"name"` Shortcode string `db:"shortcode"` @@ -34,6 +32,14 @@ type Route struct { Updated int64 `db:"updated"` Version int64 `db:"version"` Waypoints string `db:"waypoints"` +} + +// Route struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields +type Route struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + + RouteData // Embedded data fields - can be copied for write-behind queue dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record diff --git a/decoder/routes_state.go b/decoder/routes_state.go index eefc5f6b..939df250 100644 --- a/decoder/routes_state.go +++ b/decoder/routes_state.go @@ -78,7 +78,7 @@ func getRouteRecordForUpdate(ctx context.Context, db db.DbDetails, routeId strin func getOrCreateRouteRecord(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { // Create new Route atomically - function only called if key doesn't exist routeItem, _ := routeCache.GetOrSetFunc(routeId, func() *Route { - return &Route{Id: routeId, newRecord: true} + return &Route{RouteData: RouteData{Id: routeId}, newRecord: true} }) route := routeItem.Value() @@ -126,9 +126,9 @@ func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { } } - // Queue the write through the write-behind system - if writeBehindQueue != nil { - writeBehindQueue.Enqueue(route, isNewRecord, 0) + // Queue the write through the typed write-behind queue + if routeQueue != nil { + routeQueue.Enqueue(route.RouteData, isNewRecord, 0) } else { // Fallback to direct write if queue not initialized if err := routeWriteDB(db, route, isNewRecord); err != nil { diff --git a/decoder/routes_writeable.go b/decoder/routes_writeable.go deleted file mode 100644 index 8f80d140..00000000 --- a/decoder/routes_writeable.go +++ /dev/null @@ -1,27 +0,0 @@ -package decoder - -import ( - "golbat/db" - "golbat/decoder/writebehind" -) - -// Ensure Route implements Writeable -var _ writebehind.Writeable = (*Route)(nil) - -// WriteKey returns a unique key for this Route (for squashing) -func (r *Route) WriteKey() string { - return "route:" + r.Id -} - -// WriteType returns the entity type name (for metrics) -func (r *Route) WriteType() string { - return "route" -} - -// WriteToDB performs the actual database write for this Route -// This delegates to the shared direct write function -func (r *Route) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { - r.Lock() - defer r.Unlock() - return routeWriteDB(dbDetails, r, isNewRecord) -} diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 12fe8ef2..eebbc192 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -2,12 +2,9 @@ package decoder import ( "context" - "strconv" - "strings" "time" "golbat/db" - "golbat/decoder/writebehind" "github.com/golang/geo/s2" "github.com/guregu/null/v6" @@ -34,7 +31,6 @@ type S2Cell struct { func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { now := time.Now().Unix() - var cellsToWrite []*writebehind.S2CellData // prepare list of cells to update for _, cellId := range cellIds { @@ -58,63 +54,19 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { } s2Cell.Updated = now - cellsToWrite = append(cellsToWrite, &writebehind.S2CellData{ - Id: s2Cell.Id, - Latitude: s2Cell.Latitude, - Longitude: s2Cell.Longitude, - Level: s2Cell.Level.ValueOrZero(), - Updated: s2Cell.Updated, - }) - } - - if len(cellsToWrite) == 0 { - return - } - - if dbDebugEnabled { - var updatedCells []string - for _, cell := range cellsToWrite { - updatedCells = append(updatedCells, strconv.FormatUint(cell.Id, 10)) + if dbDebugEnabled { + log.Debugf("[DB_UPDATE] S2Cell Updated cell: %d", s2Cell.Id) } - log.Debugf("[DB_UPDATE] S2Cell Updated cells: %s", strings.Join(updatedCells, ",")) - } - - // Queue through the accumulator if available - if s2CellAccumulator != nil { - s2CellAccumulator.Add(cellsToWrite) - } else { - // Fallback to direct write if accumulator not initialized - _ = s2CellBatchWrite(db, cellsToWrite) - } -} - -// s2CellBatchWrite performs the actual batch database write for S2Cells -// This is called by both direct writes and the accumulator -func s2CellBatchWrite(db db.DbDetails, cells []*writebehind.S2CellData) error { - ctx := context.Background() - // Convert to slice of S2Cell for the query - s2Cells := make([]*S2Cell, len(cells)) - for i, cell := range cells { - s2Cells[i] = &S2Cell{ - Id: cell.Id, - Latitude: cell.Latitude, - Longitude: cell.Longitude, - Level: null.IntFrom(cell.Level), - Updated: cell.Updated, + // Queue through the typed queue + if s2cellQueue != nil { + s2cellQueue.Enqueue(S2CellData{ + Id: s2Cell.Id, + Latitude: s2Cell.Latitude, + Longitude: s2Cell.Longitude, + Level: s2Cell.Level.ValueOrZero(), + Updated: s2Cell.Updated, + }, false, 0) } } - - _, err := db.GeneralDb.NamedExecContext(ctx, ` - INSERT INTO s2cell (id, center_lat, center_lon, level, updated) - VALUES (:id, :center_lat, :center_lon, :level, :updated) - ON DUPLICATE KEY UPDATE updated=VALUES(updated) - `, s2Cells) - - statsCollector.IncDbQuery("insert s2cell", err) - if err != nil { - log.Errorf("s2CellBatchWrite: %s", err) - return err - } - return nil } diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 85818e8c..4bf46118 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -21,17 +21,23 @@ import ( // Used by both single-row and bulk load queries to keep them in sync. const spawnpointSelectColumns = `id, lat, lon, updated, last_seen, despawn_sec` -// Spawnpoint struct. -// REMINDER! Dirty flag pattern - use setter methods to modify fields -type Spawnpoint struct { - mu deadlock.Mutex `db:"-" json:"-"` // Object-level mutex - +// SpawnpointData contains all database-persisted fields for Spawnpoint. +// This struct is embedded in Spawnpoint and can be safely copied for write-behind queueing. +type SpawnpointData struct { Id int64 `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` Updated int64 `db:"updated"` LastSeen int64 `db:"last_seen"` DespawnSec null.Int `db:"despawn_sec"` +} + +// Spawnpoint struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields +type Spawnpoint struct { + mu deadlock.Mutex `db:"-" json:"-"` // Object-level mutex + + SpawnpointData // Embedded data fields - can be copied for write-behind queue dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record @@ -211,7 +217,7 @@ func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int6 func getOrCreateSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, func(), error) { // Create new Spawnpoint atomically - function only called if key doesn't exist spawnpointItem, _ := spawnpointCache.GetOrSetFunc(spawnpointId, func() *Spawnpoint { - return &Spawnpoint{Id: spawnpointId, newRecord: true} + return &Spawnpoint{SpawnpointData: SpawnpointData{Id: spawnpointId}, newRecord: true} }) spawnpoint := spawnpointItem.Value() @@ -303,9 +309,9 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi } } - // Queue the write through the write-behind system (no delay for spawnpoints) - if writeBehindQueue != nil { - writeBehindQueue.Enqueue(spawnpoint, isNewRecord, 0) + // Queue the write through the typed write-behind queue (no delay for spawnpoints) + if spawnpointQueue != nil { + spawnpointQueue.Enqueue(spawnpoint.SpawnpointData, isNewRecord, 0) } else { // Fallback to direct write if queue not initialized _ = spawnpointWriteDB(db, spawnpoint) diff --git a/decoder/spawnpoint_writeable.go b/decoder/spawnpoint_writeable.go deleted file mode 100644 index e85eb44d..00000000 --- a/decoder/spawnpoint_writeable.go +++ /dev/null @@ -1,29 +0,0 @@ -package decoder - -import ( - "fmt" - - "golbat/db" - "golbat/decoder/writebehind" -) - -// Ensure Spawnpoint implements Writeable -var _ writebehind.Writeable = (*Spawnpoint)(nil) - -// WriteKey returns a unique key for this Spawnpoint (for squashing) -func (s *Spawnpoint) WriteKey() string { - return fmt.Sprintf("spawnpoint:%d", s.Id) -} - -// WriteType returns the entity type name (for metrics) -func (s *Spawnpoint) WriteType() string { - return "spawnpoint" -} - -// WriteToDB performs the actual database write for this Spawnpoint -// This delegates to the shared direct write function -func (s *Spawnpoint) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { - s.Lock() - defer s.Unlock() - return spawnpointWriteDB(dbDetails, s) -} diff --git a/decoder/station.go b/decoder/station.go index 8a7b7bcd..959c0b74 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -7,11 +7,9 @@ import ( "github.com/guregu/null/v6" ) -// Station struct. -// REMINDER! Dirty flag pattern - use setter methods to modify fields -type Station struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - +// StationData contains all database-persisted fields for Station. +// This struct is embedded in Station and can be safely copied for write-behind queueing. +type StationData struct { Id string `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` @@ -41,6 +39,14 @@ type Station struct { TotalStationedPokemon null.Int `db:"total_stationed_pokemon"` TotalStationedGmax null.Int `db:"total_stationed_gmax"` StationedPokemon null.String `db:"stationed_pokemon"` +} + +// Station struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields +type Station struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + + StationData // Embedded data fields - can be copied for write-behind queue dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record diff --git a/decoder/station_state.go b/decoder/station_state.go index e6ea9c86..3db3faf1 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -113,7 +113,7 @@ func getStationRecordForUpdate(ctx context.Context, db db.DbDetails, stationId s func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { // Create new Station atomically - function only called if key doesn't exist stationItem, _ := stationCache.GetOrSetFunc(stationId, func() *Station { - return &Station{Id: stationId, newRecord: true} + return &Station{StationData: StationData{Id: stationId}, newRecord: true} }) station := stationItem.Value() @@ -162,9 +162,9 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { } } - // Queue the write through the write-behind system - if writeBehindQueue != nil { - writeBehindQueue.Enqueue(station, isNewRecord, 0) + // Queue the write through the typed write-behind queue + if stationQueue != nil { + stationQueue.Enqueue(station.StationData, isNewRecord, 0) } else { // Fallback to direct write if queue not initialized _ = stationWriteDB(db, station, isNewRecord) diff --git a/decoder/station_writeable.go b/decoder/station_writeable.go deleted file mode 100644 index ab12094d..00000000 --- a/decoder/station_writeable.go +++ /dev/null @@ -1,27 +0,0 @@ -package decoder - -import ( - "golbat/db" - "golbat/decoder/writebehind" -) - -// Ensure Station implements Writeable -var _ writebehind.Writeable = (*Station)(nil) - -// WriteKey returns a unique key for this Station (for squashing) -func (s *Station) WriteKey() string { - return "station:" + s.Id -} - -// WriteType returns the entity type name (for metrics) -func (s *Station) WriteType() string { - return "station" -} - -// WriteToDB performs the actual database write for this Station -// This delegates to the shared direct write function -func (s *Station) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { - s.Lock() - defer s.Unlock() - return stationWriteDB(dbDetails, s, isNewRecord) -} diff --git a/decoder/tappable.go b/decoder/tappable.go index 20ff851d..9a9bde6b 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -7,11 +7,9 @@ import ( "github.com/guregu/null/v6" ) -// Tappable struct. -// REMINDER! Dirty flag pattern - use setter methods to modify fields -type Tappable struct { - mu sync.Mutex `db:"-"` // Object-level mutex - +// TappableData contains all database-persisted fields for Tappable. +// This struct is embedded in Tappable and can be safely copied for write-behind queueing. +type TappableData struct { Id uint64 `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` @@ -24,6 +22,14 @@ type Tappable struct { ExpireTimestamp null.Int `db:"expire_timestamp"` ExpireTimestampVerified bool `db:"expire_timestamp_verified"` Updated int64 `db:"updated"` +} + +// Tappable struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields +type Tappable struct { + mu sync.Mutex `db:"-"` // Object-level mutex + + TappableData // Embedded data fields - can be copied for write-behind queue dirty bool `db:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-"` // Not persisted - tracks if this is a new record diff --git a/decoder/tappable_state.go b/decoder/tappable_state.go index 43e9dd4f..9cbc2c58 100644 --- a/decoder/tappable_state.go +++ b/decoder/tappable_state.go @@ -70,7 +70,7 @@ func getTappableRecordReadOnly(ctx context.Context, db db.DbDetails, id uint64) func getOrCreateTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { // Create new Tappable atomically - function only called if key doesn't exist tappableItem, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { - return &Tappable{Id: id, newRecord: true} + return &Tappable{TappableData: TappableData{Id: id}, newRecord: true} }) tappable := tappableItem.Value() @@ -115,9 +115,9 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } } - // Queue the write through the write-behind system - if writeBehindQueue != nil { - writeBehindQueue.Enqueue(tappable, isNewRecord, 0) + // Queue the write through the typed write-behind queue + if tappableQueue != nil { + tappableQueue.Enqueue(tappable.TappableData, isNewRecord, 0) } else { // Fallback to direct write if queue not initialized _ = tappableWriteDB(details, tappable, isNewRecord) diff --git a/decoder/tappable_writeable.go b/decoder/tappable_writeable.go deleted file mode 100644 index c70d47d9..00000000 --- a/decoder/tappable_writeable.go +++ /dev/null @@ -1,29 +0,0 @@ -package decoder - -import ( - "fmt" - - "golbat/db" - "golbat/decoder/writebehind" -) - -// Ensure Tappable implements Writeable -var _ writebehind.Writeable = (*Tappable)(nil) - -// WriteKey returns a unique key for this Tappable (for squashing) -func (t *Tappable) WriteKey() string { - return fmt.Sprintf("tappable:%d", t.Id) -} - -// WriteType returns the entity type name (for metrics) -func (t *Tappable) WriteType() string { - return "tappable" -} - -// WriteToDB performs the actual database write for this Tappable -// This delegates to the shared direct write function -func (t *Tappable) WriteToDB(dbDetails db.DbDetails, isNewRecord bool) error { - t.Lock() - defer t.Unlock() - return tappableWriteDB(dbDetails, t, isNewRecord) -} diff --git a/decoder/writebehind/batch.go b/decoder/writebehind/batch.go deleted file mode 100644 index bea7e0a3..00000000 --- a/decoder/writebehind/batch.go +++ /dev/null @@ -1,200 +0,0 @@ -package writebehind - -import ( - "context" - "sort" - "sync" - "time" - - "github.com/jmoiron/sqlx" - log "github.com/sirupsen/logrus" - - "golbat/db" - "golbat/stats_collector" -) - -// BatchWriter handles batched writes for a specific table type -type BatchWriter struct { - mu sync.Mutex - pending map[string]*QueueEntry // key -> entry for deduplication - timer *time.Timer - batchSize int - timeout time.Duration - flushFunc func(ctx context.Context, db db.DbDetails, entries []*QueueEntry) error - db db.DbDetails - stats stats_collector.StatsCollector - tableType string - queue *Queue // Reference to parent queue for metrics -} - -// BatchWriterConfig holds configuration for a batch writer -type BatchWriterConfig struct { - BatchSize int - Timeout time.Duration - TableType string - FlushFunc func(ctx context.Context, db db.DbDetails, entries []*QueueEntry) error - Db db.DbDetails - Stats stats_collector.StatsCollector - Queue *Queue -} - -// NewBatchWriter creates a new batch writer for a table type -func NewBatchWriter(cfg BatchWriterConfig) *BatchWriter { - return &BatchWriter{ - pending: make(map[string]*QueueEntry), - batchSize: cfg.BatchSize, - timeout: cfg.Timeout, - flushFunc: cfg.FlushFunc, - db: cfg.Db, - stats: cfg.Stats, - tableType: cfg.TableType, - queue: cfg.Queue, - } -} - -// Add adds an entry to the batch, flushing if batch is full. -// Deduplicates by key - if the same key is added twice, the newer entry replaces the older one. -func (bw *BatchWriter) Add(entry *QueueEntry) { - bw.mu.Lock() - defer bw.mu.Unlock() - - if existing, ok := bw.pending[entry.Key]; ok { - // Deduplicate: replace with newer entry, preserve IsNewRecord if either is true - entry.IsNewRecord = entry.IsNewRecord || existing.IsNewRecord - // Keep the earlier QueuedAt for latency tracking - if existing.QueuedAt.Before(entry.QueuedAt) { - entry.QueuedAt = existing.QueuedAt - } - if existing.ReadyAt.Before(entry.ReadyAt) { - entry.ReadyAt = existing.ReadyAt - } - bw.queue.stats.IncWriteBehindSquashed(bw.tableType) - } - bw.pending[entry.Key] = entry - - if len(bw.pending) >= bw.batchSize { - bw.flushLocked() - } else if bw.timer == nil { - // Start timeout for partial batch - bw.timer = time.AfterFunc(bw.timeout, func() { - bw.mu.Lock() - defer bw.mu.Unlock() - if len(bw.pending) > 0 { - bw.flushLocked() - } - }) - } -} - -// flushLocked flushes the current batch (must be called with lock held) -func (bw *BatchWriter) flushLocked() { - if bw.timer != nil { - bw.timer.Stop() - bw.timer = nil - } - - if len(bw.pending) == 0 { - return - } - - // Convert map to slice and sort by key for consistent lock ordering - entries := make([]*QueueEntry, 0, len(bw.pending)) - for _, entry := range bw.pending { - entries = append(entries, entry) - } - bw.pending = make(map[string]*QueueEntry) - - // Sort entries by key to ensure consistent lock ordering and prevent deadlocks - sort.Slice(entries, func(i, j int) bool { - return entries[i].Key < entries[j].Key - }) - - // Release lock before doing I/O - bw.mu.Unlock() - - // Execute batch write - start := time.Now() - ctx := context.Background() - err := bw.flushFunc(ctx, bw.db, entries) - batchTime := time.Since(start).Seconds() - entryCount := len(entries) - - if err != nil { - bw.stats.IncWriteBehindErrors(bw.tableType) - log.Errorf("Write-behind batch error for %s (%d entries): %v", bw.tableType, entryCount, err) - } else { - // Increment write count by number of entries in batch - for range entries { - bw.stats.IncWriteBehindWrites(bw.tableType) - } - // Record batch metrics in Prometheus - bw.stats.IncWriteBehindBatches(bw.tableType) - bw.stats.ObserveWriteBehindBatchSize(bw.tableType, float64(entryCount)) - bw.stats.ObserveWriteBehindBatchTime(bw.tableType, batchTime) - - //log.Debugf("Write-behind batch wrote %d %s entries in %.1fms", entryCount, bw.tableType, batchTime*1000) - } - - // Track batch metrics on the parent queue - bw.queue.metricsMu.Lock() - bw.queue.batchCount++ - bw.queue.batchEntryCount += int64(entryCount) - bw.queue.batchWriteTime += batchTime - for _, entry := range entries { - latency := time.Since(entry.ReadyAt).Seconds() - bw.queue.batchLatency += latency - bw.queue.batchLatencyCount++ - // Also record per-entry latency in Prometheus - bw.stats.ObserveWriteBehindLatency(bw.tableType, latency) - } - bw.queue.metricsMu.Unlock() - - // Re-acquire lock (caller expects it held) - bw.mu.Lock() -} - -// Flush forces a flush of any pending entries -func (bw *BatchWriter) Flush() { - bw.mu.Lock() - defer bw.mu.Unlock() - if len(bw.pending) > 0 { - bw.flushLocked() - } -} - -// Size returns number of pending entries -func (bw *BatchWriter) Size() int { - bw.mu.Lock() - defer bw.mu.Unlock() - return len(bw.pending) -} - -// ExecuteBatchUpsert builds and executes a batch upsert using sqlx.Named -// lockFunc should lock all entities and return an unlock function -// The query should use :field placeholders matching the struct's db tags -func ExecuteBatchUpsert( - ctx context.Context, - dbConn *sqlx.DB, - query string, - entities interface{}, - lockFunc func() func(), -) error { - // Lock all entities to read their values - unlock := lockFunc() - - // Generate SQL and args while holding locks - expandedQuery, args, err := sqlx.Named(query, entities) - - // Release locks - args now contains the values - unlock() - - if err != nil { - return err - } - - expandedQuery = dbConn.Rebind(expandedQuery) - - // Execute without locks held - _, err = dbConn.ExecContext(ctx, expandedQuery, args...) - return err -} diff --git a/decoder/writebehind/limiter.go b/decoder/writebehind/limiter.go new file mode 100644 index 00000000..b72ef207 --- /dev/null +++ b/decoder/writebehind/limiter.go @@ -0,0 +1,42 @@ +package writebehind + +import ( + "context" +) + +// SharedLimiter coordinates concurrency across multiple queues +type SharedLimiter struct { + sem chan struct{} +} + +// NewSharedLimiter creates a limiter with the given max concurrent operations +func NewSharedLimiter(maxConcurrent int) *SharedLimiter { + return &SharedLimiter{ + sem: make(chan struct{}, maxConcurrent), + } +} + +// Acquire blocks until a slot is available, respecting context cancellation +func (l *SharedLimiter) Acquire(ctx context.Context) error { + select { + case l.sem <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Release frees a slot +func (l *SharedLimiter) Release() { + <-l.sem +} + +// TryAcquire attempts to acquire without blocking, returns false if unavailable +func (l *SharedLimiter) TryAcquire() bool { + select { + case l.sem <- struct{}{}: + return true + default: + return false + } +} diff --git a/decoder/writebehind/manager.go b/decoder/writebehind/manager.go new file mode 100644 index 00000000..c23c58cb --- /dev/null +++ b/decoder/writebehind/manager.go @@ -0,0 +1,155 @@ +package writebehind + +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +// Flushable represents a queue that can be managed +type Flushable interface { + ProcessLoop(ctx context.Context) + Flush(ctx context.Context) + Size() int + BatchSize() int + Name() string + GetAndResetMetrics() TypedQueueMetrics +} + +// QueueManager coordinates all typed queues +type QueueManager struct { + mu sync.RWMutex + queues []Flushable + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + startTime time.Time + startupDelayComplete bool + startupDelaySeconds int +} + +// NewQueueManager creates a new queue manager +func NewQueueManager(startupDelaySeconds int) *QueueManager { + return &QueueManager{ + queues: make([]Flushable, 0), + startTime: time.Now(), + startupDelaySeconds: startupDelaySeconds, + } +} + +// Register adds a queue to the manager +func (m *QueueManager) Register(queue Flushable) { + m.mu.Lock() + defer m.mu.Unlock() + m.queues = append(m.queues, queue) +} + +// Start begins processing all registered queues +func (m *QueueManager) Start(ctx context.Context) { + m.ctx, m.cancel = context.WithCancel(ctx) + + m.mu.RLock() + queues := m.queues + m.mu.RUnlock() + + for _, q := range queues { + m.wg.Add(1) + go func(queue Flushable) { + defer m.wg.Done() + queue.ProcessLoop(m.ctx) + }(q) + } + + // Start status logging + go m.statusLoop() + + log.Infof("Write-behind manager started with %d queues", len(queues)) +} + +// statusLoop periodically logs queue status +func (m *QueueManager) statusLoop() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.mu.RLock() + var totalPending, totalBatch int + var totalBatchCount, totalEntryCount int64 + var totalWriteTime, totalLatency float64 + var latencyCount int64 + + for _, q := range m.queues { + totalPending += q.Size() + totalBatch += q.BatchSize() + + metrics := q.GetAndResetMetrics() + totalBatchCount += metrics.BatchCount + totalEntryCount += metrics.BatchEntryCount + if metrics.BatchCount > 0 { + totalWriteTime += metrics.BatchAvgWriteMs * float64(metrics.BatchCount) + } + if metrics.BatchEntryCount > 0 { + totalLatency += metrics.BatchAvgLatencyMs * float64(metrics.BatchEntryCount) + latencyCount += metrics.BatchEntryCount + } + } + m.mu.RUnlock() + + var avgWriteMs, avgLatencyMs float64 + if totalBatchCount > 0 { + avgWriteMs = totalWriteTime / float64(totalBatchCount) + } + if latencyCount > 0 { + avgLatencyMs = totalLatency / float64(latencyCount) + } + + log.Infof("Write-behind: %d pending, %d in batches | %d entries in %d batches (avg write: %.1fms, avg latency: %.1fms)", + totalPending, totalBatch, totalEntryCount, totalBatchCount, avgWriteMs, avgLatencyMs) + } + } +} + +// Stop signals all queues to shutdown and waits for completion +func (m *QueueManager) Stop() { + if m.cancel != nil { + m.cancel() + } + m.wg.Wait() + log.Info("Write-behind manager stopped") +} + +// Flush forces all queues to flush immediately +func (m *QueueManager) Flush() { + m.mu.RLock() + queues := m.queues + m.mu.RUnlock() + + ctx := context.Background() + for _, q := range queues { + size := q.Size() + q.BatchSize() + if size > 0 { + log.Infof("Write-behind flushing %d %s entries", size, q.Name()) + } + q.Flush(ctx) + } + log.Info("Write-behind flush complete") +} + +// TotalSize returns the total number of pending entries across all queues +func (m *QueueManager) TotalSize() int { + m.mu.RLock() + defer m.mu.RUnlock() + + total := 0 + for _, q := range m.queues { + total += q.Size() + q.BatchSize() + } + return total +} diff --git a/decoder/writebehind/processor.go b/decoder/writebehind/processor.go deleted file mode 100644 index ee7b66e1..00000000 --- a/decoder/writebehind/processor.go +++ /dev/null @@ -1,107 +0,0 @@ -package writebehind - -import ( - "context" - "time" - - log "github.com/sirupsen/logrus" -) - -const ( - // dispatchInterval is how often the dispatcher checks for ready items - dispatchInterval = 100 * time.Millisecond - - // statusLogInterval is how often to log queue status - statusLogInterval = 30 * time.Second -) - -// ProcessLoop starts the dispatcher and workers -// This should be called in a goroutine -func (q *Queue) ProcessLoop(ctx context.Context) { - // Start worker goroutines - for i := 0; i < q.workerCount; i++ { - go q.worker(ctx, i) - } - - // Run dispatcher in this goroutine - q.dispatcher(ctx) -} - -// worker reads from the work channel and routes to batch writers -func (q *Queue) worker(ctx context.Context, id int) { - for { - select { - case <-ctx.Done(): - return - case entry := <-q.workChan: - // Route to batch writer if registered, otherwise write directly - tableType := entry.Entity.WriteType() - if bw := q.getBatchWriter(tableType); bw != nil { - bw.Add(entry) - } else { - q.writeEntry(entry) - } - } - } -} - -// dispatcher moves ready entries from pending map to work channel -func (q *Queue) dispatcher(ctx context.Context) { - ticker := time.NewTicker(dispatchInterval) - defer ticker.Stop() - - statusTicker := time.NewTicker(statusLogInterval) - defer statusTicker.Stop() - - for { - select { - case <-ctx.Done(): - log.Info("Write-behind dispatcher shutting down, flushing queue...") - q.Flush() - return - case <-statusTicker.C: - queueSize := q.Size() - channelLen := len(q.workChan) - metrics := q.GetAndResetMetrics() - log.Infof("Write-behind queue: %d pending, %d in channel | single: %d entries (avg write: %.1fms, avg latency: %.1fms) | batch: %d entries in %d batches (avg write: %.1fms, avg latency: %.1fms)", - queueSize, channelLen, - metrics.SingleEntryCount, metrics.SingleAvgWriteMs, metrics.SingleAvgLatencyMs, - metrics.BatchEntryCount, metrics.BatchCount, metrics.BatchAvgWriteMs, metrics.BatchAvgLatencyMs) - case <-ticker.C: - if q.checkWarmup() { - q.dispatchReady() - } - } - } -} - -// dispatchReady moves entries that are ready (delay expired) to the work channel -func (q *Queue) dispatchReady() { - q.mu.Lock() - defer q.mu.Unlock() - - if len(q.pending) == 0 { - return - } - - now := time.Now() - dispatched := 0 - - for key, entry := range q.pending { - // Check if delay has elapsed (entry is ready when now >= ReadyAt) - if now.Before(entry.ReadyAt) { - continue - } - - // Try to send to work channel (non-blocking) - select { - case q.workChan <- entry: - delete(q.pending, key) - dispatched++ - default: - // Channel full, stop dispatching this tick - // Workers will drain it and we'll dispatch more next tick - return - } - } -} diff --git a/decoder/writebehind/queue.go b/decoder/writebehind/queue.go deleted file mode 100644 index b5cad8fb..00000000 --- a/decoder/writebehind/queue.go +++ /dev/null @@ -1,304 +0,0 @@ -package writebehind - -import ( - "context" - "sync" - "time" - - "golbat/db" - "golbat/stats_collector" - - log "github.com/sirupsen/logrus" -) - -// Queue is the write-behind queue that buffers database writes -type Queue struct { - mu sync.Mutex - pending map[string]*QueueEntry // key -> entry for squashing - - workChan chan *QueueEntry // buffered channel for dispatcher - - // Batch writers per table type - batchWriters map[string]*BatchWriter - batchWritersMu sync.RWMutex - - workerCount int - warmupComplete bool - startTime time.Time - - // Metrics tracking (protected by metricsMu) - metricsMu sync.Mutex - // Single write metrics - singleWriteCount int64 // number of single entries written - singleWriteTime float64 // sum of single write durations in seconds - singleLatency float64 // sum of single write latencies in seconds - singleLatencyCount int64 // number of single latency samples - // Batch write metrics - batchCount int64 // number of batches executed - batchEntryCount int64 // total entries written via batches - batchWriteTime float64 // sum of batch execution times in seconds - batchLatency float64 // sum of batch entry latencies in seconds - batchLatencyCount int64 // number of batch latency samples - - config QueueConfig - db db.DbDetails - stats stats_collector.StatsCollector -} - -// NewQueue creates a new write-behind queue -func NewQueue(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.StatsCollector) *Queue { - if cfg.WorkerCount <= 0 { - cfg.WorkerCount = 50 // default - } - if cfg.BatchSize <= 0 { - cfg.BatchSize = 50 // default - } - if cfg.BatchTimeout <= 0 { - cfg.BatchTimeout = 100 * time.Millisecond // default - } - - return &Queue{ - pending: make(map[string]*QueueEntry), - workChan: make(chan *QueueEntry, cfg.WorkerCount*10), // buffer 10x worker count - batchWriters: make(map[string]*BatchWriter), - workerCount: cfg.WorkerCount, - warmupComplete: false, - startTime: time.Now(), - config: cfg, - db: dbDetails, - stats: stats, - } -} - -// RegisterBatchWriter registers a batch writer for a specific table type -func (q *Queue) RegisterBatchWriter(tableType string, flushFunc func(ctx context.Context, db db.DbDetails, entries []*QueueEntry) error) { - q.batchWritersMu.Lock() - defer q.batchWritersMu.Unlock() - - q.batchWriters[tableType] = NewBatchWriter(BatchWriterConfig{ - BatchSize: q.config.BatchSize, - Timeout: q.config.BatchTimeout, - TableType: tableType, - FlushFunc: flushFunc, - Db: q.db, - Stats: q.stats, - Queue: q, // Pass queue reference for metrics - }) -} - -// getBatchWriter returns the batch writer for a table type, or nil if not registered -func (q *Queue) getBatchWriter(tableType string) *BatchWriter { - q.batchWritersMu.RLock() - defer q.batchWritersMu.RUnlock() - return q.batchWriters[tableType] -} - -// Enqueue adds or updates an entity write -// If an entry already exists for the same key: -// - Entity is replaced with the newer one -// - IsNewRecord is preserved if either is true (INSERT takes priority) -// - Delay is updated to the minimum of existing and new delay (0 means immediate) -// - QueuedAt is preserved (for total time tracking) -// - ReadyAt is updated if the new delay makes the entry ready earlier -func (q *Queue) Enqueue(entity Writeable, isNewRecord bool, delay time.Duration) { - key := entity.WriteKey() - - q.mu.Lock() - defer q.mu.Unlock() - - now := time.Now() - - if existing, ok := q.pending[key]; ok { - // Update existing entry with newer entity - existing.Entity = entity - existing.UpdatedAt = now - // Preserve INSERT status - existing.IsNewRecord = existing.IsNewRecord || isNewRecord - // Use minimum delay (0 means write immediately) - if delay < existing.Delay { - existing.Delay = delay - // Update ReadyAt if this squash makes it ready earlier - newReadyAt := now.Add(delay) - if newReadyAt.Before(existing.ReadyAt) { - existing.ReadyAt = newReadyAt - } - } - q.stats.IncWriteBehindSquashed(entity.WriteType()) - } else { - // New entry - ReadyAt is when the entry becomes eligible for dispatch - readyAt := now.Add(delay) - q.pending[key] = &QueueEntry{ - Key: key, - Entity: entity, - QueuedAt: now, - UpdatedAt: now, - ReadyAt: readyAt, - IsNewRecord: isNewRecord, - Delay: delay, - } - } - - q.stats.SetWriteBehindQueueDepth(entity.WriteType(), float64(len(q.pending))) -} - -// Size returns the current pending queue size -func (q *Queue) Size() int { - q.mu.Lock() - defer q.mu.Unlock() - return len(q.pending) -} - -// WriteMetrics holds the metrics returned by GetAndResetMetrics -type WriteMetrics struct { - // Single write metrics - SingleEntryCount int64 // number of entries written via single writes - SingleAvgWriteMs float64 // average write time per single entry in milliseconds - SingleAvgLatencyMs float64 // average latency per single entry in milliseconds - // Batch write metrics - BatchCount int64 // number of batches executed - BatchEntryCount int64 // total entries written via batches - BatchAvgWriteMs float64 // average time per batch in milliseconds - BatchAvgLatencyMs float64 // average latency per batch entry in milliseconds -} - -// GetAndResetMetrics returns write metrics then resets counters -func (q *Queue) GetAndResetMetrics() WriteMetrics { - q.metricsMu.Lock() - defer q.metricsMu.Unlock() - - var singleAvgWrite, singleAvgLatency float64 - var batchAvgWrite, batchAvgLatency float64 - - if q.singleWriteCount > 0 { - singleAvgWrite = (q.singleWriteTime / float64(q.singleWriteCount)) * 1000 - } - if q.singleLatencyCount > 0 { - singleAvgLatency = (q.singleLatency / float64(q.singleLatencyCount)) * 1000 - } - if q.batchCount > 0 { - batchAvgWrite = (q.batchWriteTime / float64(q.batchCount)) * 1000 - } - if q.batchLatencyCount > 0 { - batchAvgLatency = (q.batchLatency / float64(q.batchLatencyCount)) * 1000 - } - - metrics := WriteMetrics{ - SingleEntryCount: q.singleWriteCount, - SingleAvgWriteMs: singleAvgWrite, - SingleAvgLatencyMs: singleAvgLatency, - BatchCount: q.batchCount, - BatchEntryCount: q.batchEntryCount, - BatchAvgWriteMs: batchAvgWrite, - BatchAvgLatencyMs: batchAvgLatency, - } - - // Reset counters - q.singleWriteCount = 0 - q.singleWriteTime = 0 - q.singleLatency = 0 - q.singleLatencyCount = 0 - q.batchCount = 0 - q.batchEntryCount = 0 - q.batchWriteTime = 0 - q.batchLatency = 0 - q.batchLatencyCount = 0 - - return metrics -} - -// IsWarmupComplete returns true if the warmup period has elapsed -func (q *Queue) IsWarmupComplete() bool { - q.mu.Lock() - defer q.mu.Unlock() - return q.warmupComplete -} - -// checkWarmup checks if warmup period has elapsed and updates state -func (q *Queue) checkWarmup() bool { - if q.warmupComplete { - return true - } - - elapsed := time.Since(q.startTime) - if elapsed >= time.Duration(q.config.StartupDelaySeconds)*time.Second { - q.mu.Lock() - if !q.warmupComplete { - q.warmupComplete = true - queueSize := len(q.pending) - q.mu.Unlock() - log.Infof("Write-behind warmup complete, processing %d queued writes with %d workers", queueSize, q.workerCount) - return true - } - q.mu.Unlock() - return true - } - return false -} - -// Flush writes all pending entries immediately (used during shutdown) -func (q *Queue) Flush() { - q.mu.Lock() - entries := make([]*QueueEntry, 0, len(q.pending)) - for _, entry := range q.pending { - entries = append(entries, entry) - } - q.pending = make(map[string]*QueueEntry) - q.mu.Unlock() - - if len(entries) == 0 { - log.Info("Write-behind flush: no pending entries") - } else { - log.Infof("Write-behind flushing %d pending entries", len(entries)) - - // Route entries to batch writers or write directly - for _, entry := range entries { - tableType := entry.Entity.WriteType() - if bw := q.getBatchWriter(tableType); bw != nil { - bw.Add(entry) - } else { - q.writeEntry(entry) - } - } - } - - // Flush all batch writers - q.batchWritersMu.RLock() - for tableType, bw := range q.batchWriters { - size := bw.Size() - if size > 0 { - log.Infof("Write-behind flushing %d %s batch entries", size, tableType) - } - bw.Flush() - } - q.batchWritersMu.RUnlock() - - log.Info("Write-behind flush complete") -} - -// writeEntry performs the actual database write for an entry (single write path) -func (q *Queue) writeEntry(entry *QueueEntry) { - start := time.Now() - - err := entry.Entity.WriteToDB(q.db, entry.IsNewRecord) - writeTime := time.Since(start).Seconds() - - entityType := entry.Entity.WriteType() - if err != nil { - q.stats.IncWriteBehindErrors(entityType) - log.Errorf("Write-behind error for %s: %v", entry.Key, err) - } else { - q.stats.IncWriteBehindWrites(entityType) - } - - // Latency is from when entry became ready (ReadyAt) to write completion - latency := time.Since(entry.ReadyAt).Seconds() - q.stats.ObserveWriteBehindLatency(entityType, latency) - - // Track single write metrics for status logging - q.metricsMu.Lock() - q.singleWriteCount++ - q.singleWriteTime += writeTime - q.singleLatency += latency - q.singleLatencyCount++ - q.metricsMu.Unlock() -} diff --git a/decoder/writebehind/queue_test.go b/decoder/writebehind/queue_test.go index 0038c2a3..993a47ff 100644 --- a/decoder/writebehind/queue_test.go +++ b/decoder/writebehind/queue_test.go @@ -1,6 +1,7 @@ package writebehind import ( + "context" "testing" "time" @@ -8,65 +9,66 @@ import ( "golbat/stats_collector" ) -// mockWriteable implements Writeable for testing -type mockWriteable struct { - key string - writeType string - quality int - written bool +// testData is the data type for testing +type testData struct { + key string + quality int } -func (m *mockWriteable) WriteKey() string { return m.key } -func (m *mockWriteable) WriteType() string { return m.writeType } -func (m *mockWriteable) WriteToDB(db db.DbDetails, isNewRecord bool) error { - m.written = true - return nil -} - -func TestQueueEnqueue(t *testing.T) { +func TestTypedQueueEnqueue(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() - cfg := QueueConfig{ - StartupDelaySeconds: 0, // No delay for tests - WorkerCount: 10, // 10 workers for tests - } - q := NewQueue(cfg, db.DbDetails{}, stats) - - entity := &mockWriteable{key: "test:1", writeType: "test", quality: 1} - q.Enqueue(entity, true, 0) + q := NewTypedQueue(TypedQueueConfig[string, testData]{ + Name: "test", + BatchSize: 50, + BatchTimeout: 100 * time.Millisecond, + StartupDelaySeconds: 0, // No delay for tests + Db: db.DbDetails{}, + Stats: stats, + FlushFunc: func(ctx context.Context, db db.DbDetails, entries []testData) error { return nil }, + KeyFunc: func(d testData) string { return d.key }, + }) + + data := testData{key: "test:1", quality: 1} + q.Enqueue(data, true, 0) if q.Size() != 1 { t.Errorf("Expected queue size 1, got %d", q.Size()) } } -func TestQueueSquashing(t *testing.T) { +func TestTypedQueueSquashing(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() - cfg := QueueConfig{ + q := NewTypedQueue(TypedQueueConfig[string, testData]{ + Name: "test", + BatchSize: 50, + BatchTimeout: 100 * time.Millisecond, StartupDelaySeconds: 0, - WorkerCount: 10, - } - q := NewQueue(cfg, db.DbDetails{}, stats) + Db: db.DbDetails{}, + Stats: stats, + FlushFunc: func(ctx context.Context, db db.DbDetails, entries []testData) error { return nil }, + KeyFunc: func(d testData) string { return d.key }, + }) // Enqueue first entity - entity1 := &mockWriteable{key: "test:1", writeType: "test", quality: 1} - q.Enqueue(entity1, true, 0) + data1 := testData{key: "test:1", quality: 1} + q.Enqueue(data1, true, 0) // Enqueue second entity with same key - entity2 := &mockWriteable{key: "test:1", writeType: "test", quality: 2} - q.Enqueue(entity2, false, 0) + data2 := testData{key: "test:1", quality: 2} + q.Enqueue(data2, false, 0) // Should still only have 1 entry (squashed) if q.Size() != 1 { t.Errorf("Expected queue size 1 after squash, got %d", q.Size()) } - // The entry should use the newer entity (replaces old) + // The entry should use the newer data (replaces old) q.mu.Lock() entry := q.pending["test:1"] q.mu.Unlock() - if entry.Entity.(*mockWriteable).quality != 2 { - t.Errorf("Expected entity quality 2 (newer), got %d", entry.Entity.(*mockWriteable).quality) + if entry.Data.quality != 2 { + t.Errorf("Expected quality 2 (newer), got %d", entry.Data.quality) } // IsNewRecord should be preserved (true || false = true) @@ -75,21 +77,26 @@ func TestQueueSquashing(t *testing.T) { } } -func TestQueueNewRecordPreservation(t *testing.T) { +func TestTypedQueueNewRecordPreservation(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() - cfg := QueueConfig{ + q := NewTypedQueue(TypedQueueConfig[string, testData]{ + Name: "test", + BatchSize: 50, + BatchTimeout: 100 * time.Millisecond, StartupDelaySeconds: 0, - WorkerCount: 10, - } - q := NewQueue(cfg, db.DbDetails{}, stats) + Db: db.DbDetails{}, + Stats: stats, + FlushFunc: func(ctx context.Context, db db.DbDetails, entries []testData) error { return nil }, + KeyFunc: func(d testData) string { return d.key }, + }) // Enqueue as new record - entity1 := &mockWriteable{key: "test:1", writeType: "test", quality: 1} - q.Enqueue(entity1, true, 0) + data1 := testData{key: "test:1", quality: 1} + q.Enqueue(data1, true, 0) // Enqueue update (not new) - entity2 := &mockWriteable{key: "test:1", writeType: "test", quality: 2} - q.Enqueue(entity2, false, 0) + data2 := testData{key: "test:1", quality: 2} + q.Enqueue(data2, false, 0) q.mu.Lock() entry := q.pending["test:1"] @@ -100,17 +107,22 @@ func TestQueueNewRecordPreservation(t *testing.T) { } } -func TestQueueDelayHandling(t *testing.T) { +func TestTypedQueueDelayHandling(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() - cfg := QueueConfig{ + q := NewTypedQueue(TypedQueueConfig[string, testData]{ + Name: "test", + BatchSize: 50, + BatchTimeout: 100 * time.Millisecond, StartupDelaySeconds: 0, - WorkerCount: 10, - } - q := NewQueue(cfg, db.DbDetails{}, stats) + Db: db.DbDetails{}, + Stats: stats, + FlushFunc: func(ctx context.Context, db db.DbDetails, entries []testData) error { return nil }, + KeyFunc: func(d testData) string { return d.key }, + }) // Enqueue with 1 second delay - entity1 := &mockWriteable{key: "test:1", writeType: "test", quality: 1} - q.Enqueue(entity1, true, 1*time.Second) + data1 := testData{key: "test:1", quality: 1} + q.Enqueue(data1, true, 1*time.Second) q.mu.Lock() entry := q.pending["test:1"] @@ -121,8 +133,8 @@ func TestQueueDelayHandling(t *testing.T) { } // Enqueue same key with 0 delay (should reduce delay) - entity2 := &mockWriteable{key: "test:1", writeType: "test", quality: 2} - q.Enqueue(entity2, false, 0) + data2 := testData{key: "test:1", quality: 2} + q.Enqueue(data2, false, 0) q.mu.Lock() entry = q.pending["test:1"] @@ -133,13 +145,18 @@ func TestQueueDelayHandling(t *testing.T) { } } -func TestQueueWarmup(t *testing.T) { +func TestTypedQueueWarmup(t *testing.T) { stats := stats_collector.NewNoopStatsCollector() - cfg := QueueConfig{ + q := NewTypedQueue(TypedQueueConfig[string, testData]{ + Name: "test", + BatchSize: 50, + BatchTimeout: 100 * time.Millisecond, StartupDelaySeconds: 1, // 1 second delay - WorkerCount: 10, - } - q := NewQueue(cfg, db.DbDetails{}, stats) + Db: db.DbDetails{}, + Stats: stats, + FlushFunc: func(ctx context.Context, db db.DbDetails, entries []testData) error { return nil }, + KeyFunc: func(d testData) string { return d.key }, + }) if q.IsWarmupComplete() { t.Error("Warmup should not be complete immediately") @@ -155,3 +172,47 @@ func TestQueueWarmup(t *testing.T) { t.Error("Warmup should be complete after delay") } } + +func TestTypedQueueIntegerKey(t *testing.T) { + stats := stats_collector.NewNoopStatsCollector() + + type intKeyData struct { + id uint64 + quality int + } + + q := NewTypedQueue(TypedQueueConfig[uint64, intKeyData]{ + Name: "test", + BatchSize: 50, + BatchTimeout: 100 * time.Millisecond, + StartupDelaySeconds: 0, + Db: db.DbDetails{}, + Stats: stats, + FlushFunc: func(ctx context.Context, db db.DbDetails, entries []intKeyData) error { return nil }, + KeyFunc: func(d intKeyData) uint64 { return d.id }, + }) + + // Enqueue with integer key + data1 := intKeyData{id: 12345678901234, quality: 1} + q.Enqueue(data1, true, 0) + + if q.Size() != 1 { + t.Errorf("Expected queue size 1, got %d", q.Size()) + } + + // Enqueue same key, should squash + data2 := intKeyData{id: 12345678901234, quality: 2} + q.Enqueue(data2, false, 0) + + if q.Size() != 1 { + t.Errorf("Expected queue size 1 after squash, got %d", q.Size()) + } + + q.mu.Lock() + entry := q.pending[uint64(12345678901234)] + q.mu.Unlock() + + if entry.Data.quality != 2 { + t.Errorf("Expected quality 2 (newer), got %d", entry.Data.quality) + } +} diff --git a/decoder/writebehind/s2cell_accumulator.go b/decoder/writebehind/s2cell_accumulator.go deleted file mode 100644 index 5fd556ee..00000000 --- a/decoder/writebehind/s2cell_accumulator.go +++ /dev/null @@ -1,192 +0,0 @@ -package writebehind - -import ( - "context" - "sync" - "time" - - "golbat/db" - "golbat/stats_collector" - - log "github.com/sirupsen/logrus" -) - -// S2CellData holds the data needed for an S2Cell write -type S2CellData struct { - Id uint64 - Latitude float64 - Longitude float64 - Level int64 - Updated int64 -} - -// S2CellAccumulator collects S2Cell updates and writes them in batches -type S2CellAccumulator struct { - mu sync.Mutex - pending map[uint64]*S2CellData // Dedupe by cell ID - - warmupComplete bool - startTime time.Time - - config QueueConfig - db db.DbDetails - stats stats_collector.StatsCollector - - // Batch write function - injected to avoid circular dependency - batchWriter func(db.DbDetails, []*S2CellData) error -} - -// NewS2CellAccumulator creates a new S2Cell accumulator -func NewS2CellAccumulator(cfg QueueConfig, dbDetails db.DbDetails, stats stats_collector.StatsCollector, batchWriter func(db.DbDetails, []*S2CellData) error) *S2CellAccumulator { - return &S2CellAccumulator{ - pending: make(map[uint64]*S2CellData), - warmupComplete: false, - startTime: time.Now(), - config: cfg, - db: dbDetails, - stats: stats, - batchWriter: batchWriter, - } -} - -// Add adds S2Cell data to the accumulator (deduplicates by ID) -func (a *S2CellAccumulator) Add(cells []*S2CellData) { - a.mu.Lock() - defer a.mu.Unlock() - - for _, cell := range cells { - if existing, ok := a.pending[cell.Id]; ok { - // Update existing - latest timestamp wins - if cell.Updated > existing.Updated { - a.pending[cell.Id] = cell - } - a.stats.IncWriteBehindSquashed("s2cell") - } else { - a.pending[cell.Id] = cell - } - } - - a.stats.SetWriteBehindQueueDepth("s2cell", float64(len(a.pending))) -} - -// Size returns the current number of pending cells -func (a *S2CellAccumulator) Size() int { - a.mu.Lock() - defer a.mu.Unlock() - return len(a.pending) -} - -// IsWarmupComplete returns true if the warmup period has elapsed -func (a *S2CellAccumulator) IsWarmupComplete() bool { - a.mu.Lock() - defer a.mu.Unlock() - return a.warmupComplete -} - -// checkWarmup checks if warmup period has elapsed and updates state -func (a *S2CellAccumulator) checkWarmup() bool { - if a.warmupComplete { - return true - } - - elapsed := time.Since(a.startTime) - if elapsed >= time.Duration(a.config.StartupDelaySeconds)*time.Second { - a.mu.Lock() - if !a.warmupComplete { - a.warmupComplete = true - queueSize := len(a.pending) - a.mu.Unlock() - log.Infof("S2Cell accumulator warmup complete, processing %d queued cells", queueSize) - return true - } - a.mu.Unlock() - return true - } - return false -} - -// Start begins the background processing loop -func (a *S2CellAccumulator) Start(ctx context.Context) { - go a.processLoop(ctx) -} - -// processLoop runs the background flush loop -func (a *S2CellAccumulator) processLoop(ctx context.Context) { - ticker := time.NewTicker(5 * time.Second) // Flush every 5 seconds - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - a.Flush() - log.Info("S2Cell accumulator stopped") - return - case <-ticker.C: - if a.checkWarmup() { - a.flushBatch() - } - } - } -} - -// flushBatch writes pending cells -func (a *S2CellAccumulator) flushBatch() { - a.mu.Lock() - if len(a.pending) == 0 { - a.mu.Unlock() - return - } - - // Take all pending cells - cells := make([]*S2CellData, 0, len(a.pending)) - for _, cell := range a.pending { - cells = append(cells, cell) - } - a.pending = make(map[uint64]*S2CellData) - a.mu.Unlock() - - batchSize := len(cells) - log.Debugf("S2Cell accumulator flushing batch of %d cells", batchSize) - - // Write batch - start := time.Now() - err := a.batchWriter(a.db, cells) - latency := time.Since(start).Seconds() - - if err != nil { - a.stats.IncWriteBehindErrors("s2cell") - log.Errorf("S2Cell batch write error: %v", err) - } else { - a.stats.IncWriteBehindWrites("s2cell") - } - - a.stats.ObserveWriteBehindLatency("s2cell", latency) - a.stats.SetWriteBehindQueueDepth("s2cell", float64(len(a.pending))) - a.stats.SetS2CellBatchSize(batchSize) -} - -// Flush writes all pending cells immediately (used during shutdown) -func (a *S2CellAccumulator) Flush() { - a.mu.Lock() - if len(a.pending) == 0 { - a.mu.Unlock() - return - } - - cells := make([]*S2CellData, 0, len(a.pending)) - for _, cell := range a.pending { - cells = append(cells, cell) - } - a.pending = make(map[uint64]*S2CellData) - a.mu.Unlock() - - log.Infof("S2Cell accumulator flushing %d cells", len(cells)) - - // Write without rate limiting during shutdown - err := a.batchWriter(a.db, cells) - if err != nil { - log.Errorf("S2Cell flush error: %v", err) - } - - log.Info("S2Cell accumulator flush complete") -} diff --git a/decoder/writebehind/typed_queue.go b/decoder/writebehind/typed_queue.go new file mode 100644 index 00000000..3a1c78a2 --- /dev/null +++ b/decoder/writebehind/typed_queue.go @@ -0,0 +1,400 @@ +package writebehind + +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/stats_collector" +) + +// Entry represents a pending write in a typed queue +type Entry[K comparable, T any] struct { + Key K + Data T + QueuedAt time.Time + UpdatedAt time.Time + ReadyAt time.Time // When the entry becomes eligible for dispatch + IsNewRecord bool // Track if this needs INSERT (preserved across updates) + Delay time.Duration // Minimum delay before writing (0 = immediate) +} + +// TypedQueueConfig holds configuration for a typed queue +type TypedQueueConfig[K comparable, T any] struct { + Name string + BatchSize int + BatchTimeout time.Duration + StartupDelaySeconds int + Limiter *SharedLimiter + Db db.DbDetails + Stats stats_collector.StatsCollector + // FlushFunc is called to write a batch of entries to the database + FlushFunc func(ctx context.Context, db db.DbDetails, entries []T) error + // KeyFunc extracts the unique key from an entry's data + KeyFunc func(data T) K +} + +// TypedQueue is a type-safe write-behind queue for a specific entity type +type TypedQueue[K comparable, T any] struct { + mu sync.Mutex + pending map[K]*Entry[K, T] // key -> entry for squashing + + // Batch accumulation + batchMu sync.Mutex + batchPending map[K]*Entry[K, T] + batchTimer *time.Timer + + name string + batchSize int + batchTimeout time.Duration + limiter *SharedLimiter + flushFunc func(ctx context.Context, db db.DbDetails, entries []T) error + keyFunc func(data T) K + db db.DbDetails + stats stats_collector.StatsCollector + + // Warmup tracking + warmupComplete bool + startTime time.Time + startupDelay time.Duration + + // Metrics (protected by metricsMu) + metricsMu sync.Mutex + batchCount int64 + batchEntryCount int64 + batchWriteTime float64 + batchLatency float64 + batchLatencyCount int64 +} + +// NewTypedQueue creates a new type-safe write-behind queue +func NewTypedQueue[K comparable, T any](cfg TypedQueueConfig[K, T]) *TypedQueue[K, T] { + if cfg.BatchSize <= 0 { + cfg.BatchSize = 50 + } + if cfg.BatchTimeout <= 0 { + cfg.BatchTimeout = 100 * time.Millisecond + } + + return &TypedQueue[K, T]{ + pending: make(map[K]*Entry[K, T]), + batchPending: make(map[K]*Entry[K, T]), + name: cfg.Name, + batchSize: cfg.BatchSize, + batchTimeout: cfg.BatchTimeout, + limiter: cfg.Limiter, + flushFunc: cfg.FlushFunc, + keyFunc: cfg.KeyFunc, + db: cfg.Db, + stats: cfg.Stats, + warmupComplete: false, + startTime: time.Now(), + startupDelay: time.Duration(cfg.StartupDelaySeconds) * time.Second, + } +} + +// Enqueue adds or updates an entry in the queue +// Takes data snapshot directly - caller is responsible for calling while holding entity lock +func (q *TypedQueue[K, T]) Enqueue(data T, isNewRecord bool, delay time.Duration) { + key := q.keyFunc(data) + + q.mu.Lock() + defer q.mu.Unlock() + + now := time.Now() + + if existing, ok := q.pending[key]; ok { + // Update existing entry with newer data + existing.Data = data + existing.UpdatedAt = now + // Preserve INSERT status + existing.IsNewRecord = existing.IsNewRecord || isNewRecord + // Use minimum delay + if delay < existing.Delay { + existing.Delay = delay + newReadyAt := now.Add(delay) + if newReadyAt.Before(existing.ReadyAt) { + existing.ReadyAt = newReadyAt + } + } + q.stats.IncWriteBehindSquashed(q.name) + } else { + readyAt := now.Add(delay) + q.pending[key] = &Entry[K, T]{ + Key: key, + Data: data, + QueuedAt: now, + UpdatedAt: now, + ReadyAt: readyAt, + IsNewRecord: isNewRecord, + Delay: delay, + } + } + + q.stats.SetWriteBehindQueueDepth(q.name, float64(len(q.pending))) +} + +// Size returns the current pending queue size +func (q *TypedQueue[K, T]) Size() int { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.pending) +} + +// BatchSize returns the current batch accumulation size +func (q *TypedQueue[K, T]) BatchSize() int { + q.batchMu.Lock() + defer q.batchMu.Unlock() + return len(q.batchPending) +} + +// IsWarmupComplete returns true if the warmup period has elapsed +func (q *TypedQueue[K, T]) IsWarmupComplete() bool { + q.mu.Lock() + defer q.mu.Unlock() + return q.warmupComplete +} + +// checkWarmup checks if warmup period has elapsed and updates state +func (q *TypedQueue[K, T]) checkWarmup() bool { + if q.warmupComplete { + return true + } + + elapsed := time.Since(q.startTime) + if elapsed >= q.startupDelay { + q.mu.Lock() + if !q.warmupComplete { + q.warmupComplete = true + queueSize := len(q.pending) + q.mu.Unlock() + log.Infof("Write-behind [%s] warmup complete, processing %d queued writes", q.name, queueSize) + return true + } + q.mu.Unlock() + return true + } + return false +} + +// ProcessLoop starts the dispatch loop for this queue +// Should be called in a goroutine +func (q *TypedQueue[K, T]) ProcessLoop(ctx context.Context) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Infof("Write-behind [%s] shutting down, flushing...", q.name) + q.Flush(ctx) + return + case <-ticker.C: + if q.checkWarmup() { + q.dispatchReady(ctx) + } + } + } +} + +// dispatchReady moves entries that are ready to the batch accumulator +func (q *TypedQueue[K, T]) dispatchReady(ctx context.Context) { + q.mu.Lock() + + if len(q.pending) == 0 { + q.mu.Unlock() + return + } + + now := time.Now() + toDispatch := make([]*Entry[K, T], 0) + + for key, entry := range q.pending { + if now.Before(entry.ReadyAt) { + continue + } + toDispatch = append(toDispatch, entry) + delete(q.pending, key) + } + q.mu.Unlock() + + // Add to batch accumulator + for _, entry := range toDispatch { + q.addToBatch(ctx, entry) + } +} + +// addToBatch adds an entry to the batch accumulator, flushing if batch is full +func (q *TypedQueue[K, T]) addToBatch(ctx context.Context, entry *Entry[K, T]) { + q.batchMu.Lock() + defer q.batchMu.Unlock() + + // Deduplicate within batch (squash) + if existing, ok := q.batchPending[entry.Key]; ok { + entry.IsNewRecord = entry.IsNewRecord || existing.IsNewRecord + if existing.QueuedAt.Before(entry.QueuedAt) { + entry.QueuedAt = existing.QueuedAt + } + if existing.ReadyAt.Before(entry.ReadyAt) { + entry.ReadyAt = existing.ReadyAt + } + q.stats.IncWriteBehindSquashed(q.name) + } + q.batchPending[entry.Key] = entry + + if len(q.batchPending) >= q.batchSize { + q.flushBatchLocked(ctx) + } else if q.batchTimer == nil { + q.batchTimer = time.AfterFunc(q.batchTimeout, func() { + q.batchMu.Lock() + defer q.batchMu.Unlock() + if len(q.batchPending) > 0 { + q.flushBatchLocked(context.Background()) + } + }) + } +} + +// flushBatchLocked flushes the current batch (must be called with batchMu held) +func (q *TypedQueue[K, T]) flushBatchLocked(ctx context.Context) { + if q.batchTimer != nil { + q.batchTimer.Stop() + q.batchTimer = nil + } + + if len(q.batchPending) == 0 { + return + } + + // Convert map to slice (order doesn't matter for upserts) + entries := make([]*Entry[K, T], 0, len(q.batchPending)) + for _, entry := range q.batchPending { + entries = append(entries, entry) + } + q.batchPending = make(map[K]*Entry[K, T]) + + // Release batch lock before doing I/O + q.batchMu.Unlock() + + // Acquire concurrency slot from shared limiter + if q.limiter != nil { + if err := q.limiter.Acquire(ctx); err != nil { + // Context cancelled, re-queue entries + q.batchMu.Lock() + for _, entry := range entries { + q.batchPending[entry.Key] = entry + } + return + } + defer q.limiter.Release() + } + + // Execute batch write + start := time.Now() + data := make([]T, len(entries)) + for i, entry := range entries { + data[i] = entry.Data + } + + err := q.flushFunc(ctx, q.db, data) + batchTime := time.Since(start).Seconds() + entryCount := len(entries) + + if err != nil { + q.stats.IncWriteBehindErrors(q.name) + log.Errorf("Write-behind [%s] batch error (%d entries): %v", q.name, entryCount, err) + } else { + for range entries { + q.stats.IncWriteBehindWrites(q.name) + } + q.stats.IncWriteBehindBatches(q.name) + q.stats.ObserveWriteBehindBatchSize(q.name, float64(entryCount)) + q.stats.ObserveWriteBehindBatchTime(q.name, batchTime) + } + + // Track metrics + q.metricsMu.Lock() + q.batchCount++ + q.batchEntryCount += int64(entryCount) + q.batchWriteTime += batchTime + for _, entry := range entries { + latency := time.Since(entry.ReadyAt).Seconds() + q.batchLatency += latency + q.batchLatencyCount++ + q.stats.ObserveWriteBehindLatency(q.name, latency) + } + q.metricsMu.Unlock() + + // Re-acquire lock (caller expects it held) + q.batchMu.Lock() +} + +// Flush writes all pending entries immediately +func (q *TypedQueue[K, T]) Flush(ctx context.Context) { + // Move all pending to batch + q.mu.Lock() + entries := make([]*Entry[K, T], 0, len(q.pending)) + for _, entry := range q.pending { + entries = append(entries, entry) + } + q.pending = make(map[K]*Entry[K, T]) + q.mu.Unlock() + + // Add all to batch + for _, entry := range entries { + q.addToBatch(ctx, entry) + } + + // Force flush the batch + q.batchMu.Lock() + if len(q.batchPending) > 0 { + q.flushBatchLocked(ctx) + } + q.batchMu.Unlock() +} + +// TypedQueueMetrics holds the metrics for a typed queue +type TypedQueueMetrics struct { + BatchCount int64 + BatchEntryCount int64 + BatchAvgWriteMs float64 + BatchAvgLatencyMs float64 +} + +// GetAndResetMetrics returns metrics then resets counters +func (q *TypedQueue[K, T]) GetAndResetMetrics() TypedQueueMetrics { + q.metricsMu.Lock() + defer q.metricsMu.Unlock() + + var batchAvgWrite, batchAvgLatency float64 + if q.batchCount > 0 { + batchAvgWrite = (q.batchWriteTime / float64(q.batchCount)) * 1000 + } + if q.batchLatencyCount > 0 { + batchAvgLatency = (q.batchLatency / float64(q.batchLatencyCount)) * 1000 + } + + metrics := TypedQueueMetrics{ + BatchCount: q.batchCount, + BatchEntryCount: q.batchEntryCount, + BatchAvgWriteMs: batchAvgWrite, + BatchAvgLatencyMs: batchAvgLatency, + } + + // Reset + q.batchCount = 0 + q.batchEntryCount = 0 + q.batchWriteTime = 0 + q.batchLatency = 0 + q.batchLatencyCount = 0 + + return metrics +} + +// Name returns the queue name +func (q *TypedQueue[K, T]) Name() string { + return q.name +} diff --git a/decoder/writebehind/types.go b/decoder/writebehind/types.go index 5f8c5f04..1a0cfc08 100644 --- a/decoder/writebehind/types.go +++ b/decoder/writebehind/types.go @@ -2,35 +2,9 @@ package writebehind import ( "time" - - "golbat/db" ) -// Writeable is implemented by any entity that can be written through the queue -type Writeable interface { - // WriteKey returns a unique key for this entity (for squashing) - WriteKey() string - - // WriteType returns the entity type name (for metrics) - WriteType() string - - // WriteToDB performs the actual database write - // isNewRecord determines INSERT vs UPDATE - WriteToDB(db db.DbDetails, isNewRecord bool) error -} - -// QueueEntry represents a pending write in the queue -type QueueEntry struct { - Key string - Entity Writeable - QueuedAt time.Time - UpdatedAt time.Time - ReadyAt time.Time // When the entry became ready to write (set by dispatcher) - IsNewRecord bool // Track if this needs INSERT (preserved across updates) - Delay time.Duration // Minimum delay before writing (0 = immediate) -} - -// QueueConfig holds configuration for the write-behind queue +// QueueConfig holds configuration for write-behind queues type QueueConfig struct { StartupDelaySeconds int // Delay before processing starts (warmup period) WorkerCount int // Number of concurrent write workers (default 50) diff --git a/decoder/writebehind_batch.go b/decoder/writebehind_batch.go index c9dce2de..78a3cd04 100644 --- a/decoder/writebehind_batch.go +++ b/decoder/writebehind_batch.go @@ -2,129 +2,260 @@ package decoder import ( "context" + "time" log "github.com/sirupsen/logrus" + "golbat/config" "golbat/db" "golbat/decoder/writebehind" + "golbat/stats_collector" ) -// RegisterBatchWriters registers all batch writers with the write-behind queue -func RegisterBatchWriters(queue *writebehind.Queue) { - queue.RegisterBatchWriter("pokestop", flushPokestopBatch) - queue.RegisterBatchWriter("gym", flushGymBatch) - queue.RegisterBatchWriter("pokemon", flushPokemonBatch) - queue.RegisterBatchWriter("spawnpoint", flushSpawnpointBatch) - - log.Info("Write-behind batch writers registered for: pokestop, gym, pokemon, spawnpoint") +// S2CellData holds the data needed for an S2Cell write +type S2CellData struct { + Id uint64 `db:"id"` + Latitude float64 `db:"center_lat"` + Longitude float64 `db:"center_lon"` + Level int64 `db:"level"` + Updated int64 `db:"updated"` } -// flushPokestopBatch writes a batch of pokestops using INSERT ... ON DUPLICATE KEY UPDATE -func flushPokestopBatch(ctx context.Context, dbDetails db.DbDetails, entries []*writebehind.QueueEntry) error { - pokestops := make([]*Pokestop, len(entries)) - for i, e := range entries { - pokestops[i] = e.Entity.(*Pokestop) +// Typed queues for each entity type - using native key types for efficiency +var ( + pokestopQueue *writebehind.TypedQueue[string, PokestopData] + gymQueue *writebehind.TypedQueue[string, GymData] + pokemonQueue *writebehind.TypedQueue[uint64, PokemonData] + spawnpointQueue *writebehind.TypedQueue[int64, SpawnpointData] + routeQueue *writebehind.TypedQueue[string, RouteData] + tappableQueue *writebehind.TypedQueue[uint64, TappableData] + stationQueue *writebehind.TypedQueue[string, StationData] + incidentQueue *writebehind.TypedQueue[string, IncidentData] + s2cellQueue *writebehind.TypedQueue[uint64, S2CellData] + + // QueueManager coordinates all queues + queueManager *writebehind.QueueManager +) + +// InitTypedQueues initializes all typed write-behind queues +func InitTypedQueues(ctx context.Context, dbDetails db.DbDetails, stats stats_collector.StatsCollector) { + startupDelay := config.Config.Tuning.WriteBehindStartupDelay + batchSize := config.Config.Tuning.WriteBehindBatchSize + batchTimeout := time.Duration(config.Config.Tuning.WriteBehindBatchTimeoutMs) * time.Millisecond + workerCount := config.Config.Tuning.WriteBehindWorkerCount + + if batchSize <= 0 { + batchSize = 50 + } + if batchTimeout <= 0 { + batchTimeout = 100 * time.Millisecond + } + if workerCount <= 0 { + workerCount = 50 } - return writebehind.ExecuteBatchUpsert( - ctx, - dbDetails.GeneralDb, - pokestopBatchUpsertQuery, - pokestops, - func() func() { - // Lock all pokestops in sorted order - for _, p := range pokestops { - p.Lock() - } - // Return unlock function (unlock in reverse order) - return func() { - for i := len(pokestops) - 1; i >= 0; i-- { - pokestops[i].Unlock() - } - } - }, - ) -} + // Shared limiter coordinates concurrency across all queues + limiter := writebehind.NewSharedLimiter(workerCount) + + // Create queue manager + queueManager = writebehind.NewQueueManager(startupDelay) + + // Create typed queues for each entity type - using native key types + pokestopQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[string, PokestopData]{ + Name: "pokestop", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushPokestopBatch, + KeyFunc: func(d PokestopData) string { return d.Id }, + }) + queueManager.Register(pokestopQueue) + + gymQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[string, GymData]{ + Name: "gym", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushGymBatch, + KeyFunc: func(d GymData) string { return d.Id }, + }) + queueManager.Register(gymQueue) + + pokemonQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[uint64, PokemonData]{ + Name: "pokemon", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushPokemonBatchTyped, + KeyFunc: func(d PokemonData) uint64 { return uint64(d.Id) }, + }) + queueManager.Register(pokemonQueue) + + spawnpointQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[int64, SpawnpointData]{ + Name: "spawnpoint", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushSpawnpointBatch, + KeyFunc: func(d SpawnpointData) int64 { return d.Id }, + }) + queueManager.Register(spawnpointQueue) + + routeQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[string, RouteData]{ + Name: "route", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushRouteBatch, + KeyFunc: func(d RouteData) string { return d.Id }, + }) + queueManager.Register(routeQueue) + + tappableQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[uint64, TappableData]{ + Name: "tappable", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushTappableBatch, + KeyFunc: func(d TappableData) uint64 { return d.Id }, + }) + queueManager.Register(tappableQueue) + + stationQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[string, StationData]{ + Name: "station", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushStationBatch, + KeyFunc: func(d StationData) string { return d.Id }, + }) + queueManager.Register(stationQueue) + + incidentQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[string, IncidentData]{ + Name: "incident", + BatchSize: batchSize, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushIncidentBatch, + KeyFunc: func(d IncidentData) string { return d.Id }, + }) + queueManager.Register(incidentQueue) + + s2cellQueue = writebehind.NewTypedQueue(writebehind.TypedQueueConfig[uint64, S2CellData]{ + Name: "s2cell", + BatchSize: 100, + BatchTimeout: batchTimeout, + StartupDelaySeconds: startupDelay, + Limiter: limiter, + Db: dbDetails, + Stats: stats, + FlushFunc: flushS2CellBatch, + KeyFunc: func(d S2CellData) uint64 { return d.Id }, + }) + queueManager.Register(s2cellQueue) + + log.Infof("Typed write-behind queues initialized: startup_delay=%ds, batch_size=%d, batch_timeout=%dms, max_concurrent=%d", + startupDelay, batchSize, batchTimeout.Milliseconds(), workerCount) -// flushGymBatch writes a batch of gyms using INSERT ... ON DUPLICATE KEY UPDATE -func flushGymBatch(ctx context.Context, dbDetails db.DbDetails, entries []*writebehind.QueueEntry) error { - gyms := make([]*Gym, len(entries)) - for i, e := range entries { - gyms[i] = e.Entity.(*Gym) + // Warn if concurrency exceeds half of database pool size + maxPool := config.Config.Database.MaxPool + if workerCount > maxPool/2 { + log.Warnf("Write-behind concurrency (%d) exceeds half of database pool size (%d). "+ + "Consider increasing database.max_pool or reducing tuning.write_behind_worker_count", + workerCount, maxPool) } - return writebehind.ExecuteBatchUpsert( - ctx, - dbDetails.GeneralDb, - gymBatchUpsertQuery, - gyms, - func() func() { - // Lock all gyms in sorted order - for _, g := range gyms { - g.Lock() - } - // Return unlock function (unlock in reverse order) - return func() { - for i := len(gyms) - 1; i >= 0; i-- { - gyms[i].Unlock() - } - } - }, - ) + // Start the queue manager + queueManager.Start(ctx) } -// flushPokemonBatch writes a batch of pokemon using INSERT ... ON DUPLICATE KEY UPDATE -func flushPokemonBatch(ctx context.Context, dbDetails db.DbDetails, entries []*writebehind.QueueEntry) error { - pokemon := make([]*Pokemon, len(entries)) - for i, e := range entries { - pokemon[i] = e.Entity.(*Pokemon) +// FlushTypedQueues flushes all typed queues (for shutdown) +func FlushTypedQueues() { + if queueManager != nil { + queueManager.Flush() } - - return writebehind.ExecuteBatchUpsert( - ctx, - dbDetails.PokemonDb, - pokemonBatchUpsertQuery, - pokemon, - func() func() { - // Lock all pokemon in sorted order - for _, p := range pokemon { - p.Lock() - } - // Return unlock function (unlock in reverse order) - return func() { - for i := len(pokemon) - 1; i >= 0; i-- { - pokemon[i].Unlock() - } - } - }, - ) } -// flushSpawnpointBatch writes a batch of spawnpoints using INSERT ... ON DUPLICATE KEY UPDATE -func flushSpawnpointBatch(ctx context.Context, dbDetails db.DbDetails, entries []*writebehind.QueueEntry) error { - spawnpoints := make([]*Spawnpoint, len(entries)) - for i, e := range entries { - spawnpoints[i] = e.Entity.(*Spawnpoint) +// StopTypedQueues stops all typed queues gracefully +func StopTypedQueues() { + if queueManager != nil { + queueManager.Stop() } +} + +// Flush functions for typed queues - receive []T directly, no type assertions needed + +func flushPokestopBatch(ctx context.Context, dbDetails db.DbDetails, pokestops []PokestopData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, pokestopBatchUpsertQuery, pokestops) + return err +} + +func flushGymBatch(ctx context.Context, dbDetails db.DbDetails, gyms []GymData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, gymBatchUpsertQuery, gyms) + return err +} + +func flushPokemonBatchTyped(ctx context.Context, dbDetails db.DbDetails, pokemon []PokemonData) error { + _, err := dbDetails.PokemonDb.NamedExecContext(ctx, pokemonBatchUpsertQuery, pokemon) + return err +} + +func flushSpawnpointBatch(ctx context.Context, dbDetails db.DbDetails, spawnpoints []SpawnpointData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, spawnpointBatchUpsertQuery, spawnpoints) + return err +} + +func flushRouteBatch(ctx context.Context, dbDetails db.DbDetails, routes []RouteData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, routeBatchUpsertQuery, routes) + return err +} + +func flushTappableBatch(ctx context.Context, dbDetails db.DbDetails, tappables []TappableData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, tappableBatchUpsertQuery, tappables) + return err +} + +func flushStationBatch(ctx context.Context, dbDetails db.DbDetails, stations []StationData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, stationBatchUpsertQuery, stations) + return err +} + +func flushIncidentBatch(ctx context.Context, dbDetails db.DbDetails, incidents []IncidentData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, incidentBatchUpsertQuery, incidents) + return err +} - return writebehind.ExecuteBatchUpsert( - ctx, - dbDetails.GeneralDb, - spawnpointBatchUpsertQuery, - spawnpoints, - func() func() { - // Lock all spawnpoints in sorted order - for _, s := range spawnpoints { - s.Lock() - } - // Return unlock function (unlock in reverse order) - return func() { - for i := len(spawnpoints) - 1; i >= 0; i-- { - spawnpoints[i].Unlock() - } - } - }, - ) +func flushS2CellBatch(ctx context.Context, dbDetails db.DbDetails, cells []S2CellData) error { + _, err := dbDetails.GeneralDb.NamedExecContext(ctx, s2cellBatchUpsertQuery, cells) + if err != nil { + log.Errorf("flushS2CellBatch: %s", err) + } + statsCollector.IncDbQuery("insert s2cell", err) + return err } // Batch upsert queries - using INSERT ... ON DUPLICATE KEY UPDATE @@ -328,7 +459,7 @@ ON DUPLICATE KEY UPDATE expire_timestamp_verified = VALUES(expire_timestamp_verified), shiny = VALUES(shiny), username = VALUES(username), - pvp = VALUES(pvp), + pvp = COALESCE(VALUES(pvp), pvp), is_event = VALUES(is_event), seen_type = VALUES(seen_type) ` @@ -347,3 +478,138 @@ ON DUPLICATE KEY UPDATE last_seen=VALUES(last_seen), despawn_sec = VALUES(despawn_sec) ` + +const routeBatchUpsertQuery = ` +INSERT INTO route ( + id, name, shortcode, description, distance_meters, + duration_seconds, end_fort_id, end_image, end_lat, end_lon, + image, image_border_color, reversible, start_fort_id, start_image, + start_lat, start_lon, tags, type, updated, version, waypoints +) +VALUES ( + :id, :name, :shortcode, :description, :distance_meters, + :duration_seconds, :end_fort_id, :end_image, :end_lat, :end_lon, + :image, :image_border_color, :reversible, :start_fort_id, :start_image, + :start_lat, :start_lon, :tags, :type, :updated, :version, :waypoints +) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + shortcode = VALUES(shortcode), + description = VALUES(description), + distance_meters = VALUES(distance_meters), + duration_seconds = VALUES(duration_seconds), + end_fort_id = VALUES(end_fort_id), + end_image = VALUES(end_image), + end_lat = VALUES(end_lat), + end_lon = VALUES(end_lon), + image = VALUES(image), + image_border_color = VALUES(image_border_color), + reversible = VALUES(reversible), + start_fort_id = VALUES(start_fort_id), + start_image = VALUES(start_image), + start_lat = VALUES(start_lat), + start_lon = VALUES(start_lon), + tags = VALUES(tags), + type = VALUES(type), + updated = VALUES(updated), + version = VALUES(version), + waypoints = VALUES(waypoints) +` + +const tappableBatchUpsertQuery = ` +INSERT INTO tappable ( + id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, + count, expire_timestamp, expire_timestamp_verified, updated +) +VALUES ( + :id, :lat, :lon, :fort_id, :spawn_id, :type, :pokemon_id, :item_id, + :count, :expire_timestamp, :expire_timestamp_verified, :updated +) +ON DUPLICATE KEY UPDATE + lat = VALUES(lat), + lon = VALUES(lon), + fort_id = VALUES(fort_id), + spawn_id = VALUES(spawn_id), + type = VALUES(type), + pokemon_id = VALUES(pokemon_id), + item_id = VALUES(item_id), + count = VALUES(count), + expire_timestamp = VALUES(expire_timestamp), + expire_timestamp_verified = VALUES(expire_timestamp_verified), + updated = VALUES(updated) +` + +const stationBatchUpsertQuery = ` +INSERT INTO station ( + id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, + is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, + battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, + battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, + battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, + total_stationed_gmax, stationed_pokemon +) +VALUES ( + :id, :lat, :lon, :name, :cell_id, :start_time, :end_time, :cooldown_complete, + :is_battle_available, :is_inactive, :updated, :battle_level, :battle_start, :battle_end, + :battle_pokemon_id, :battle_pokemon_form, :battle_pokemon_costume, :battle_pokemon_gender, + :battle_pokemon_alignment, :battle_pokemon_bread_mode, :battle_pokemon_move_1, :battle_pokemon_move_2, + :battle_pokemon_stamina, :battle_pokemon_cp_multiplier, :total_stationed_pokemon, + :total_stationed_gmax, :stationed_pokemon +) +ON DUPLICATE KEY UPDATE + lat = VALUES(lat), + lon = VALUES(lon), + name = VALUES(name), + cell_id = VALUES(cell_id), + start_time = VALUES(start_time), + end_time = VALUES(end_time), + cooldown_complete = VALUES(cooldown_complete), + is_battle_available = VALUES(is_battle_available), + is_inactive = VALUES(is_inactive), + updated = VALUES(updated), + battle_level = VALUES(battle_level), + battle_start = VALUES(battle_start), + battle_end = VALUES(battle_end), + battle_pokemon_id = VALUES(battle_pokemon_id), + battle_pokemon_form = VALUES(battle_pokemon_form), + battle_pokemon_costume = VALUES(battle_pokemon_costume), + battle_pokemon_gender = VALUES(battle_pokemon_gender), + battle_pokemon_alignment = VALUES(battle_pokemon_alignment), + battle_pokemon_bread_mode = VALUES(battle_pokemon_bread_mode), + battle_pokemon_move_1 = VALUES(battle_pokemon_move_1), + battle_pokemon_move_2 = VALUES(battle_pokemon_move_2), + battle_pokemon_stamina = VALUES(battle_pokemon_stamina), + battle_pokemon_cp_multiplier = VALUES(battle_pokemon_cp_multiplier), + total_stationed_pokemon = VALUES(total_stationed_pokemon), + total_stationed_gmax = VALUES(total_stationed_gmax), + stationed_pokemon = VALUES(stationed_pokemon) +` + +const incidentBatchUpsertQuery = "INSERT INTO incident (" + + "id, pokestop_id, start, expiration, display_type, style, `character`, " + + "updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, " + + "slot_2_form, slot_3_pokemon_id, slot_3_form" + + ") VALUES (" + + ":id, :pokestop_id, :start, :expiration, :display_type, :style, :character, " + + ":updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, " + + ":slot_2_form, :slot_3_pokemon_id, :slot_3_form" + + ") ON DUPLICATE KEY UPDATE " + + "start = VALUES(start), " + + "expiration = VALUES(expiration), " + + "display_type = VALUES(display_type), " + + "style = VALUES(style), " + + "`character` = VALUES(`character`), " + + "updated = VALUES(updated), " + + "confirmed = VALUES(confirmed), " + + "slot_1_pokemon_id = VALUES(slot_1_pokemon_id), " + + "slot_1_form = VALUES(slot_1_form), " + + "slot_2_pokemon_id = VALUES(slot_2_pokemon_id), " + + "slot_2_form = VALUES(slot_2_form), " + + "slot_3_pokemon_id = VALUES(slot_3_pokemon_id), " + + "slot_3_form = VALUES(slot_3_form)" + +const s2cellBatchUpsertQuery = ` +INSERT INTO s2cell (id, center_lat, center_lon, level, updated) +VALUES (:id, :center_lat, :center_lon, :level, :updated) +ON DUPLICATE KEY UPDATE updated = VALUES(updated) +` diff --git a/main.go b/main.go index 068c6968..2979073e 100644 --- a/main.go +++ b/main.go @@ -209,7 +209,6 @@ func main() { } decoder.LoadStatsGeofences() decoder.InitWriteBehindQueue(ctx, dbDetails) - decoder.InitS2CellAccumulator(ctx, dbDetails) InitDeviceCache() wg.Add(1) @@ -402,7 +401,6 @@ func main() { wg.Wait() log.Info("go routines have exited, flushing write-behind queue...") - decoder.FlushS2CellAccumulator() decoder.FlushWriteBehindQueue() log.Info("flushing webhooks now...") diff --git a/webhooks/webhooks_test.go b/webhooks/webhooks_test.go index c56b35e9..130a807f 100644 --- a/webhooks/webhooks_test.go +++ b/webhooks/webhooks_test.go @@ -161,7 +161,7 @@ func TestWebhooksFull(t *testing.T) { defer wg.Done() err := sender.Run(ctx) if err != nil { - t.Fatalf("unexpected error starting webhooksSender: %s", err) + t.Errorf("unexpected error starting webhooksSender: %s", err) } }() From 7a16db092d8ca953975f28a037c068dc2143933a Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 16:38:02 +0000 Subject: [PATCH 58/78] Remove deadlock library --- decoder/gym.go | 4 ++-- decoder/pokemon.go | 4 ++-- decoder/pokestop.go | 4 ++-- decoder/spawnpoint.go | 4 ++-- go.mod | 8 +++----- go.sum | 16 ++++++---------- 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index 32aaa49b..84b54896 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -2,9 +2,9 @@ package decoder import ( "fmt" + "sync" "github.com/guregu/null/v6" - "github.com/sasha-s/go-deadlock" ) // GymData contains all database-persisted fields for a Gym. @@ -56,7 +56,7 @@ type GymData struct { // Gym struct. // REMINDER! Keep hasChangesGym updated after making changes type Gym struct { - mu deadlock.Mutex `db:"-"` // Object-level mutex + mu sync.Mutex `db:"-"` // Object-level mutex GymData // Embedded data fields (all db columns) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index a2b99ceb..e03d7090 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -2,11 +2,11 @@ package decoder import ( "fmt" + "sync" "golbat/grpc" "github.com/guregu/null/v6" - "github.com/sasha-s/go-deadlock" ) // PokemonData contains all database-persisted fields for Pokemon. @@ -62,7 +62,7 @@ type PokemonData struct { // // FirstSeenTimestamp: This field is used in IsNewRecord. It should only be set in savePokemonRecord. type Pokemon struct { - mu deadlock.Mutex `db:"-"` // Object-level mutex + mu sync.Mutex `db:"-"` // Object-level mutex PokemonData // Embedded data fields - can be copied for write-behind queue diff --git a/decoder/pokestop.go b/decoder/pokestop.go index db8567e8..e693ab94 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -2,9 +2,9 @@ package decoder import ( "fmt" + "sync" "github.com/guregu/null/v6" - "github.com/sasha-s/go-deadlock" ) // PokestopData contains all database-persisted fields for Pokestop. @@ -67,7 +67,7 @@ type PokestopData struct { // Pokestop struct. type Pokestop struct { - mu deadlock.Mutex `db:"-"` // Object-level mutex + mu sync.Mutex `db:"-"` // Object-level mutex PokestopData // Embedded data fields - can be copied for write-behind queue diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 4bf46118..fc21a6ff 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strconv" + "sync" "time" "golbat/db" @@ -13,7 +14,6 @@ import ( "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" - "github.com/sasha-s/go-deadlock" log "github.com/sirupsen/logrus" ) @@ -35,7 +35,7 @@ type SpawnpointData struct { // Spawnpoint struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Spawnpoint struct { - mu deadlock.Mutex `db:"-" json:"-"` // Object-level mutex + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex SpawnpointData // Embedded data fields - can be copied for write-behind queue diff --git a/go.mod b/go.mod index 6ba08f3b..ccd5a87c 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/cloudwego/base64x v0.1.6 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -66,20 +66,18 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect github.com/ringsaturn/tzf-rel-lite v0.0.2025-c // indirect - github.com/sasha-s/go-deadlock v0.3.6 // indirect github.com/tidwall/geoindex v1.7.0 // indirect github.com/tidwall/geojson v1.4.6 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twpayne/go-polyline v1.1.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect - go.mongodb.org/mongo-driver v1.17.8 // indirect + go.mongodb.org/mongo-driver v1.17.9 // indirect go.uber.org/mock v0.6.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/arch v0.23.0 // indirect @@ -88,5 +86,5 @@ require ( golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect ) diff --git a/go.sum b/go.sum index dd75feb2..d0f7f9c2 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= -github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8= github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= @@ -174,8 +174,6 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe h1:vHpqOnPlnkba8iSxU4j/CvDSS9J4+F4473esQsYLGoE= -github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -207,8 +205,6 @@ github.com/ringsaturn/tzf-rel-lite v0.0.2025-c h1:CUs4l73ApN87MhlAhp1UtcRe3E5UFM github.com/ringsaturn/tzf-rel-lite v0.0.2025-c/go.mod h1:SyVF6OU+Le0vKajtTA7PvYabdYCJsDlmplHuXeCZDrw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/sasha-s/go-deadlock v0.3.6 h1:TR7sfOnZ7x00tWPfD397Peodt57KzMDo+9Ae9rMiUmw= -github.com/sasha-s/go-deadlock v0.3.6/go.mod h1:CUqNyyvMxTyjFqDT7MRg9mb4Dv/btmGTqSR+rky/UXo= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -259,8 +255,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.17.8 h1:BDP3+U3Y8K0vTrpqDJIRaXNhb/bKyoVeg6tIJsW5EhM= -go.mongodb.org/mongo-driver v1.17.8/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= +go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= +go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= @@ -330,8 +326,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= From 3c33579ceae81a2dd014d7759fb25bf7ff71ce7c Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 17:22:51 +0000 Subject: [PATCH 59/78] Update config example --- config.toml.example | 50 +++++++++++++++++++++++++++++++++------------ config/config.go | 1 + main.go | 6 ++++-- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/config.toml.example b/config.toml.example index 629aea9d..449626fb 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1,14 +1,11 @@ port = 9001 # Listening port for golbat -#grpc_port = 50001 # Listening port for grpc +#grpc_port = 50001 # Listening port for grpc raw_bearer = "" # Raw bearer (password) required api_secret = "golbat" # Golbat secret required on api calls (blank for none) -pokemon_memory_only = false # Use in-memory storage for pokemon only - -# preload: Pre-load objects into cache on startup (faster initial responses, but increases startup time) -# fort_in_memory: Keep forts in memory with rtree for spatial lookups -preload = false -fort_in_memory = false +pokemon_memory_only = true # Use in-memory storage for pokemon only +preload = false # preload: Pre-load objects into cache on startup (faster initial responses, but increases startup time) +fort_in_memory = false # fort_in_memory: Keep forts in memory for api lookups [experimental] [koji] url = "http://{koji_url}/api/v1/geofence/feature-collection/{golbat_project}" @@ -27,24 +24,28 @@ forts_stale_threshold = 3600 # Seconds before a fort is considered stale (def [logging] debug = false save_logs = true -max_size = 50 # Size in MB -max_backups = 10 # Amount of files to keep -max_age = 30 # Day(s) to keep files -compress = true # Compress to gz archive +max_size = 50 # Size in MB +max_backups = 10 # Amount of files to keep +max_age = 30 # Day(s) to keep files +compress = true # Compress to gz archive [database] user = "" password = "" address = "127.0.0.1:3306" db = "" +max_pool = 100 # Maximum database connection pool size [pvp] enabled = true include_hundos_under_cap = false +level_caps = [50, 51] # Level caps used in PVP rankings +#ranking_comparator = "prefer_higher_cp" # Options: "prefer_higher_cp", "prefer_lower_cp" (default: tied rankings) # you can enable prometheus by uncommenting this section #[prometheus] #enabled = true +#token = "" # Optional auth token for /metrics endpoint # You can specify more than one webhook destination by including the [[webhooks]] section # multiple times. The hook types can optionally be filtered by using the types array @@ -69,16 +70,39 @@ url = "http://localhost:4201" max_pokemon_distance = 100 # Maximum distance in kilometers for searching pokemon max_pokemon_results = 3000 # Maximum number of pokemon to return extended_timeout = false # Extend timeouts for processing, hopefully not needed +write_behind_startup_delay = 120 # Writes will be queued for this quiet time +write_behind_worker_count = 50 # Maximum number of parallel batch writes +write_behind_batch_size = 50 # Number of entries per batch write +write_behind_batch_timeout = 100 # Max wait time in ms before flushing partial batch profile_routes = false # Turn on debugging endpoints +profile_contention = false # Collect data for contention (use with above) - has a perf impact # When enabled, reduce_updates will make fort update debounce windows much longer # to reduce database churn. Specifically, gym/pokestop/station debounce will be # extended from 15 minutes (900s) to 12 hours (43200s) and spawnpoint last_seen # will be updated every 12 hours instead of the default 6 hours. reduce_updates = false -write_behind_startup_delay = 120 -write_behind_worker_count = 50 [weather] proactive_iv_switching = true # Enable proactive IV switching upon weather changes (default: true) proactive_iv_switching_to_db = false # Write proactive IV changes to database (default: false) + +# Scan rules allow controlling which game objects are processed based on area or scanner context. +# Rules are processed in order - first match applies. +# Available options: pokemon, wild_pokemon, nearby_pokemon, nearby_cell_pokemon, weather, cells, +# pokestops, gyms, stations, tappables, proactive_iv_switching +# +# Example: Disable nearby pokemon in MainArea, allow everything from Scout context, disable all pokemon elsewhere + +# Under heavy load you will gain an advantage from disabling nearby_pokemon +# Contexts sent from dragonite are Fort, Pokemon, Quest, Scout, RarePokemon + +#[[scan_rules]] +#areas = ["MainArea"] +#nearby_pokemon = false +# +#[[scan_rules]] +#context = ["Scout"] +# +#[[scan_rules]] +#pokemon = false diff --git a/config/config.go b/config/config.go index b481c87b..cfff9f09 100644 --- a/config/config.go +++ b/config/config.go @@ -124,6 +124,7 @@ type tuning struct { MaxPokemonResults int `koanf:"max_pokemon_results"` MaxPokemonDistance float64 `koanf:"max_pokemon_distance"` ProfileRoutes bool `koanf:"profile_routes"` + ProfileContention bool `koanf:"profile_contention"` // Enable mutex/block profiling (has overhead) MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` ReduceUpdates bool `koanf:"reduce_updates"` WriteBehindStartupDelay int `koanf:"write_behind_startup_delay"` // seconds, default: 120 diff --git a/main.go b/main.go index 2979073e..0cdf9156 100644 --- a/main.go +++ b/main.go @@ -354,8 +354,10 @@ func main() { pprofGroup.GET("/goroutine", func(c *gin.Context) { pprof.Handler("goroutine").ServeHTTP(c.Writer, c.Request) }) - runtime.SetBlockProfileRate(1) - runtime.SetMutexProfileFraction(1) + if config.Config.Tuning.ProfileContention { + runtime.SetBlockProfileRate(1) + runtime.SetMutexProfileFraction(1) + } } srv := &http.Server{ From 735fa2ca4b95303bf359ca8b3653673e32adceeb Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 17:32:40 +0000 Subject: [PATCH 60/78] No need for removed forts to spin off a go-routine --- decode.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/decode.go b/decode.go index daab1cb3..6a2f4d4c 100644 --- a/decode.go +++ b/decode.go @@ -562,11 +562,7 @@ func decodeGMO(ctx context.Context, protoData *ProtoData, scanParameters decoder } if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - decoder.CheckRemovedForts(ctx, dbDetails, cellsToBeCleaned, cellForts) - }() + decoder.CheckRemovedForts(ctx, dbDetails, cellsToBeCleaned, cellForts) } newFortsLen := len(newForts) From 2e32bee75853d4d1f1ab33a1ec95325273d6e77b Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 17:38:38 +0000 Subject: [PATCH 61/78] Pass correct context on shutdown --- decoder/writebehind/typed_queue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decoder/writebehind/typed_queue.go b/decoder/writebehind/typed_queue.go index 3a1c78a2..518dfefa 100644 --- a/decoder/writebehind/typed_queue.go +++ b/decoder/writebehind/typed_queue.go @@ -190,7 +190,7 @@ func (q *TypedQueue[K, T]) ProcessLoop(ctx context.Context) { select { case <-ctx.Done(): log.Infof("Write-behind [%s] shutting down, flushing...", q.name) - q.Flush(ctx) + q.Flush(context.Background()) return case <-ticker.C: if q.checkWarmup() { From 526420cfd533fbd740b17333e58d5d2691787605 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 18:26:57 +0000 Subject: [PATCH 62/78] Option to store pokemon to disk on shutdown --- config.toml.example | 1 + config/config.go | 41 +++++------ decoder/pokemon_preserve.go | 136 ++++++++++++++++++++++++++++++++++++ decoder/pokemon_state.go | 14 ++-- main.go | 11 +++ 5 files changed, 178 insertions(+), 25 deletions(-) create mode 100644 decoder/pokemon_preserve.go diff --git a/config.toml.example b/config.toml.example index 449626fb..df9f9c2d 100644 --- a/config.toml.example +++ b/config.toml.example @@ -4,6 +4,7 @@ raw_bearer = "" # Raw bearer (password) required api_secret = "golbat" # Golbat secret required on api calls (blank for none) pokemon_memory_only = true # Use in-memory storage for pokemon only +#preserve_pokemon = false # Save pokemon to DB on shutdown, restore on startup (for memory-only mode) preload = false # preload: Pre-load objects into cache on startup (faster initial responses, but increases startup time) fort_in_memory = false # fort_in_memory: Keep forts in memory for api lookups [experimental] diff --git a/config/config.go b/config/config.go index cfff9f09..af002e8a 100644 --- a/config/config.go +++ b/config/config.go @@ -7,26 +7,27 @@ import ( ) type configDefinition struct { - Port int `koanf:"port"` - GrpcPort int `koanf:"grpc_port"` - Webhooks []Webhook `koanf:"webhooks"` - Database database `koanf:"database"` - Logging logging `koanf:"logging"` - Sentry sentry `koanf:"sentry"` - Pyroscope pyroscope `koanf:"pyroscope"` - Prometheus Prometheus `koanf:"prometheus"` - PokemonMemoryOnly bool `koanf:"pokemon_memory_only"` - PokemonInternalToDb bool `koanf:"pokemon_internal_to_db"` - Preload bool `koanf:"preload"` // Pre-load forts, stations, spawnpoints into cache on startup - FortInMemory bool `koanf:"fort_in_memory"` // Keep forts in memory with rtree for spatial lookups - Cleanup cleanup `koanf:"cleanup"` - RawBearer string `koanf:"raw_bearer"` - ApiSecret string `koanf:"api_secret"` - Pvp pvp `koanf:"pvp"` - Koji koji `koanf:"koji"` - Tuning tuning `koanf:"tuning"` - Weather weather `koanf:"weather"` - ScanRules []scanRule `koanf:"scan_rules"` + Port int `koanf:"port"` + GrpcPort int `koanf:"grpc_port"` + Webhooks []Webhook `koanf:"webhooks"` + Database database `koanf:"database"` + Logging logging `koanf:"logging"` + Sentry sentry `koanf:"sentry"` + Pyroscope pyroscope `koanf:"pyroscope"` + Prometheus Prometheus `koanf:"prometheus"` + PokemonMemoryOnly bool `koanf:"pokemon_memory_only"` + PokemonInternalToDb bool `koanf:"pokemon_internal_to_db"` + PreserveInMemoryPokemon bool `koanf:"preserve_pokemon"` // Save/restore pokemon cache on shutdown/startup + Preload bool `koanf:"preload"` // Pre-load forts, stations, spawnpoints into cache on startup + FortInMemory bool `koanf:"fort_in_memory"` // Keep forts in memory with rtree for spatial lookups + Cleanup cleanup `koanf:"cleanup"` + RawBearer string `koanf:"raw_bearer"` + ApiSecret string `koanf:"api_secret"` + Pvp pvp `koanf:"pvp"` + Koji koji `koanf:"koji"` + Tuning tuning `koanf:"tuning"` + Weather weather `koanf:"weather"` + ScanRules []scanRule `koanf:"scan_rules"` } func (configDefinition configDefinition) GetWebhookInterval() time.Duration { diff --git a/decoder/pokemon_preserve.go b/decoder/pokemon_preserve.go new file mode 100644 index 00000000..a5475f9d --- /dev/null +++ b/decoder/pokemon_preserve.go @@ -0,0 +1,136 @@ +package decoder + +import ( + "context" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/db" +) + +// preserveBatchSize is the number of pokemon to write per batch +const preserveBatchSize = 1000 + +// PreservePokemonToDatabase writes all non-expired pokemon from cache to database. +// Called during shutdown when preserve_pokemon is enabled. +// Does not take locks since cache is no longer being modified at shutdown. +func PreservePokemonToDatabase(dbDetails db.DbDetails) { + startTime := time.Now() + now := time.Now().Unix() + + var saved, skipped, errored int + batch := make([]PokemonData, 0, preserveBatchSize) + ctx := context.Background() + + flushBatch := func() { + if len(batch) == 0 { + return + } + _, err := dbDetails.PokemonDb.NamedExecContext(ctx, pokemonBatchUpsertQuery, batch) + if err != nil { + log.Errorf("PreservePokemon: batch write error - %s", err) + errored += len(batch) + } else { + saved += len(batch) + } + // Reset batch by reslicing to zero length (reuses backing array) + batch = batch[:0] + } + + // Stream through cache, batching writes + pokemonCache.Range(func(item *ttlcache.Item[uint64, *Pokemon]) bool { + pokemon := item.Value() + + // Skip if expired or no valid expire timestamp (no lock needed at shutdown) + if !pokemon.ExpireTimestamp.Valid || pokemon.ExpireTimestamp.Int64 <= now { + skipped++ + return true + } + + // Add to batch + batch = append(batch, pokemon.PokemonData) + + // Flush when batch is full + if len(batch) >= preserveBatchSize { + flushBatch() + if saved%10000 == 0 && saved > 0 { + log.Infof("PreservePokemon: saved %d pokemon...", saved) + } + } + + return true + }) + + // Flush remaining + flushBatch() + + log.Infof("PreservePokemon: saved %d pokemon, skipped %d expired, %d errors in %v", + saved, skipped, errored, time.Since(startTime)) +} + +// PreloadPreservedPokemon loads non-expired pokemon from database into cache. +// Called during startup when preserve_pokemon is enabled. +func PreloadPreservedPokemon(dbDetails db.DbDetails) int32 { + startTime := time.Now() + now := time.Now().Unix() + + // Load pokemon that haven't expired yet + query := "SELECT " + pokemonSelectColumns + " FROM pokemon WHERE expire_timestamp > ?" + rows, err := dbDetails.PokemonDb.Queryx(query, now) + if err != nil { + log.Errorf("PreloadPokemon: failed to query pokemon - %s", err) + return 0 + } + defer rows.Close() + + numWorkers := runtime.NumCPU() + jobs := make(chan *Pokemon, 100) + var wg sync.WaitGroup + var count int32 + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + currentTime := time.Now().Unix() + for pokemon := range jobs { + // Calculate remaining TTL + ttl := pokemon.remainingDuration(currentTime) + if ttl <= 0 { + continue + } + + // Add to cache with appropriate TTL + pokemonCache.Set(uint64(pokemon.Id), pokemon, ttl) + + // Update rtree + pokemonRtreeUpdatePokemonOnGet(pokemon) + + c := atomic.AddInt32(&count, 1) + if c%10000 == 0 { + log.Infof("PreloadPokemon: loaded %d pokemon...", c) + } + } + }() + } + + for rows.Next() { + var pokemon Pokemon + err := rows.StructScan(&pokemon) + if err != nil { + log.Errorf("PreloadPokemon: pokemon scan error - %s", err) + continue + } + jobs <- &pokemon + } + close(jobs) + wg.Wait() + + log.Infof("PreloadPokemon: loaded %d pokemon in %v", count, time.Since(startTime)) + return count +} diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index 1c5ebd81..c1c6582b 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -23,6 +23,13 @@ import ( // wildPokemonDelay is how long wild Pokemon wait for encounter data before writing const wildPokemonDelay = 30 * time.Second +// pokemonSelectColumns defines the columns for pokemon queries. +// Used by both single-row and bulk load queries to keep them in sync. +const pokemonSelectColumns = `id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, + golbat_internal, iv, move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, + display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, + expire_timestamp_verified, shiny, username, pvp, is_event, seen_type` + // peekPokemonRecordReadOnly acquires lock, does NOT take snapshot. // Use for read-only checks which will not cause a backing database lookup // Caller must use returned unlock function @@ -38,11 +45,8 @@ func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { func loadPokemonFromDatabase(ctx context.Context, db db.DbDetails, encounterId uint64, pokemon *Pokemon) error { err := db.PokemonDb.GetContext(ctx, pokemon, - "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ - "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ - "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ - "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ - "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) + "SELECT "+pokemonSelectColumns+" FROM pokemon WHERE id = ?", + strconv.FormatUint(encounterId, 10)) statsCollector.IncDbQuery("select pokemon", err) return err diff --git a/main.go b/main.go index 0cdf9156..656729a8 100644 --- a/main.go +++ b/main.go @@ -279,6 +279,11 @@ func main() { } } + // Load preserved pokemon if enabled + if cfg.PreserveInMemoryPokemon && cfg.PokemonMemoryOnly { + decoder.PreloadPreservedPokemon(dbDetails) + } + // Start the GRPC receiver if cfg.GrpcPort > 0 { @@ -402,6 +407,12 @@ func main() { log.Info("http server is shutdown, waiting for other go routines to exit...") wg.Wait() + // Preserve in-memory pokemon if enabled + if cfg.PreserveInMemoryPokemon && cfg.PokemonMemoryOnly { + log.Info("preserving in-memory pokemon to database...") + decoder.PreservePokemonToDatabase(dbDetails) + } + log.Info("go routines have exited, flushing write-behind queue...") decoder.FlushWriteBehindQueue() From c398dd8c3b14162f8432c995c71afb9a1be1f233 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 18:44:04 +0000 Subject: [PATCH 63/78] Fix quest seed in webhook --- decoder/gym.go | 4 ++-- decoder/gym_decode.go | 3 +-- decoder/gym_state.go | 8 +++++++- decoder/pokestop_state.go | 17 ++++++++++++++--- main.go | 6 +++--- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index 84b54896..022fbf6d 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -61,7 +61,7 @@ type Gym struct { GymData // Embedded data fields (all db columns) // Memory-only fields (not persisted to DB) - RaidSeed null.String `db:"-"` // Raid seed (memory only, sent in webhook) + RaidSeed null.Int `db:"-"` // Raid seed (memory only, sent in webhook as string) dirty bool `db:"-"` // Not persisted - tracks if object needs saving (to db) internalDirty bool `db:"-"` // Not persisted - tracks if object needs saving (in memory only) @@ -532,7 +532,7 @@ func (gym *Gym) SetRsvps(v null.String) { } // SetRaidSeed sets the raid seed (memory only, not saved to DB) -func (gym *Gym) SetRaidSeed(v null.String) { +func (gym *Gym) SetRaidSeed(v null.Int) { if gym.RaidSeed != v { if dbDebugEnabled { gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidSeed:%s->%s", FormatNull(gym.RaidSeed), FormatNull(v))) diff --git a/decoder/gym_decode.go b/decoder/gym_decode.go index d30397d4..a1026511 100644 --- a/decoder/gym_decode.go +++ b/decoder/gym_decode.go @@ -4,7 +4,6 @@ import ( "cmp" "encoding/json" "slices" - "strconv" "strings" "time" @@ -124,7 +123,7 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64 if fortData.RaidInfo != nil { gym.SetRaidEndTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000)) gym.SetRaidSpawnTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000)) - gym.SetRaidSeed(null.StringFrom(strconv.FormatInt(fortData.RaidInfo.RaidSeed, 10))) + gym.SetRaidSeed(null.IntFrom(fortData.RaidInfo.RaidSeed)) raidBattleTimestamp := int64(fortData.RaidInfo.RaidBattleMs) / 1000 if gym.RaidBattleTimestamp.ValueOrZero() != raidBattleTimestamp { diff --git a/decoder/gym_state.go b/decoder/gym_state.go index f9170dbd..c60097ba 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "strconv" "time" "github.com/guregu/null/v6" @@ -345,7 +346,12 @@ func createGymWebhooks(gym *Gym, areas []geo.AreaName) { PowerUpEndTimestamp: gym.PowerUpEndTimestamp.ValueOrZero(), ArScanEligible: gym.ArScanEligible.ValueOrZero(), Rsvps: rsvps, - RaidSeed: gym.RaidSeed, + RaidSeed: func() null.String { + if gym.RaidSeed.Valid { + return null.StringFrom(strconv.FormatInt(gym.RaidSeed.Int64, 10)) + } + return null.String{} + }(), } webhooksSender.AddMessage(webhooks.Raid, raidHook, areas) diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index 9321ac5c..40a0f960 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "strconv" "time" "github.com/guregu/null/v6" @@ -165,7 +166,7 @@ type QuestWebhook struct { ArScanEligible int64 `json:"ar_scan_eligible"` PokestopUrl string `json:"pokestop_url"` WithAr bool `json:"with_ar"` - QuestSeed null.Int `json:"quest_seed"` + QuestSeed null.String `json:"quest_seed"` } type PokestopWebhook struct { @@ -235,7 +236,12 @@ func createPokestopWebhooks(stop *Pokestop) { ArScanEligible: stop.ArScanEligible.ValueOrZero(), PokestopUrl: stop.Url.ValueOrZero(), WithAr: false, - QuestSeed: stop.AlternativeQuestSeed, + QuestSeed: func() null.String { + if stop.AlternativeQuestSeed.Valid { + return null.StringFrom(strconv.FormatInt(stop.AlternativeQuestSeed.Int64, 10)) + } + return null.String{} + }(), } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } @@ -256,7 +262,12 @@ func createPokestopWebhooks(stop *Pokestop) { ArScanEligible: stop.ArScanEligible.ValueOrZero(), PokestopUrl: stop.Url.ValueOrZero(), WithAr: true, - QuestSeed: stop.QuestSeed, + QuestSeed: func() null.String { + if stop.QuestSeed.Valid { + return null.StringFrom(strconv.FormatInt(stop.QuestSeed.Int64, 10)) + } + return null.String{} + }(), } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } diff --git a/main.go b/main.go index 656729a8..ce66c422 100644 --- a/main.go +++ b/main.go @@ -407,15 +407,15 @@ func main() { log.Info("http server is shutdown, waiting for other go routines to exit...") wg.Wait() + log.Info("go routines have exited, flushing write-behind queue...") + decoder.FlushWriteBehindQueue() + // Preserve in-memory pokemon if enabled if cfg.PreserveInMemoryPokemon && cfg.PokemonMemoryOnly { log.Info("preserving in-memory pokemon to database...") decoder.PreservePokemonToDatabase(dbDetails) } - log.Info("go routines have exited, flushing write-behind queue...") - decoder.FlushWriteBehindQueue() - log.Info("flushing webhooks now...") webhooksSender.Flush() From 619b6283339789490d08881863601e3eafbb5434 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 18:49:13 +0000 Subject: [PATCH 64/78] Skip preserve pokemon api call --- decoder/pokemon_preserve.go | 13 +++++++++++++ main.go | 12 +++++++++--- routes.go | 7 +++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/decoder/pokemon_preserve.go b/decoder/pokemon_preserve.go index a5475f9d..5f9d63d1 100644 --- a/decoder/pokemon_preserve.go +++ b/decoder/pokemon_preserve.go @@ -16,6 +16,19 @@ import ( // preserveBatchSize is the number of pokemon to write per batch const preserveBatchSize = 1000 +// skipPreservePokemon is set via API to prevent PreservePokemonToDatabase on shutdown +var skipPreservePokemon atomic.Bool + +// SetSkipPreservePokemon sets the flag to skip pokemon preservation on shutdown +func SetSkipPreservePokemon(skip bool) { + skipPreservePokemon.Store(skip) +} + +// ShouldPreservePokemon returns true if preservation should be performed +func ShouldPreservePokemon() bool { + return !skipPreservePokemon.Load() +} + // PreservePokemonToDatabase writes all non-expired pokemon from cache to database. // Called during shutdown when preserve_pokemon is enabled. // Does not take locks since cache is no longer being modified at shutdown. diff --git a/main.go b/main.go index ce66c422..4556fbc0 100644 --- a/main.go +++ b/main.go @@ -330,6 +330,8 @@ func main() { apiGroup.GET("/tappable/id/:tappable_id", GetTappable) apiGroup.GET("/devices/all", GetDevices) + apiGroup.GET("/skip-preserve-pokemon", SkipPreservePokemon) + apiGroup.POST("/skip-preserve-pokemon", SkipPreservePokemon) debugGroup := r.Group("/debug") @@ -410,10 +412,14 @@ func main() { log.Info("go routines have exited, flushing write-behind queue...") decoder.FlushWriteBehindQueue() - // Preserve in-memory pokemon if enabled + // Preserve in-memory pokemon if enabled and not skipped via API if cfg.PreserveInMemoryPokemon && cfg.PokemonMemoryOnly { - log.Info("preserving in-memory pokemon to database...") - decoder.PreservePokemonToDatabase(dbDetails) + if decoder.ShouldPreservePokemon() { + log.Info("preserving in-memory pokemon to database...") + decoder.PreservePokemonToDatabase(dbDetails) + } else { + log.Info("skipping pokemon preservation (disabled via API)") + } } log.Info("flushing webhooks now...") diff --git a/routes.go b/routes.go index 75719d36..61349880 100644 --- a/routes.go +++ b/routes.go @@ -808,3 +808,10 @@ func GetTappable(c *gin.Context) { func GetDevices(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"devices": GetAllDevices()}) } + +// SkipPreservePokemon sets a flag to prevent pokemon preservation on shutdown +func SkipPreservePokemon(c *gin.Context) { + decoder.SetSkipPreservePokemon(true) + log.Info("Skip preserve pokemon flag set - pokemon will not be preserved on shutdown") + c.JSON(http.StatusOK, gin.H{"status": "ok", "message": "Pokemon preservation will be skipped on shutdown"}) +} From b0b0948368c637bbdfec83ccac71ca30bc7f7d89 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 18:54:26 +0000 Subject: [PATCH 65/78] Fix overlength gym/pokestop names --- decoder/gym.go | 8 ++++++++ decoder/pokestop.go | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/decoder/gym.go b/decoder/gym.go index 022fbf6d..c072f87f 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -5,6 +5,8 @@ import ( "sync" "github.com/guregu/null/v6" + + "golbat/util" ) // GymData contains all database-persisted fields for a Gym. @@ -170,6 +172,12 @@ func (gym *Gym) SetLon(v float64) { } func (gym *Gym) SetName(v null.String) { + // Truncate to 128 runes to fit database column + if v.Valid { + if truncated, changed := util.TruncateUTF8(v.String, 128); changed { + v = null.StringFrom(truncated) + } + } if gym.Name != v { if dbDebugEnabled { gym.changedFields = append(gym.changedFields, fmt.Sprintf("Name:%s->%s", FormatNull(gym.Name), FormatNull(v))) diff --git a/decoder/pokestop.go b/decoder/pokestop.go index e693ab94..74ee6ecb 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -5,6 +5,8 @@ import ( "sync" "github.com/guregu/null/v6" + + "golbat/util" ) // PokestopData contains all database-persisted fields for Pokestop. @@ -216,6 +218,12 @@ func (p *Pokestop) SetLon(v float64) { } func (p *Pokestop) SetName(v null.String) { + // Truncate to 128 runes to fit database column + if v.Valid { + if truncated, changed := util.TruncateUTF8(v.String, 128); changed { + v = null.StringFrom(truncated) + } + } if p.Name != v { if dbDebugEnabled { p.changedFields = append(p.changedFields, fmt.Sprintf("Name:%s->%s", FormatNull(p.Name), FormatNull(v))) From c15470cbfb240932953bfa6324569b8a5e4b6b2d Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 7 Feb 2026 19:01:28 +0000 Subject: [PATCH 66/78] Make cleanup pokemon work with preserver --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 4556fbc0..64552213 100644 --- a/main.go +++ b/main.go @@ -231,7 +231,7 @@ func main() { log.Info("Extended timeout enabled") } - if cfg.Cleanup.Pokemon == true && !cfg.PokemonMemoryOnly { + if cfg.Cleanup.Pokemon && (!cfg.PokemonMemoryOnly || cfg.PreserveInMemoryPokemon) { StartDatabaseArchiver(db) } From 2964f702eb3106b3d6ea83532ee2d44dadb759de Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 8 Feb 2026 15:54:11 +0000 Subject: [PATCH 67/78] Max length for routes --- decoder/routes.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/decoder/routes.go b/decoder/routes.go index bedc33e4..ba52db2f 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -5,6 +5,8 @@ import ( "sync" "github.com/guregu/null/v6" + + "golbat/util" ) // RouteData contains all database-persisted fields for Route. @@ -89,6 +91,8 @@ func (r *Route) snapshotOldValues() { // --- Set methods with dirty tracking --- func (r *Route) SetName(v string) { + // Truncate to 50 runes to fit database column + v, _ = util.TruncateUTF8(v, 50) if r.Name != v { if dbDebugEnabled { r.changedFields = append(r.changedFields, fmt.Sprintf("Name:%s->%s", r.Name, v)) From 28e6885759ef0249b52bdbad6c85276791c8b6e1 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 8 Feb 2026 16:01:29 +0000 Subject: [PATCH 68/78] Try to avoid deadlocks through sorting, and also retry mechanism --- decoder/writebehind/typed_queue.go | 35 +++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/decoder/writebehind/typed_queue.go b/decoder/writebehind/typed_queue.go index 518dfefa..d2ae14e1 100644 --- a/decoder/writebehind/typed_queue.go +++ b/decoder/writebehind/typed_queue.go @@ -1,18 +1,26 @@ package writebehind import ( + "cmp" "context" + "slices" "sync" "time" + "github.com/go-sql-driver/mysql" log "github.com/sirupsen/logrus" "golbat/db" "golbat/stats_collector" ) +const ( + mysqlDeadlock = 1213 + deadlockRetries = 3 +) + // Entry represents a pending write in a typed queue -type Entry[K comparable, T any] struct { +type Entry[K cmp.Ordered, T any] struct { Key K Data T QueuedAt time.Time @@ -23,7 +31,7 @@ type Entry[K comparable, T any] struct { } // TypedQueueConfig holds configuration for a typed queue -type TypedQueueConfig[K comparable, T any] struct { +type TypedQueueConfig[K cmp.Ordered, T any] struct { Name string BatchSize int BatchTimeout time.Duration @@ -38,7 +46,7 @@ type TypedQueueConfig[K comparable, T any] struct { } // TypedQueue is a type-safe write-behind queue for a specific entity type -type TypedQueue[K comparable, T any] struct { +type TypedQueue[K cmp.Ordered, T any] struct { mu sync.Mutex pending map[K]*Entry[K, T] // key -> entry for squashing @@ -71,7 +79,7 @@ type TypedQueue[K comparable, T any] struct { } // NewTypedQueue creates a new type-safe write-behind queue -func NewTypedQueue[K comparable, T any](cfg TypedQueueConfig[K, T]) *TypedQueue[K, T] { +func NewTypedQueue[K cmp.Ordered, T any](cfg TypedQueueConfig[K, T]) *TypedQueue[K, T] { if cfg.BatchSize <= 0 { cfg.BatchSize = 50 } @@ -292,6 +300,11 @@ func (q *TypedQueue[K, T]) flushBatchLocked(ctx context.Context) { defer q.limiter.Release() } + // Sort entries by key to ensure consistent lock ordering and avoid deadlocks + slices.SortFunc(entries, func(a, b *Entry[K, T]) int { + return cmp.Compare(a.Key, b.Key) + }) + // Execute batch write start := time.Now() data := make([]T, len(entries)) @@ -299,7 +312,19 @@ func (q *TypedQueue[K, T]) flushBatchLocked(ctx context.Context) { data[i] = entry.Data } - err := q.flushFunc(ctx, q.db, data) + var err error + for attempt := 0; attempt <= deadlockRetries; attempt++ { + err = q.flushFunc(ctx, q.db, data) + if err == nil { + break + } + if mysqlErr, ok := err.(*mysql.MySQLError); ok && mysqlErr.Number == mysqlDeadlock && attempt < deadlockRetries { + log.Warnf("Write-behind [%s] deadlock on attempt %d/%d (%d entries), retrying...", q.name, attempt+1, deadlockRetries, len(entries)) + time.Sleep(time.Duration(50*(attempt+1)) * time.Millisecond) + continue + } + break + } batchTime := time.Since(start).Seconds() entryCount := len(entries) From 92f6a4beb49e8e2559e62768deebd95e45a6f475 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 8 Feb 2026 16:10:04 +0000 Subject: [PATCH 69/78] Improve GetAvailablePokemon performance --- decoder/api_pokemon.go | 33 +++++++++------------------------ decoder/pokemonRtree.go | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/decoder/api_pokemon.go b/decoder/api_pokemon.go index 6a414eb6..d9843b83 100644 --- a/decoder/api_pokemon.go +++ b/decoder/api_pokemon.go @@ -22,32 +22,17 @@ type ApiPokemonAvailableResult struct { } func GetAvailablePokemon() []*ApiPokemonAvailableResult { - type pokemonFormKey struct { - pokemonId int16 - form int16 - } - - start := time.Now() - - pkmnMap := make(map[pokemonFormKey]int) - pokemonLookupCache.Range(func(key uint64, pokemon PokemonLookupCacheItem) bool { - pkmnMap[pokemonFormKey{pokemon.PokemonLookup.PokemonId, pokemon.PokemonLookup.Form}]++ - return true - }) - var available []*ApiPokemonAvailableResult - for key, count := range pkmnMap { - - pkmn := &ApiPokemonAvailableResult{ - PokemonId: key.pokemonId, - Form: key.form, - Count: count, + pokemonFormCount.Range(func(key pokemonFormKey, count int64) bool { + if count > 0 { + available = append(available, &ApiPokemonAvailableResult{ + PokemonId: key.pokemonId, + Form: key.form, + Count: int(count), + }) } - available = append(available, pkmn) - } - - log.Infof("GetAvailablePokemon - total time %s (locked time --)", time.Since(start)) - + return true + }) return available } diff --git a/decoder/pokemonRtree.go b/decoder/pokemonRtree.go index 20a18d51..121541f1 100644 --- a/decoder/pokemonRtree.go +++ b/decoder/pokemonRtree.go @@ -43,12 +43,26 @@ type PokemonPvpLookup struct { Ultra int16 } +type pokemonFormKey struct { + pokemonId int16 + form int16 +} + var pokemonLookupCache *xsync.MapOf[uint64, PokemonLookupCacheItem] +var pokemonFormCount *xsync.MapOf[pokemonFormKey, int64] var pokemonTreeMutex sync.RWMutex var pokemonTree rtree.RTreeG[uint64] +func adjustPokemonFormCount(key pokemonFormKey, delta int64) { + pokemonFormCount.Compute(key, func(oldValue int64, loaded bool) (int64, bool) { + newValue := oldValue + delta + return newValue, newValue <= 0 // delete entry when count reaches zero + }) +} + func initPokemonRtree() { pokemonLookupCache = xsync.NewMapOf[uint64, PokemonLookupCacheItem]() + pokemonFormCount = xsync.NewMapOf[pokemonFormKey, int64]() // Set up OnEviction callback on all shards pokemonCache.OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, *Pokemon]) { @@ -80,7 +94,13 @@ func valueOrMinus1(n null.Int) int { func updatePokemonLookup(pokemon *Pokemon, changePvp bool, pvpResults map[string][]gohbem.PokemonEntry) { pokemonId := uint64(pokemon.Id) - pokemonLookupCacheItem, _ := pokemonLookupCache.Load(pokemonId) + pokemonLookupCacheItem, existed := pokemonLookupCache.Load(pokemonId) + + // Track old form key so we can adjust counts + var oldKey pokemonFormKey + if existed && pokemonLookupCacheItem.PokemonLookup != nil { + oldKey = pokemonFormKey{pokemonLookupCacheItem.PokemonLookup.PokemonId, pokemonLookupCacheItem.PokemonLookup.Form} + } pokemonLookupCacheItem.PokemonLookup = &PokemonLookup{ PokemonId: pokemon.PokemonId, @@ -109,6 +129,15 @@ func updatePokemonLookup(pokemon *Pokemon, changePvp bool, pvpResults map[string } pokemonLookupCache.Store(pokemonId, pokemonLookupCacheItem) + + // Update form counts + newKey := pokemonFormKey{pokemonLookupCacheItem.PokemonLookup.PokemonId, pokemonLookupCacheItem.PokemonLookup.Form} + if existed && oldKey != newKey { + adjustPokemonFormCount(oldKey, -1) + } + if !existed || oldKey != newKey { + adjustPokemonFormCount(newKey, 1) + } } func calculatePokemonPvpLookup(pokemon *Pokemon, pvpResults map[string][]gohbem.PokemonEntry) *PokemonPvpLookup { @@ -164,7 +193,9 @@ func removePokemonFromTree(pokemonId uint64, lat, lon float64) { pokemonTree.Delete([2]float64{lon, lat}, [2]float64{lon, lat}, pokemonId) afterLen := pokemonTree.Len() pokemonTreeMutex.Unlock() - pokemonLookupCache.Delete(pokemonId) + if item, ok := pokemonLookupCache.LoadAndDelete(pokemonId); ok && item.PokemonLookup != nil { + adjustPokemonFormCount(pokemonFormKey{item.PokemonLookup.PokemonId, item.PokemonLookup.Form}, -1) + } if beforeLen != afterLen+1 { log.Infof("PokemonRtree - UNEXPECTED removing %d, lat %f lon %f size %d->%d Map Len %d", pokemonId, lat, lon, beforeLen, afterLen, pokemonLookupCache.Size()) From 2837b15c0f0db2191966c9cfd32dbc765a032cb5 Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 9 Feb 2026 14:44:56 +0000 Subject: [PATCH 70/78] Fort rtree improvements --- decoder/api_fort.go | 416 ++++++++++++++++++++++---------------- decoder/api_station.go | 55 +++++ decoder/fort.go | 9 +- decoder/fortRtree.go | 212 ++++++++++++------- decoder/incident_state.go | 11 + decoder/main.go | 23 ++- decoder/preload.go | 28 ++- decoder/station_state.go | 10 + main.go | 2 + routes.go | 36 ++++ 10 files changed, 539 insertions(+), 263 deletions(-) create mode 100644 decoder/api_station.go diff --git a/decoder/api_fort.go b/decoder/api_fort.go index 289fcc29..dedec767 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -23,41 +23,33 @@ type ApiFortDnfFilter struct { PowerUpLevel *ApiFortDnfMinMax8 `json:"power_up_level"` IsArScanEligible *bool `json:"is_ar_scan_eligible"` + // Gym AvailableSlots *ApiFortDnfMinMax8 `json:"available_slots"` TeamId []int8 `json:"team_id"` - InBattle bool `json:"in_battle"` RaidLevel []int8 `json:"raid_level"` RaidPokemon []ApiDnfId `json:"raid_pokemon_id"` - LureId []int16 `json:"lure_id"` - - ArQuestRewardType []int16 `json:"ar_quest_reward_type"` - ArQuestRewardAmount *ApiFortDnfMinMax16 `json:"ar_quest_reward_amount"` - ArQuestRewardItemId []int16 `json:"ar_quest_reward_item_id"` - ArQuestRewardPokemon []ApiDnfId `json:"ar_quest_reward_pokemon"` - ArQuestType []int16 `json:"ar_quest_type"` - ArQuestTarget []int16 `json:"ar_quest_target"` - ArQuestTemplate []string `json:"ar_quest_template"` - - NoArQuestRewardType []int16 `json:"noar_quest_reward_type"` - NoArQuestRewardAmount *ApiFortDnfMinMax16 `json:"noar_quest_reward_amount"` - NoArQuestRewardItemId []int16 `json:"noar_quest_reward_item_id"` - NoArQuestRewardPokemon []ApiDnfId `json:"noar_quest_reward_pokemon"` - NoArQuestType []int16 `json:"noar_quest_type"` - NoArQuestTarget []int16 `json:"noar_quest_target"` - NoArQuestTemplate []string `json:"noar_quest_template"` - - IncidentDisplayType []int8 `json:"incident_display_type"` - IncidentStyle []int8 `json:"incident_style"` - IncidentCharacter []int16 `json:"incident_character"` - IncidentSlot1 []ApiDnfId `json:"incident_slot_1"` - IncidentSlot2 []ApiDnfId `json:"incident_slot_2"` - IncidentSlot3 []ApiDnfId `json:"incident_slot_3"` - ContestPokemon []ApiDnfId `json:"contest_pokemon"` - ContestPokemonType1 []int8 `json:"contest_pokemon_type_1"` - ContestPokemonType2 []int8 `json:"contest_pokemon_type_2"` - ContestRankingStandard []int8 `json:"contest_ranking_standard"` - ContestTotalEntries *ApiFortDnfMinMax16 `json:"contest_total_entries"` + // Pokestop - unified quest (matches AR or no-AR) + LureId []int16 `json:"lure_id"` + QuestRewardType []int16 `json:"quest_reward_type"` + QuestRewardAmount *ApiFortDnfMinMax16 `json:"quest_reward_amount"` + QuestRewardItemId []int16 `json:"quest_reward_item_id"` + QuestRewardPokemon []ApiDnfId `json:"quest_reward_pokemon"` + + // Pokestop - incident + IncidentDisplayType []int8 `json:"incident_display_type"` + IncidentStyle []int8 `json:"incident_style"` + IncidentCharacter []int16 `json:"incident_character"` + IncidentPokemon []ApiDnfId `json:"incident_pokemon"` + + // Pokestop - contest + ContestPokemon []ApiDnfId `json:"contest_pokemon"` + ContestPokemonType []int8 `json:"contest_pokemon_type"` + ContestTotalEntries *ApiFortDnfMinMax16 `json:"contest_total_entries"` + + // Station + BattleLevel []int8 `json:"battle_level"` + BattlePokemon []ApiDnfId `json:"battle_pokemon"` } type ApiDnfId struct { @@ -89,8 +81,35 @@ type ApiPokestopScanResult struct { Total int `json:"total"` } -func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDnfFilter) bool { - if fortType != fortLookup.FortType { +type ApiStationScanResult struct { + Stations []*ApiStationResult `json:"stations"` + Examined int `json:"examined"` + Skipped int `json:"skipped"` + Total int `json:"total"` +} + +type ApiFortCombinedScanResult struct { + Gyms []*ApiGymResult `json:"gyms"` + Pokestops []*ApiPokestopResult `json:"pokestops"` + Stations []*ApiStationResult `json:"stations"` + Examined int `json:"examined"` + Skipped int `json:"skipped"` + Total int `json:"total"` +} + +// matchDnfIdPair checks if any ApiDnfId in the filter matches the given pokemon/form pair +func matchDnfIdPair(filter []ApiDnfId, pokemonId int16, form int16) bool { + for _, f := range filter { + if f.Pokemon == pokemonId && (f.Form == nil || *f.Form == form) { + return true + } + } + return false +} + +func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDnfFilter, now int64) bool { + // fortType 0 means "match any type" (used by combined scan) + if fortType != 0 && fortType != fortLookup.FortType { return false } if filter.PowerUpLevel != nil && (fortLookup.PowerUpLevel < filter.PowerUpLevel.Min || fortLookup.PowerUpLevel > filter.PowerUpLevel.Max) { @@ -107,21 +126,16 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn if filter.TeamId != nil && !slices.Contains(filter.TeamId, fortLookup.TeamId) { return false } - if filter.InBattle && !fortLookup.InBattle { - return false - } - if filter.RaidLevel != nil && !slices.Contains(filter.RaidLevel, fortLookup.RaidLevel) { - return false - } - if filter.RaidPokemon != nil { - raidPokemonMatch := false - for _, raidPokemon := range filter.RaidPokemon { - if raidPokemon.Pokemon == fortLookup.RaidPokemonId && (raidPokemon.Form == nil || *raidPokemon.Form == fortLookup.RaidPokemonForm) { - raidPokemonMatch = true - break - } + if filter.RaidLevel != nil || filter.RaidPokemon != nil { + // Check if raid has expired + raidActive := fortLookup.RaidBattleTimestamp > now || fortLookup.RaidEndTimestamp > now + if !raidActive { + return false + } + if filter.RaidLevel != nil && !slices.Contains(filter.RaidLevel, fortLookup.RaidLevel) { + return false } - if !raidPokemonMatch { + if filter.RaidPokemon != nil && !matchDnfIdPair(filter.RaidPokemon, fortLookup.RaidPokemonId, fortLookup.RaidPokemonForm) { return false } } @@ -129,152 +143,69 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn if filter.LureId != nil && !slices.Contains(filter.LureId, fortLookup.LureId) { return false } - // AR Quest Filters - if filter.ArQuestRewardType != nil && !slices.Contains(filter.ArQuestRewardType, fortLookup.QuestArRewardType) { - return false - } - if filter.ArQuestRewardAmount != nil && (fortLookup.QuestArRewardAmount < filter.ArQuestRewardAmount.Min || fortLookup.QuestArRewardAmount > filter.ArQuestRewardAmount.Max) { - return false - } - if filter.ArQuestRewardItemId != nil && !slices.Contains(filter.ArQuestRewardItemId, fortLookup.QuestArRewardItemId) { - return false - } - if filter.ArQuestType != nil && !slices.Contains(filter.ArQuestType, fortLookup.QuestArType) { - return false - } - if filter.ArQuestTarget != nil && !slices.Contains(filter.ArQuestTarget, fortLookup.QuestArTarget) { - return false + + // Unified quest filters - match if AR or no-AR value matches + if filter.QuestRewardType != nil { + if !slices.Contains(filter.QuestRewardType, fortLookup.QuestArRewardType) && + !slices.Contains(filter.QuestRewardType, fortLookup.QuestNoArRewardType) { + return false + } } - if filter.ArQuestTemplate != nil && !slices.Contains(filter.ArQuestTemplate, fortLookup.QuestArTemplate) { - return false + if filter.QuestRewardAmount != nil { + arMatch := fortLookup.QuestArRewardAmount >= filter.QuestRewardAmount.Min && fortLookup.QuestArRewardAmount <= filter.QuestRewardAmount.Max + noArMatch := fortLookup.QuestNoArRewardAmount >= filter.QuestRewardAmount.Min && fortLookup.QuestNoArRewardAmount <= filter.QuestRewardAmount.Max + if !arMatch && !noArMatch { + return false + } } - if filter.ArQuestRewardPokemon != nil { - match := false - for _, pkm := range filter.ArQuestRewardPokemon { - if pkm.Pokemon == fortLookup.QuestArRewardPokemonId && (pkm.Form == nil || *pkm.Form == fortLookup.QuestArRewardPokemonForm) { - match = true - break - } + if filter.QuestRewardItemId != nil { + if !slices.Contains(filter.QuestRewardItemId, fortLookup.QuestArRewardItemId) && + !slices.Contains(filter.QuestRewardItemId, fortLookup.QuestNoArRewardItemId) { + return false } - if !match { + } + if filter.QuestRewardPokemon != nil { + arMatch := matchDnfIdPair(filter.QuestRewardPokemon, fortLookup.QuestArRewardPokemonId, fortLookup.QuestArRewardPokemonForm) + noArMatch := matchDnfIdPair(filter.QuestRewardPokemon, fortLookup.QuestNoArRewardPokemonId, fortLookup.QuestNoArRewardPokemonForm) + if !arMatch && !noArMatch { return false } } - // No-AR Quest Filters - if filter.NoArQuestRewardType != nil && !slices.Contains(filter.NoArQuestRewardType, fortLookup.QuestNoArRewardType) { + // Contest filters + if filter.ContestPokemonType != nil && !slices.Contains(filter.ContestPokemonType, fortLookup.ContestPokemonType) { return false } - if filter.NoArQuestRewardAmount != nil && (fortLookup.QuestNoArRewardAmount < filter.NoArQuestRewardAmount.Min || fortLookup.QuestNoArRewardAmount > filter.NoArQuestRewardAmount.Max) { - return false - } - if filter.NoArQuestRewardItemId != nil && !slices.Contains(filter.NoArQuestRewardItemId, fortLookup.QuestNoArRewardItemId) { - return false - } - if filter.NoArQuestType != nil && !slices.Contains(filter.NoArQuestType, fortLookup.QuestNoArType) { - return false - } - if filter.NoArQuestTarget != nil && !slices.Contains(filter.NoArQuestTarget, fortLookup.QuestNoArTarget) { + if filter.ContestTotalEntries != nil && (fortLookup.ContestTotalEntries < filter.ContestTotalEntries.Min || fortLookup.ContestTotalEntries > filter.ContestTotalEntries.Max) { return false } - if filter.NoArQuestTemplate != nil && !slices.Contains(filter.NoArQuestTemplate, fortLookup.QuestNoArTemplate) { + if filter.ContestPokemon != nil && !matchDnfIdPair(filter.ContestPokemon, fortLookup.ContestPokemonId, fortLookup.ContestPokemonForm) { return false } - if filter.NoArQuestRewardPokemon != nil { - match := false - for _, pkm := range filter.NoArQuestRewardPokemon { - if pkm.Pokemon == fortLookup.QuestNoArRewardPokemonId && (pkm.Form == nil || *pkm.Form == fortLookup.QuestNoArRewardPokemonForm) { - match = true - break - } - } - if !match { - return false - } - } - // Contest Filters - if filter.ContestPokemonType1 != nil && !slices.Contains(filter.ContestPokemonType1, fortLookup.ContestPokemonType1) { + // Incident filters - flat field checks + if filter.IncidentDisplayType != nil && !slices.Contains(filter.IncidentDisplayType, fortLookup.IncidentDisplayType) { return false } - if filter.ContestPokemonType2 != nil && !slices.Contains(filter.ContestPokemonType2, fortLookup.ContestPokemonType2) { + if filter.IncidentStyle != nil && !slices.Contains(filter.IncidentStyle, fortLookup.IncidentStyle) { return false } - if filter.ContestRankingStandard != nil && !slices.Contains(filter.ContestRankingStandard, fortLookup.ContestRankingStandard) { + if filter.IncidentCharacter != nil && !slices.Contains(filter.IncidentCharacter, fortLookup.IncidentCharacter) { return false } - if filter.ContestTotalEntries != nil && (fortLookup.ContestTotalEntries < filter.ContestTotalEntries.Min || fortLookup.ContestTotalEntries > filter.ContestTotalEntries.Max) { + if filter.IncidentPokemon != nil && !matchDnfIdPair(filter.IncidentPokemon, fortLookup.IncidentPokemonId, fortLookup.IncidentPokemonForm) { return false } - if filter.ContestPokemon != nil { - match := false - for _, pkm := range filter.ContestPokemon { - if pkm.Pokemon == fortLookup.ContestPokemonId && (pkm.Form == nil || *pkm.Form == fortLookup.ContestPokemonForm) { - match = true - break - } - } - if !match { + } else if fortLookup.FortType == STATION { + if filter.BattleLevel != nil || filter.BattlePokemon != nil { + // Check if battle has expired + if fortLookup.BattleEndTimestamp <= now { return false } - } - - // Incident Filters - if len(filter.IncidentDisplayType) > 0 || len(filter.IncidentStyle) > 0 || len(filter.IncidentCharacter) > 0 || len(filter.IncidentSlot1) > 0 || len(filter.IncidentSlot2) > 0 || len(filter.IncidentSlot3) > 0 { - incidentMatch := false - for _, incident := range fortLookup.Incidents { - incidentFilterMatch := true - if filter.IncidentDisplayType != nil && !slices.Contains(filter.IncidentDisplayType, incident.DisplayType) { - incidentFilterMatch = false - } - if incidentFilterMatch && filter.IncidentStyle != nil && !slices.Contains(filter.IncidentStyle, incident.Style) { - incidentFilterMatch = false - } - if incidentFilterMatch && filter.IncidentCharacter != nil && !slices.Contains(filter.IncidentCharacter, incident.Character) { - incidentFilterMatch = false - } - if incidentFilterMatch && filter.IncidentSlot1 != nil { - slotMatch := false - for _, pkm := range filter.IncidentSlot1 { - if pkm.Pokemon == incident.Slot1PokemonId && (pkm.Form == nil || *pkm.Form == incident.Slot1PokemonForm) { - slotMatch = true - break - } - } - if !slotMatch { - incidentFilterMatch = false - } - } - if incidentFilterMatch && filter.IncidentSlot2 != nil { - slotMatch := false - for _, pkm := range filter.IncidentSlot2 { - if pkm.Pokemon == incident.Slot2PokemonId && (pkm.Form == nil || *pkm.Form == incident.Slot2PokemonForm) { - slotMatch = true - break - } - } - if !slotMatch { - incidentFilterMatch = false - } - } - if incidentFilterMatch && filter.IncidentSlot3 != nil { - slotMatch := false - for _, pkm := range filter.IncidentSlot3 { - if pkm.Pokemon == incident.Slot3PokemonId && (pkm.Form == nil || *pkm.Form == incident.Slot3PokemonForm) { - slotMatch = true - break - } - } - if !slotMatch { - incidentFilterMatch = false - } - } - if incidentFilterMatch { - incidentMatch = true - break - } + if filter.BattleLevel != nil && !slices.Contains(filter.BattleLevel, fortLookup.BattleLevel) { + return false } - if !incidentMatch { + if filter.BattlePokemon != nil && !matchDnfIdPair(filter.BattlePokemon, fortLookup.BattlePokemonId, fortLookup.BattlePokemonForm) { return false } } @@ -295,6 +226,7 @@ func internalGetForts(fortType FortType, retrieveParameters ApiFortScan) ([]stri fortsExamined := 0 fortsSkipped := 0 + now := time.Now().Unix() fortTreeMutex.RLock() fortTreeCopy := fortTree.Copy() @@ -319,7 +251,7 @@ func internalGetForts(fortType FortType, retrieveParameters ApiFortScan) ([]stri matched = fortType == fortLookup.FortType } else { for i := 0; i < len(retrieveParameters.DnfFilters); i++ { - if isFortDnfMatch(fortType, &fortLookup, &retrieveParameters.DnfFilters[i]) { + if isFortDnfMatch(fortType, &fortLookup, &retrieveParameters.DnfFilters[i], now) { matched = true break } @@ -351,7 +283,6 @@ func GymScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *Ap for _, key := range returnKeys { gym, unlock, err := GetGymRecordReadOnly(context.Background(), dbDetails, key) if err == nil && gym != nil { - // Make a copy to avoid holding locks gymCopy := buildGymResult(gym) results = append(results, &gymCopy) } @@ -377,7 +308,6 @@ func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails for _, key := range returnKeys { pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key) if err == nil && pokestop != nil { - // Make a copy to avoid holding locks pokestopCopy := buildPokestopResult(pokestop) results = append(results, &pokestopCopy) } @@ -394,3 +324,149 @@ func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails Total: total, } } + +func StationScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiStationScanResult { + returnKeys, examined, skipped, total := internalGetForts(STATION, retrieveParameters) + results := make([]*ApiStationResult, 0, len(returnKeys)) + start := time.Now() + + for _, key := range returnKeys { + station, unlock, err := getStationRecordReadOnly(context.Background(), dbDetails, key) + if err == nil && station != nil { + stationCopy := buildStationResult(station) + results = append(results, &stationCopy) + } + if unlock != nil { + unlock() + } + } + log.Infof("StationScan - result buffer time %s, %d added", time.Since(start), len(results)) + + return &ApiStationScanResult{ + Stations: results, + Examined: examined, + Skipped: skipped, + Total: total, + } +} + +func FortCombinedScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiFortCombinedScanResult { + gymKeys, pokestopKeys, stationKeys, examined, skipped, total := internalGetFortsCombined(retrieveParameters) + start := time.Now() + + gyms := make([]*ApiGymResult, 0, len(gymKeys)) + for _, key := range gymKeys { + gym, unlock, err := GetGymRecordReadOnly(context.Background(), dbDetails, key) + if err == nil && gym != nil { + gymCopy := buildGymResult(gym) + gyms = append(gyms, &gymCopy) + } + if unlock != nil { + unlock() + } + } + + pokestops := make([]*ApiPokestopResult, 0, len(pokestopKeys)) + for _, key := range pokestopKeys { + pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key) + if err == nil && pokestop != nil { + pokestopCopy := buildPokestopResult(pokestop) + pokestops = append(pokestops, &pokestopCopy) + } + if unlock != nil { + unlock() + } + } + + stations := make([]*ApiStationResult, 0, len(stationKeys)) + for _, key := range stationKeys { + station, unlock, err := getStationRecordReadOnly(context.Background(), dbDetails, key) + if err == nil && station != nil { + stationCopy := buildStationResult(station) + stations = append(stations, &stationCopy) + } + if unlock != nil { + unlock() + } + } + + log.Infof("FortCombinedScan - result buffer time %s, %d+%d+%d added", + time.Since(start), len(gyms), len(pokestops), len(stations)) + + return &ApiFortCombinedScanResult{ + Gyms: gyms, + Pokestops: pokestops, + Stations: stations, + Examined: examined, + Skipped: skipped, + Total: total, + } +} + +func internalGetFortsCombined(retrieveParameters ApiFortScan) (gymKeys, pokestopKeys, stationKeys []string, examined, skipped, total int) { + start := time.Now() + + minLocation := retrieveParameters.Min + maxLocation := retrieveParameters.Max + + maxForts := config.Config.Tuning.MaxPokemonResults + if retrieveParameters.Limit > 0 && retrieveParameters.Limit < maxForts { + maxForts = retrieveParameters.Limit + } + + now := time.Now().Unix() + totalMatched := 0 + + fortTreeMutex.RLock() + fortTreeCopy := fortTree.Copy() + fortTreeMutex.RUnlock() + + lockedTime := time.Since(start) + + fortTreeCopy.Search([2]float64{minLocation.Longitude, minLocation.Latitude}, [2]float64{maxLocation.Longitude, maxLocation.Latitude}, + func(min, max [2]float64, fortId string) bool { + examined++ + + fortLookup, found := fortLookupCache.Load(fortId) + if !found { + skipped++ + return true + } + + matched := false + if len(retrieveParameters.DnfFilters) == 0 { + matched = true + } else { + for i := range retrieveParameters.DnfFilters { + if isFortDnfMatch(0, &fortLookup, &retrieveParameters.DnfFilters[i], now) { + matched = true + break + } + } + } + + if matched { + switch fortLookup.FortType { + case GYM: + gymKeys = append(gymKeys, fortId) + case POKESTOP: + pokestopKeys = append(pokestopKeys, fortId) + case STATION: + stationKeys = append(stationKeys, fortId) + } + totalMatched++ + if totalMatched >= maxForts { + log.Infof("GetFortsInArea - result would exceed maximum size (%d), stopping scan", maxForts) + return false + } + } + + return true + }) + + log.Infof("GetFortsInArea (combined) - scan time %s (locked time %s), %d scanned, %d skipped, %d+%d+%d returned, tree size %d", + time.Since(start), lockedTime, examined, skipped, len(gymKeys), len(pokestopKeys), len(stationKeys), fortTreeCopy.Len()) + + total = fortTreeCopy.Len() + return +} diff --git a/decoder/api_station.go b/decoder/api_station.go new file mode 100644 index 00000000..c581d9e6 --- /dev/null +++ b/decoder/api_station.go @@ -0,0 +1,55 @@ +package decoder + +import "github.com/guregu/null/v6" + +type ApiStationResult struct { + Id string `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + Name string `json:"name"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + IsBattleAvailable bool `json:"is_battle_available"` + Updated int64 `json:"updated"` + BattleLevel null.Int `json:"battle_level"` + BattleStart null.Int `json:"battle_start"` + BattleEnd null.Int `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` + TotalStationedGmax null.Int `json:"total_stationed_gmax"` + StationedPokemon null.String `json:"stationed_pokemon"` +} + +func buildStationResult(station *Station) ApiStationResult { + return ApiStationResult{ + Id: station.Id, + Lat: station.Lat, + Lon: station.Lon, + Name: station.Name, + StartTime: station.StartTime, + EndTime: station.EndTime, + IsBattleAvailable: station.IsBattleAvailable, + Updated: station.Updated, + BattleLevel: station.BattleLevel, + BattleStart: station.BattleStart, + BattleEnd: station.BattleEnd, + BattlePokemonId: station.BattlePokemonId, + BattlePokemonForm: station.BattlePokemonForm, + BattlePokemonCostume: station.BattlePokemonCostume, + BattlePokemonGender: station.BattlePokemonGender, + BattlePokemonAlignment: station.BattlePokemonAlignment, + BattlePokemonBreadMode: station.BattlePokemonBreadMode, + BattlePokemonMove1: station.BattlePokemonMove1, + BattlePokemonMove2: station.BattlePokemonMove2, + TotalStationedPokemon: station.TotalStationedPokemon, + TotalStationedGmax: station.TotalStationedGmax, + StationedPokemon: station.StationedPokemon, + } +} diff --git a/decoder/fort.go b/decoder/fort.go index 17e20212..09f596dc 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -36,7 +36,7 @@ type FortChangeWebhook struct { } type FortChange string -type FortType string +type FortType int8 func (f FortType) String() string { switch f { @@ -44,6 +44,8 @@ func (f FortType) String() string { return "pokestop" case GYM: return "gym" + case STATION: + return "station" } return "unknown" } @@ -65,8 +67,9 @@ const ( REMOVAL FortChange = "removal" EDIT FortChange = "edit" - POKESTOP FortType = "pokestop" - GYM FortType = "gym" + POKESTOP FortType = iota + 1 + GYM + STATION ) func InitWebHookFortFromGym(gym *Gym) *FortWebhook { diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 0c17f6cd..7fb0f44b 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -10,58 +10,53 @@ import ( "github.com/tidwall/rtree" ) -type IncidentLookup struct { - DisplayType int8 - Style int8 - Character int16 - Slot1PokemonId int16 - Slot1PokemonForm int16 - Slot2PokemonId int16 - Slot2PokemonForm int16 - Slot3PokemonId int16 - Slot3PokemonForm int16 -} - type FortLookup struct { FortType FortType - PowerUpLevel int8 - IsArScanEligible bool Lat float64 Lon float64 + PowerUpLevel int8 + IsArScanEligible bool - // Gym-specific fields - AvailableSlots int8 - TeamId int8 - InBattle bool - RaidLevel int8 - RaidPokemonId int16 - RaidPokemonForm int16 + // Gym + AvailableSlots int8 + TeamId int8 + RaidEndTimestamp int64 // used to check expiry at filter time + RaidBattleTimestamp int64 + RaidLevel int8 + RaidPokemonId int16 + RaidPokemonForm int16 - // Pokestop-specific fields + // Pokestop - quest rewards only (both AR and no-AR stored, filter matches either) LureId int16 QuestNoArRewardType int16 QuestNoArRewardAmount int16 QuestNoArRewardItemId int16 QuestNoArRewardPokemonId int16 QuestNoArRewardPokemonForm int16 - QuestNoArType int16 - QuestNoArTarget int16 - QuestNoArTemplate string QuestArRewardType int16 QuestArRewardAmount int16 QuestArRewardItemId int16 QuestArRewardPokemonId int16 QuestArRewardPokemonForm int16 - QuestArType int16 - QuestArTarget int16 - QuestArTemplate string - Incidents []*IncidentLookup - ContestPokemonId int16 - ContestPokemonForm int16 - ContestPokemonType1 int8 - ContestPokemonType2 int8 - ContestRankingStandard int8 - ContestTotalEntries int16 + + // Pokestop - incident (first active incident, flat fields) + IncidentDisplayType int8 + IncidentStyle int8 + IncidentCharacter int16 + IncidentPokemonId int16 + IncidentPokemonForm int16 + + // Pokestop - contest + ContestPokemonId int16 + ContestPokemonForm int16 + ContestPokemonType int8 + ContestTotalEntries int16 + + // Station + BattleEndTimestamp int64 // used to check expiry at filter time + BattleLevel int8 + BattlePokemonId int16 + BattlePokemonForm int16 } var fortLookupCache *xsync.MapOf[string, FortLookup] @@ -76,7 +71,7 @@ type IdRecord struct { Id string `db:"id"` } -// genericUpdateFort handles rtree updates for fort location changes and deletions +// genericUpdateFort handles rtree updates for fort location changes and deletions. func genericUpdateFort(id string, lat float64, lon float64, deleted bool) { oldFort, inMap := fortLookupCache.Load(id) @@ -99,16 +94,26 @@ func genericUpdateFort(id string, lat float64, lon float64, deleted bool) { // fortRtreeUpdatePokestopOnSave updates rtree and lookup cache when a pokestop is saved func fortRtreeUpdatePokestopOnSave(pokestop *Pokestop) { genericUpdateFort(pokestop.Id, pokestop.Lat, pokestop.Lon, pokestop.Deleted) - updatePokestopLookup(pokestop) + if !pokestop.Deleted { + updatePokestopLookup(pokestop) + } } // fortRtreeUpdateGymOnSave updates rtree and lookup cache when a gym is saved func fortRtreeUpdateGymOnSave(gym *Gym) { genericUpdateFort(gym.Id, gym.Lat, gym.Lon, gym.Deleted) - updateGymLookup(gym) + if !gym.Deleted { + updateGymLookup(gym) + } +} + +// fortRtreeUpdateStationOnSave updates rtree and lookup cache when a station is saved +func fortRtreeUpdateStationOnSave(station *Station) { + genericUpdateFort(station.Id, station.Lat, station.Lon, false) + updateStationLookup(station) } -// fortRtreeUpdatePokestopOnGet updates rtree when a pokestop is loaded from DB (legacy pattern) +// fortRtreeUpdatePokestopOnGet updates rtree when a pokestop is loaded from DB (cache miss) func fortRtreeUpdatePokestopOnGet(pokestop *Pokestop) { _, inMap := fortLookupCache.Load(pokestop.Id) if !inMap { @@ -117,7 +122,7 @@ func fortRtreeUpdatePokestopOnGet(pokestop *Pokestop) { } } -// fortRtreeUpdateGymOnGet updates rtree when a gym is loaded from DB (legacy pattern) +// fortRtreeUpdateGymOnGet updates rtree when a gym is loaded from DB (cache miss) func fortRtreeUpdateGymOnGet(gym *Gym) { _, inMap := fortLookupCache.Load(gym.Id) if !inMap { @@ -126,79 +131,132 @@ func fortRtreeUpdateGymOnGet(gym *Gym) { } } -// getContestTotalEntries parses showcase rankings JSON to get total entries -func getContestTotalEntries(rankingsString null.String) int16 { - if !rankingsString.Valid { - return -1 - } - - type contestJson struct { - TotalEntries int `json:"total_entries"` - } - var cj contestJson - if json.Unmarshal([]byte(rankingsString.String), &cj) == nil { - return int16(cj.TotalEntries) +// fortRtreeUpdateStationOnGet updates rtree when a station is loaded from DB (cache miss) +func fortRtreeUpdateStationOnGet(station *Station) { + _, inMap := fortLookupCache.Load(station.Id) + if !inMap { + addFortToTree(station.Id, station.Lat, station.Lon) + updateStationLookup(station) } - return -1 } func updatePokestopLookup(pokestop *Pokestop) { - contestTotalEntries := getContestTotalEntries(pokestop.ShowcaseRankings) + // Preserve existing incident fields if present + var incidentDisplayType int8 + var incidentStyle int8 + var incidentCharacter int16 + var incidentPokemonId int16 + var incidentPokemonForm int16 + if existing, ok := fortLookupCache.Load(pokestop.Id); ok { + incidentDisplayType = existing.IncidentDisplayType + incidentStyle = existing.IncidentStyle + incidentCharacter = existing.IncidentCharacter + incidentPokemonId = existing.IncidentPokemonId + incidentPokemonForm = existing.IncidentPokemonForm + } fortLookupCache.Store(pokestop.Id, FortLookup{ FortType: POKESTOP, - PowerUpLevel: int8(valueOrMinus1(pokestop.PowerUpLevel)), - IsArScanEligible: pokestop.ArScanEligible.ValueOrZero() == 1, Lat: pokestop.Lat, Lon: pokestop.Lon, + PowerUpLevel: int8(valueOrMinus1(pokestop.PowerUpLevel)), + IsArScanEligible: pokestop.ArScanEligible.ValueOrZero() == 1, LureId: pokestop.LureId, QuestNoArRewardType: int16(pokestop.QuestRewardType.ValueOrZero()), QuestNoArRewardAmount: int16(pokestop.QuestRewardAmount.ValueOrZero()), QuestNoArRewardItemId: int16(pokestop.QuestItemId.ValueOrZero()), QuestNoArRewardPokemonId: int16(pokestop.QuestPokemonId.ValueOrZero()), QuestNoArRewardPokemonForm: int16(pokestop.QuestPokemonFormId.ValueOrZero()), - QuestNoArType: int16(valueOrMinus1(pokestop.QuestType)), - QuestNoArTarget: int16(valueOrMinus1(pokestop.QuestTarget)), - QuestNoArTemplate: pokestop.QuestTemplate.ValueOrZero(), QuestArRewardType: int16(pokestop.AlternativeQuestRewardType.ValueOrZero()), QuestArRewardAmount: int16(pokestop.AlternativeQuestRewardAmount.ValueOrZero()), QuestArRewardItemId: int16(pokestop.AlternativeQuestItemId.ValueOrZero()), QuestArRewardPokemonId: int16(pokestop.AlternativeQuestPokemonId.ValueOrZero()), QuestArRewardPokemonForm: int16(pokestop.AlternativeQuestPokemonFormId.ValueOrZero()), - QuestArType: int16(valueOrMinus1(pokestop.AlternativeQuestType)), - QuestArTarget: int16(valueOrMinus1(pokestop.AlternativeQuestTarget)), - QuestArTemplate: pokestop.AlternativeQuestTemplate.ValueOrZero(), + IncidentDisplayType: incidentDisplayType, + IncidentStyle: incidentStyle, + IncidentCharacter: incidentCharacter, + IncidentPokemonId: incidentPokemonId, + IncidentPokemonForm: incidentPokemonForm, ContestPokemonId: int16(pokestop.ShowcasePokemon.ValueOrZero()), ContestPokemonForm: int16(pokestop.ShowcasePokemonForm.ValueOrZero()), - ContestPokemonType1: int8(pokestop.ShowcasePokemonType.ValueOrZero()), - ContestPokemonType2: -1, // TODO: this should probably be saved in the db - ContestRankingStandard: int8(pokestop.ShowcaseRankingStandard.ValueOrZero()), - ContestTotalEntries: contestTotalEntries, + ContestPokemonType: int8(pokestop.ShowcasePokemonType.ValueOrZero()), + ContestTotalEntries: getContestTotalEntries(pokestop.ShowcaseRankings), }) } func updateGymLookup(gym *Gym) { fortLookupCache.Store(gym.Id, FortLookup{ - FortType: GYM, - PowerUpLevel: int8(valueOrMinus1(gym.PowerUpLevel)), - Lat: gym.Lat, - Lon: gym.Lon, - IsArScanEligible: gym.ArScanEligible.ValueOrZero() == 1, - AvailableSlots: int8(gym.AvailableSlots.ValueOrZero()), - TeamId: int8(gym.TeamId.ValueOrZero()), - InBattle: gym.InBattle.ValueOrZero() == 1, - RaidLevel: int8(gym.RaidLevel.ValueOrZero()), - RaidPokemonId: int16(gym.RaidPokemonId.ValueOrZero()), - RaidPokemonForm: int16(gym.RaidPokemonForm.ValueOrZero()), + FortType: GYM, + Lat: gym.Lat, + Lon: gym.Lon, + PowerUpLevel: int8(valueOrMinus1(gym.PowerUpLevel)), + IsArScanEligible: gym.ArScanEligible.ValueOrZero() == 1, + AvailableSlots: int8(gym.AvailableSlots.ValueOrZero()), + TeamId: int8(gym.TeamId.ValueOrZero()), + RaidEndTimestamp: gym.RaidEndTimestamp.ValueOrZero(), + RaidBattleTimestamp: gym.RaidBattleTimestamp.ValueOrZero(), + RaidLevel: int8(gym.RaidLevel.ValueOrZero()), + RaidPokemonId: int16(gym.RaidPokemonId.ValueOrZero()), + RaidPokemonForm: int16(gym.RaidPokemonForm.ValueOrZero()), + }) +} + +func updateStationLookup(station *Station) { + fortLookupCache.Store(station.Id, FortLookup{ + FortType: STATION, + Lat: station.Lat, + Lon: station.Lon, + BattleEndTimestamp: station.BattleEnd.ValueOrZero(), + BattleLevel: int8(station.BattleLevel.ValueOrZero()), + BattlePokemonId: int16(station.BattlePokemonId.ValueOrZero()), + BattlePokemonForm: int16(station.BattlePokemonForm.ValueOrZero()), }) } +// updatePokestopIncidentLookup updates the incident fields on a pokestop's FortLookup entry +func updatePokestopIncidentLookup(pokestopId string, incident *Incident) { + existing, ok := fortLookupCache.Load(pokestopId) + if !ok { + return + } + + existing.IncidentDisplayType = int8(incident.DisplayType) + existing.IncidentStyle = int8(incident.Style) + existing.IncidentCharacter = incident.Character + existing.IncidentPokemonId = int16(incident.Slot1PokemonId.ValueOrZero()) + existing.IncidentPokemonForm = int16(incident.Slot1Form.ValueOrZero()) + + fortLookupCache.Store(pokestopId, existing) +} + +// getContestTotalEntries parses showcase rankings JSON to get total entries +func getContestTotalEntries(rankingsString null.String) int16 { + if !rankingsString.Valid { + return -1 + } + + type contestJson struct { + TotalEntries int `json:"total_entries"` + } + var cj contestJson + if json.Unmarshal([]byte(rankingsString.String), &cj) == nil { + return int16(cj.TotalEntries) + } + return -1 +} + func addFortToTree(id string, lat float64, lon float64) { fortTreeMutex.Lock() fortTree.Insert([2]float64{lon, lat}, [2]float64{lon, lat}, id) fortTreeMutex.Unlock() } +// evictFortFromTree is called from cache eviction callbacks to clean up all fort state +func evictFortFromTree(fortId string, lat, lon float64) { + fortLookupCache.Delete(fortId) + removeFortFromTree(fortId, lat, lon) +} + func removeFortFromTree(fortId string, lat, lon float64) { fortTreeMutex.Lock() beforeLen := fortTree.Len() diff --git a/decoder/incident_state.go b/decoder/incident_state.go index 9d6766cd..f122bb2f 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -9,6 +9,7 @@ import ( "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" + "golbat/config" "golbat/db" "golbat/webhooks" ) @@ -59,6 +60,9 @@ func getIncidentRecordReadOnly(ctx context.Context, db db.DbDetails, incidentId // Atomically cache the loaded Incident - if another goroutine raced us, // we'll get their Incident and use that instead (ensuring same mutex) existingIncident, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + if config.Config.FortInMemory { + updatePokestopIncidentLookup(dbIncident.PokestopId, &dbIncident) + } return &dbIncident }) @@ -101,6 +105,9 @@ func getOrCreateIncidentRecord(ctx context.Context, db db.DbDetails, incidentId // We loaded from DB incident.newRecord = false incident.ClearDirty() + if config.Config.FortInMemory { + updatePokestopIncidentLookup(incident.PokestopId, incident) + } } } @@ -148,6 +155,10 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident areas := MatchStatsGeofence(stopLat, stopLon) updateIncidentStats(incident, areas) + if config.Config.FortInMemory { + updatePokestopIncidentLookup(incident.PokestopId, incident) + } + if dbDebugEnabled { incident.changedFields = incident.changedFields[:0] } diff --git a/decoder/main.go b/decoder/main.go index d163ddaf..33f1c6c2 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -91,35 +91,48 @@ func (cl *gohbemLogger) Print(message string) { func initDataCache() { // Sharded caches for high-concurrency tables + // When fort_in_memory is enabled, extend TTL to 25 hours so that the + // rtree stays populated between daily quest resets. + fortCacheTTL := 60 * time.Minute + if config.Config.FortInMemory { + fortCacheTTL = 25 * time.Hour + } + pokestopCache = NewShardedCache(ShardedCacheConfig[string, *Pokestop]{ NumShards: runtime.NumCPU(), - TTL: 60 * time.Minute, + TTL: fortCacheTTL, KeyToShard: StringKeyToShard, }) if config.Config.FortInMemory { pokestopCache.OnEviction(func(ctx context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *Pokestop]) { p := item.Value() - removeFortFromTree(p.Id, p.Lat, p.Lon) + evictFortFromTree(p.Id, p.Lat, p.Lon) }) } gymCache = NewShardedCache(ShardedCacheConfig[string, *Gym]{ NumShards: runtime.NumCPU(), - TTL: 60 * time.Minute, + TTL: fortCacheTTL, KeyToShard: StringKeyToShard, }) if config.Config.FortInMemory { gymCache.OnEviction(func(ctx context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *Gym]) { g := item.Value() - removeFortFromTree(g.Id, g.Lat, g.Lon) + evictFortFromTree(g.Id, g.Lat, g.Lon) }) } stationCache = NewShardedCache(ShardedCacheConfig[string, *Station]{ NumShards: runtime.NumCPU(), - TTL: 60 * time.Minute, + TTL: fortCacheTTL, KeyToShard: StringKeyToShard, }) + if config.Config.FortInMemory { + stationCache.OnEviction(func(ctx context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *Station]) { + s := item.Value() + evictFortFromTree(s.Id, s.Lat, s.Lon) + }) + } tappableCache = ttlcache.New[uint64, *Tappable]( ttlcache.WithTTL[uint64, *Tappable](60 * time.Minute), diff --git a/decoder/preload.go b/decoder/preload.go index b659acae..b14fe7f9 100644 --- a/decoder/preload.go +++ b/decoder/preload.go @@ -19,7 +19,10 @@ func Preload(dbDetails db.DbDetails, populateRtree bool) { var wg sync.WaitGroup var pokestopCount, gymCount, stationCount, incidentCount, spawnpointCount int32 - wg.Add(5) + // Phase 1: Load forts and spawnpoints in parallel. + // Forts must be loaded before incidents so that the fort lookup cache + // has entries for incidents to update. + wg.Add(4) go func() { defer wg.Done() pokestopCount = preloadPokestops(dbDetails, populateRtree) @@ -30,11 +33,7 @@ func Preload(dbDetails db.DbDetails, populateRtree bool) { }() go func() { defer wg.Done() - stationCount = preloadStations(dbDetails) - }() - go func() { - defer wg.Done() - incidentCount = preloadIncidents(dbDetails) + stationCount = preloadStations(dbDetails, populateRtree) }() go func() { defer wg.Done() @@ -42,6 +41,9 @@ func Preload(dbDetails db.DbDetails, populateRtree bool) { }() wg.Wait() + // Phase 2: Load incidents (needs pokestop lookup entries to exist) + incidentCount = preloadIncidents(dbDetails, populateRtree) + log.Infof("Preload: loaded %d pokestops, %d gyms, %d stations, %d incidents, %d spawnpoints in %v (rtree=%v)", pokestopCount, gymCount, stationCount, incidentCount, spawnpointCount, time.Since(startTime), populateRtree) } @@ -192,7 +194,7 @@ func preloadGyms(dbDetails db.DbDetails, populateRtree bool) int32 { return count } -func preloadStations(dbDetails db.DbDetails) int32 { +func preloadStations(dbDetails db.DbDetails, populateRtree bool) int32 { query := "SELECT " + stationSelectColumns + " FROM station" rows, err := dbDetails.GeneralDb.Queryx(query) if err != nil { @@ -214,6 +216,11 @@ func preloadStations(dbDetails db.DbDetails) int32 { // Add to cache stationCache.Set(station.Id, station, 0) // 0 = use default TTL + // Update rtree if enabled + if populateRtree { + fortRtreeUpdateStationOnSave(station) + } + c := atomic.AddInt32(&count, 1) if c%10000 == 0 { log.Infof("Preload: loaded %d stations...", c) @@ -237,7 +244,7 @@ func preloadStations(dbDetails db.DbDetails) int32 { return count } -func preloadIncidents(dbDetails db.DbDetails) int32 { +func preloadIncidents(dbDetails db.DbDetails, populateRtree bool) int32 { // Load active incidents (not yet expired) now := time.Now().Unix() query := "SELECT " + incidentSelectColumns + " FROM incident WHERE expiration > ?" @@ -261,6 +268,11 @@ func preloadIncidents(dbDetails db.DbDetails) int32 { // Add to cache incidentCache.Set(incident.Id, incident, 0) // 0 = use default TTL + // Update fort rtree with incident data + if populateRtree { + updatePokestopIncidentLookup(incident.PokestopId, incident) + } + c := atomic.AddInt32(&count, 1) if c%10000 == 0 { log.Infof("Preload: loaded %d incidents...", c) diff --git a/decoder/station_state.go b/decoder/station_state.go index 3db3faf1..267dbdd4 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -10,6 +10,7 @@ import ( "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" + "golbat/config" "golbat/db" "golbat/webhooks" ) @@ -89,6 +90,9 @@ func getStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId st // Atomically cache the loaded Station - if another goroutine raced us, // we'll get their Station and use that instead (ensuring same mutex) existingStation, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + if config.Config.FortInMemory { + fortRtreeUpdateStationOnGet(&dbStation) + } return &dbStation }) @@ -131,6 +135,9 @@ func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId st // We loaded from DB station.newRecord = false station.ClearDirty() + if config.Config.FortInMemory { + fortRtreeUpdateStationOnGet(station) + } } } @@ -179,6 +186,9 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { stationCache.Set(station.Id, station, ttlcache.DefaultTTL) station.newRecord = false } + if config.Config.FortInMemory { + fortRtreeUpdateStationOnSave(station) + } } // stationWriteDB performs the actual database INSERT/UPDATE for a Station diff --git a/main.go b/main.go index 64552213..5f7db207 100644 --- a/main.go +++ b/main.go @@ -317,6 +317,8 @@ func main() { apiGroup.POST("/gym/search", SearchGyms) apiGroup.POST("/gym/scan", GymScan) apiGroup.POST("/pokestop/scan", PokestopScan) + apiGroup.POST("/station/scan", StationScan) + apiGroup.POST("/fort/scan", FortScan) apiGroup.POST("/reload-geojson", ReloadGeojson) apiGroup.GET("/reload-geojson", ReloadGeojson) diff --git a/routes.go b/routes.go index 61349880..98af2af5 100644 --- a/routes.go +++ b/routes.go @@ -780,6 +780,42 @@ func PokestopScan(c *gin.Context) { c.JSON(http.StatusOK, result) } +// POST /api/fort/scan +// Combined in-memory fort scan returning gyms, pokestops, and stations in a single rtree traversal +func FortScan(c *gin.Context) { + if !config.Config.FortInMemory { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "fort_in_memory not enabled"}) + return + } + + var params decoder.ApiFortScan + if err := c.ShouldBindJSON(¶ms); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid JSON body"}) + return + } + + result := decoder.FortCombinedScanEndpoint(params, dbDetails) + c.JSON(http.StatusOK, result) +} + +// POST /api/station/scan +// In-memory fort scan with DNF filters (requires fort_in_memory=true) +func StationScan(c *gin.Context) { + if !config.Config.FortInMemory { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "fort_in_memory not enabled"}) + return + } + + var params decoder.ApiFortScan + if err := c.ShouldBindJSON(¶ms); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid JSON body"}) + return + } + + result := decoder.StationScanEndpoint(params, dbDetails) + c.JSON(http.StatusOK, result) +} + func GetTappable(c *gin.Context) { id := c.Param("tappable_id") tappableId, err := strconv.ParseUint(id, 10, 64) From 220d5222d7dda3efdd21b599f4e0e1c9df147e52 Mon Sep 17 00:00:00 2001 From: Fabio1988 Date: Sat, 21 Feb 2026 10:07:55 +0100 Subject: [PATCH 71/78] fix: unused/wrong imports after merge --- decoder/weather_iv.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/decoder/weather_iv.go b/decoder/weather_iv.go index bfbebec5..059e1262 100644 --- a/decoder/weather_iv.go +++ b/decoder/weather_iv.go @@ -2,11 +2,6 @@ package decoder import ( "context" - "encoding/json" - "errors" - "net/http" - "os" - "reflect" "time" "golbat/db" From 05776a1a415b1e5753b3f4a7a38ac7bc88ff7631 Mon Sep 17 00:00:00 2001 From: lenisko <10072920+lenisko@users.noreply.github.com> Date: Sun, 8 Mar 2026 23:16:36 +0100 Subject: [PATCH 72/78] feat: area pre-rtree lookup table --- config/config.go | 1 + decoder/geography.go | 20 ++++- decoder/geography_test.go | 125 +++++++++++++++++++++++++++++++ decoder/gym_state.go | 2 +- decoder/incident_state.go | 8 +- decoder/pokemon_state.go | 2 +- decoder/pokestop_process.go | 2 +- decoder/pokestop_state.go | 2 +- decoder/station_state.go | 2 +- geo/s2_lookup.go | 142 ++++++++++++++++++++++++++++++++++++ 10 files changed, 297 insertions(+), 9 deletions(-) create mode 100644 decoder/geography_test.go create mode 100644 geo/s2_lookup.go diff --git a/config/config.go b/config/config.go index af002e8a..10ba2143 100644 --- a/config/config.go +++ b/config/config.go @@ -132,6 +132,7 @@ type tuning struct { WriteBehindWorkerCount int `koanf:"write_behind_worker_count"` // concurrent writers, default: 50 WriteBehindBatchSize int `koanf:"write_behind_batch_size"` // entries per batch, default: 50 WriteBehindBatchTimeoutMs int `koanf:"write_behind_batch_timeout"` // max wait for batch in ms, default: 100 + S2CellLookup bool `koanf:"s2_cell_lookup"` // Use S2 cell lookup for geofence matching, default: false } type scanRule struct { diff --git a/decoder/geography.go b/decoder/geography.go index ba452e34..e50f947f 100644 --- a/decoder/geography.go +++ b/decoder/geography.go @@ -2,13 +2,16 @@ package decoder import ( "encoding/json" - "github.com/tidwall/rtree" - "golbat/geo" "io/ioutil" "net/http" + "golbat/config" + "golbat/geo" + + "github.com/golang/geo/s2" "github.com/paulmach/orb/geojson" log "github.com/sirupsen/logrus" + "github.com/tidwall/rtree" ) type KojiResponse struct { @@ -21,6 +24,7 @@ type KojiResponse struct { var statsTree *rtree.RTreeG[*geojson.Feature] var nestTree *rtree.RTreeG[*geojson.Feature] +var statsS2Lookup *geo.S2CellLookup var kojiUrl = "" var kojiBearerToken = "" @@ -106,11 +110,23 @@ func ReadGeofences() error { } statsTree = geo.LoadRtree(statsFeatureCollection) + if config.Config.Tuning.S2CellLookup { + statsS2Lookup = geo.BuildS2LookupFromFeatures(statsFeatureCollection) + } return nil } func MatchStatsGeofence(lat, lon float64) []geo.AreaName { + return MatchStatsGeofenceWithCell(lat, lon, 0) +} + +func MatchStatsGeofenceWithCell(lat, lon float64, cellId uint64) []geo.AreaName { + if cellId != 0 && statsS2Lookup != nil { + if areas := statsS2Lookup.Lookup(s2.CellID(cellId)); len(areas) > 0 { + return areas + } + } return geo.MatchGeofencesRtree(statsTree, lat, lon) } diff --git a/decoder/geography_test.go b/decoder/geography_test.go new file mode 100644 index 00000000..6d892941 --- /dev/null +++ b/decoder/geography_test.go @@ -0,0 +1,125 @@ +package decoder + +import ( + "encoding/json" + "math/rand" + "os" + "reflect" + "sort" + "testing" + "time" + + "golbat/geo" + + "github.com/golang/geo/s2" + "github.com/paulmach/orb/geojson" +) + +type KojiTestResponse struct { + Data geojson.FeatureCollection `json:"data"` +} + +func TestMatchStatsGeofenceWithCellVsRtree(t *testing.T) { + data, err := os.ReadFile("../cache/geofences.txt") + if err != nil { + t.Fatalf("Failed to read geofences.txt: %v", err) + } + + var response KojiTestResponse + if err := json.Unmarshal(data, &response); err != nil { + t.Fatalf("Failed to parse geofences.txt: %v", err) + } + + fc := &response.Data + t.Logf("Loaded %d features from geofences.txt", len(fc.Features)) + + testStatsTree := geo.LoadRtree(fc) + t.Logf("Built rtree") + testS2Lookup := geo.BuildS2LookupFromFeatures(fc) + t.Logf("Built S2 lookup with %d cells, size: %.2f MB", testS2Lookup.CellCount(), float64(testS2Lookup.SizeBytes())/(1024*1024)) + + minLat, maxLat := 49.002043, 54.854478 + minLon, maxLon := 14.120178, 24.145783 + + const numPoints = 50000 + rng := rand.New(rand.NewSource(42)) + + var mismatches int + var s2Hits int + var rtreeHits int + var rtreeTime time.Duration + var lookupTime time.Duration + + for i := 0; i < numPoints; i++ { + lat := minLat + rng.Float64()*(maxLat-minLat) + lon := minLon + rng.Float64()*(maxLon-minLon) + + cellID := s2.CellIDFromLatLng(s2.LatLngFromDegrees(lat, lon)).Parent(geo.S2LookupLevel) + + start := time.Now() + areasRtree := geo.MatchGeofencesRtree(testStatsTree, lat, lon) + rtreeTime += time.Since(start) + + start = time.Now() + areasS2 := testS2Lookup.Lookup(cellID) + var areasWithCell []geo.AreaName + if len(areasS2) > 0 { + areasWithCell = areasS2 + s2Hits++ + } else { + areasWithCell = geo.MatchGeofencesRtree(testStatsTree, lat, lon) + rtreeHits++ + } + lookupTime += time.Since(start) + + if !areasEqual(areasRtree, areasWithCell) { + mismatches++ + if mismatches <= 10 { + t.Logf("Mismatch at point %d (%.6f, %.6f): rtree=%v, withCell=%v", + i, lat, lon, areasRtree, areasWithCell) + } + } + } + + t.Logf("Results: %d points tested", numPoints) + t.Logf("S2 lookup hits: %d (%.2f%%)", s2Hits, float64(s2Hits)/float64(numPoints)*100) + t.Logf("Rtree fallback: %d (%.2f%%)", rtreeHits, float64(rtreeHits)/float64(numPoints)*100) + t.Logf("Mismatches: %d (%.2f%%)", mismatches, float64(mismatches)/float64(numPoints)*100) + + // Note: Mismatches are expected for edge cells where S2 lookup returns + // fewer areas than rtree (cell fully inside some polygons but on edge of others) + t.Logf("Note: S2 lookup may return fewer areas for edge cells - this is expected") + + t.Logf("Timing: rtree only: %v (%.2f µs/call)", rtreeTime, float64(rtreeTime.Microseconds())/float64(numPoints)) + t.Logf("Timing: lookup+fallback: %v (%.2f µs/call)", lookupTime, float64(lookupTime.Microseconds())/float64(numPoints)) + t.Logf("Speedup: %.2fx", float64(rtreeTime)/float64(lookupTime)) +} + +func areasEqual(a, b []geo.AreaName) bool { + if len(a) != len(b) { + return false + } + if len(a) == 0 { + return true + } + + aCopy := make([]geo.AreaName, len(a)) + bCopy := make([]geo.AreaName, len(b)) + copy(aCopy, a) + copy(bCopy, b) + + sort.Slice(aCopy, func(i, j int) bool { + if aCopy[i].Parent != aCopy[j].Parent { + return aCopy[i].Parent < aCopy[j].Parent + } + return aCopy[i].Name < aCopy[j].Name + }) + sort.Slice(bCopy, func(i, j int) bool { + if bCopy[i].Parent != bCopy[j].Parent { + return bCopy[i].Parent < bCopy[j].Parent + } + return bCopy[i].Name < bCopy[j].Name + }) + + return reflect.DeepEqual(aCopy, bCopy) +} diff --git a/decoder/gym_state.go b/decoder/gym_state.go index c60097ba..ff395751 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -401,7 +401,7 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { fortRtreeUpdateGymOnSave(gym) } - areas := MatchStatsGeofence(gym.Lat, gym.Lon) + areas := MatchStatsGeofenceWithCell(gym.Lat, gym.Lon, uint64(gym.CellId.ValueOrZero())) createGymWebhooks(gym, areas) createGymFortWebhooks(gym) updateRaidStats(gym, areas) diff --git a/decoder/incident_state.go b/decoder/incident_state.go index f122bb2f..a4fd5916 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -146,13 +146,15 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident createIncidentWebhooks(ctx, db, incident) var stopLat, stopLon float64 + var stopCellId uint64 stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) if stop != nil { stopLat, stopLon = stop.Lat, stop.Lon + stopCellId = uint64(stop.CellId.ValueOrZero()) unlock() } - areas := MatchStatsGeofence(stopLat, stopLon) + areas := MatchStatsGeofenceWithCell(stopLat, stopLon, stopCellId) updateIncidentStats(incident, areas) if config.Config.FortInMemory { @@ -217,12 +219,14 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Inci var pokestopName, stopUrl string var stopLat, stopLon float64 var stopEnabled bool + var stopCellId uint64 stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) if stop != nil { pokestopName = stop.Name.ValueOrZero() stopLat, stopLon = stop.Lat, stop.Lon stopUrl = stop.Url.ValueOrZero() stopEnabled = stop.Enabled.ValueOrZero() + stopCellId = uint64(stop.CellId.ValueOrZero()) unlock() } if pokestopName == "" { @@ -270,7 +274,7 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Inci Lineup: lineup, } - areas := MatchStatsGeofence(stopLat, stopLon) + areas := MatchStatsGeofenceWithCell(stopLat, stopLon, stopCellId) webhooksSender.AddMessage(webhooks.Invasion, incidentHook, areas) statsCollector.UpdateIncidentCount(areas) } diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index c1c6582b..3db16224 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -297,7 +297,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po updatePokemonLookup(pokemon, changePvpField, pvpResults) // Webhooks and stats happen immediately (not queued) - areas := MatchStatsGeofence(pokemon.Lat, pokemon.Lon) + areas := MatchStatsGeofenceWithCell(pokemon.Lat, pokemon.Lon, uint64(pokemon.CellId.ValueOrZero())) if webhook { createPokemonWebhooks(ctx, db, pokemon, areas) } diff --git a/decoder/pokestop_process.go b/decoder/pokestop_process.go index e2792fd2..ca196eeb 100644 --- a/decoder/pokestop_process.go +++ b/decoder/pokestop_process.go @@ -53,7 +53,7 @@ func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.F updatePokestopGetMapFortCache(pokestop) savePokestopRecord(ctx, db, pokestop) - areas := MatchStatsGeofence(pokestop.Lat, pokestop.Lon) + areas := MatchStatsGeofenceWithCell(pokestop.Lat, pokestop.Lon, uint64(pokestop.CellId.ValueOrZero())) updateQuestStats(pokestop, haveAr, areas) return fmt.Sprintf("%s %s %s", quest.FortId, haveArStr, questTitle) diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index 40a0f960..1ed79c9c 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -213,7 +213,7 @@ func createPokestopFortWebhooks(stop *Pokestop) { func createPokestopWebhooks(stop *Pokestop) { - areas := MatchStatsGeofence(stop.Lat, stop.Lon) + areas := MatchStatsGeofenceWithCell(stop.Lat, stop.Lon, uint64(stop.CellId.ValueOrZero())) pokestopName := "Unknown" if stop.Name.Valid { diff --git a/decoder/station_state.go b/decoder/station_state.go index 267dbdd4..5ac0f06c 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -286,7 +286,7 @@ func createStationWebhooks(station *Station) { TotalStationedGmax: station.TotalStationedGmax, Updated: station.Updated, } - areas := MatchStatsGeofence(station.Lat, station.Lon) + areas := MatchStatsGeofenceWithCell(station.Lat, station.Lon, uint64(station.CellId)) webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) statsCollector.UpdateMaxBattleCount(areas, station.BattleLevel.ValueOrZero()) } diff --git a/geo/s2_lookup.go b/geo/s2_lookup.go new file mode 100644 index 00000000..f2b9258b --- /dev/null +++ b/geo/s2_lookup.go @@ -0,0 +1,142 @@ +package geo + +import ( + "unsafe" + + "github.com/golang/geo/s2" + "github.com/paulmach/orb" + "github.com/paulmach/orb/geojson" + log "github.com/sirupsen/logrus" +) + +const S2LookupLevel = 15 + +type S2CellLookup struct { + cells map[s2.CellID][]AreaName + edgeCells map[s2.CellID]struct{} +} + +func NewS2CellLookup() *S2CellLookup { + return &S2CellLookup{ + cells: make(map[s2.CellID][]AreaName), + edgeCells: make(map[s2.CellID]struct{}), + } +} + +func (l *S2CellLookup) addArea(cellID s2.CellID, area AreaName) { + l.cells[cellID] = append(l.cells[cellID], area) +} + +func (l *S2CellLookup) addEdgeCell(cellID s2.CellID) { + l.edgeCells[cellID] = struct{}{} +} + +func (l *S2CellLookup) removeEdgeCells() int { + removed := 0 + for cellID := range l.edgeCells { + if _, exists := l.cells[cellID]; exists { + delete(l.cells, cellID) + removed++ + } + } + l.edgeCells = nil // free memory + return removed +} + +func (l *S2CellLookup) Lookup(cellID s2.CellID) []AreaName { + return l.cells[cellID] +} + +func (l *S2CellLookup) SizeBytes() int64 { + var size int64 + + size += int64(unsafe.Sizeof(l.cells)) + + for cellID, areas := range l.cells { + size += int64(unsafe.Sizeof(cellID)) + size += int64(unsafe.Sizeof(areas)) + for _, area := range areas { + size += int64(unsafe.Sizeof(area)) + size += int64(len(area.Name)) + size += int64(len(area.Parent)) + } + } + + return size +} + +func (l *S2CellLookup) CellCount() int { + return len(l.cells) +} + +func BuildS2LookupFromFeatures(featureCollection *geojson.FeatureCollection) *S2CellLookup { + if featureCollection == nil { + return NewS2CellLookup() + } + + lookup := NewS2CellLookup() + + for _, f := range featureCollection.Features { + name := f.Properties.MustString("name", "unknown") + parent := f.Properties.MustString("parent", name) + area := AreaName{Parent: parent, Name: name} + + geoType := f.Geometry.GeoJSONType() + switch geoType { + case "Polygon": + polygon := f.Geometry.(orb.Polygon) + processPolygon(lookup, polygon, area) + case "MultiPolygon": + multiPolygon := f.Geometry.(orb.MultiPolygon) + for _, polygon := range multiPolygon { + processPolygon(lookup, polygon, area) + } + } + } + + removed := lookup.removeEdgeCells() + log.Infof("GEO: Removed %d edge cells from lookup", removed) + + sizeMB := float64(lookup.SizeBytes()) / (1024 * 1024) + log.Infof("GEO: S2 lookup table built with %d cells, size: %.2f MB", lookup.CellCount(), sizeMB) + + return lookup +} + +func processPolygon(lookup *S2CellLookup, polygon orb.Polygon, area AreaName) { + if len(polygon) == 0 || len(polygon[0]) == 0 { + return + } + + // Convert orb.Polygon to s2.Loop for efficient covering + ring := polygon[0] // outer ring + points := make([]s2.Point, len(ring)) + for i, p := range ring { + points[i] = s2.PointFromLatLng(s2.LatLngFromDegrees(p.Lat(), p.Lon())) + } + + loop := s2.LoopFromPoints(points) + s2Polygon := s2.PolygonFromLoops([]*s2.Loop{loop}) + + coverer := s2.RegionCoverer{ + MinLevel: S2LookupLevel, + MaxLevel: S2LookupLevel, + } + covering := coverer.Covering(s2Polygon) + + for _, cellID := range covering { + cell := s2.CellFromCellID(cellID) + allInside := true + for i := range 4 { + if !s2Polygon.ContainsPoint(cell.Vertex(i)) { + allInside = false + break + } + } + if allInside { + lookup.addArea(cellID, area) + } else { + lookup.addEdgeCell(cellID) + } + } +} From e70ed93ff54c6490e235c498855dfba022195819 Mon Sep 17 00:00:00 2001 From: lenisko <10072920+lenisko@users.noreply.github.com> Date: Sun, 8 Mar 2026 23:38:40 +0100 Subject: [PATCH 73/78] fix: fortwebhook pass cellid --- decoder/fort.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index 09f596dc..73a03b28 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -26,6 +26,7 @@ type FortWebhook struct { Description *string `json:"description"` ImageUrl *string `json:"image_url"` Location Location `json:"location"` + CellId uint64 `json:"-"` // internal use only, not sent in webhook } type FortChangeWebhook struct { @@ -83,6 +84,7 @@ func InitWebHookFortFromGym(gym *Gym) *FortWebhook { fort.ImageUrl = gym.Url.Ptr() fort.Description = gym.Description.Ptr() fort.Location = Location{Latitude: gym.Lat, Longitude: gym.Lon} + fort.CellId = uint64(gym.CellId.ValueOrZero()) return fort } @@ -97,6 +99,7 @@ func InitWebHookFortFromPokestop(stop *Pokestop) *FortWebhook { fort.ImageUrl = stop.Url.Ptr() fort.Description = stop.Description.Ptr() fort.Location = Location{Latitude: stop.Lat, Longitude: stop.Lon} + fort.CellId = uint64(stop.CellId.ValueOrZero()) return fort } @@ -137,7 +140,7 @@ func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []strin func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { if change == NEW { - areas := MatchStatsGeofence(new.Location.Latitude, new.Location.Longitude) + areas := MatchStatsGeofenceWithCell(new.Location.Latitude, new.Location.Longitude, new.CellId) hook := FortChangeWebhook{ ChangeType: change.String(), New: new, @@ -145,7 +148,7 @@ func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) statsCollector.UpdateFortCount(areas, new.Type, "addition") } else if change == REMOVAL { - areas := MatchStatsGeofence(old.Location.Latitude, old.Location.Longitude) + areas := MatchStatsGeofenceWithCell(old.Location.Latitude, old.Location.Longitude, old.CellId) hook := FortChangeWebhook{ ChangeType: change.String(), Old: old, @@ -153,7 +156,7 @@ func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) statsCollector.UpdateFortCount(areas, old.Type, "removal") } else if change == EDIT { - areas := MatchStatsGeofence(new.Location.Latitude, new.Location.Longitude) + areas := MatchStatsGeofenceWithCell(new.Location.Latitude, new.Location.Longitude, new.CellId) var editTypes []string // Check if Name has changed From df97a187e47baed40a9cc38605c9ace29ff0db7a Mon Sep 17 00:00:00 2001 From: lenisko <10072920+lenisko@users.noreply.github.com> Date: Sun, 8 Mar 2026 23:49:12 +0100 Subject: [PATCH 74/78] fix: ignore geography_test in build --- decoder/geography_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/decoder/geography_test.go b/decoder/geography_test.go index 6d892941..6a04f674 100644 --- a/decoder/geography_test.go +++ b/decoder/geography_test.go @@ -1,3 +1,5 @@ +//go:build ignore + package decoder import ( From 38fd0c498f7c5bc77a0503ffceb90994610791f2 Mon Sep 17 00:00:00 2001 From: lenisko <10072920+lenisko@users.noreply.github.com> Date: Mon, 9 Mar 2026 02:35:09 +0100 Subject: [PATCH 75/78] feat: speed up processing areas at startup --- config.toml.example | 1 + geo/s2_lookup.go | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/config.toml.example b/config.toml.example index df9f9c2d..c1fa29b2 100644 --- a/config.toml.example +++ b/config.toml.example @@ -77,6 +77,7 @@ write_behind_batch_size = 50 # Number of entries per batch write write_behind_batch_timeout = 100 # Max wait time in ms before flushing partial batch profile_routes = false # Turn on debugging endpoints profile_contention = false # Collect data for contention (use with above) - has a perf impact +s2_cell_lookup = false # Use S2 cell lookup before rtree for lat, lon > area matching (default: false) # When enabled, reduce_updates will make fort update debounce windows much longer # to reduce database churn. Specifically, gym/pokestop/station debounce will be diff --git a/geo/s2_lookup.go b/geo/s2_lookup.go index f2b9258b..9dead22d 100644 --- a/geo/s2_lookup.go +++ b/geo/s2_lookup.go @@ -1,6 +1,8 @@ package geo import ( + "runtime" + "sync" "unsafe" "github.com/golang/geo/s2" @@ -12,6 +14,7 @@ import ( const S2LookupLevel = 15 type S2CellLookup struct { + mu sync.Mutex cells map[s2.CellID][]AreaName edgeCells map[s2.CellID]struct{} } @@ -24,11 +27,15 @@ func NewS2CellLookup() *S2CellLookup { } func (l *S2CellLookup) addArea(cellID s2.CellID, area AreaName) { + l.mu.Lock() l.cells[cellID] = append(l.cells[cellID], area) + l.mu.Unlock() } func (l *S2CellLookup) addEdgeCell(cellID s2.CellID) { + l.mu.Lock() l.edgeCells[cellID] = struct{}{} + l.mu.Unlock() } func (l *S2CellLookup) removeEdgeCells() int { @@ -69,6 +76,11 @@ func (l *S2CellLookup) CellCount() int { return len(l.cells) } +type polygonWork struct { + polygon orb.Polygon + area AreaName +} + func BuildS2LookupFromFeatures(featureCollection *geojson.FeatureCollection) *S2CellLookup { if featureCollection == nil { return NewS2CellLookup() @@ -76,6 +88,19 @@ func BuildS2LookupFromFeatures(featureCollection *geojson.FeatureCollection) *S2 lookup := NewS2CellLookup() + numWorkers := max(runtime.NumCPU(), 4) + + workChan := make(chan polygonWork, 100) + var wg sync.WaitGroup + + for range numWorkers { + wg.Go(func() { + for work := range workChan { + processPolygon(lookup, work.polygon, work.area) + } + }) + } + for _, f := range featureCollection.Features { name := f.Properties.MustString("name", "unknown") parent := f.Properties.MustString("parent", name) @@ -85,15 +110,18 @@ func BuildS2LookupFromFeatures(featureCollection *geojson.FeatureCollection) *S2 switch geoType { case "Polygon": polygon := f.Geometry.(orb.Polygon) - processPolygon(lookup, polygon, area) + workChan <- polygonWork{polygon: polygon, area: area} case "MultiPolygon": multiPolygon := f.Geometry.(orb.MultiPolygon) for _, polygon := range multiPolygon { - processPolygon(lookup, polygon, area) + workChan <- polygonWork{polygon: polygon, area: area} } } } + close(workChan) + wg.Wait() + removed := lookup.removeEdgeCells() log.Infof("GEO: Removed %d edge cells from lookup", removed) From f45eca99244908e777673903cfdd913094e2b648 Mon Sep 17 00:00:00 2001 From: lenisko <10072920+lenisko@users.noreply.github.com> Date: Mon, 9 Mar 2026 10:49:39 +0100 Subject: [PATCH 76/78] fix: race condition on reload --- decoder/geography.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/decoder/geography.go b/decoder/geography.go index e50f947f..79409bf6 100644 --- a/decoder/geography.go +++ b/decoder/geography.go @@ -4,6 +4,7 @@ import ( "encoding/json" "io/ioutil" "net/http" + "sync/atomic" "golbat/config" "golbat/geo" @@ -22,9 +23,9 @@ type KojiResponse struct { // Stats KojiStats `json:"stats"` } -var statsTree *rtree.RTreeG[*geojson.Feature] +var statsTree atomic.Value var nestTree *rtree.RTreeG[*geojson.Feature] -var statsS2Lookup *geo.S2CellLookup +var statsS2Lookup atomic.Value var kojiUrl = "" var kojiBearerToken = "" @@ -109,11 +110,15 @@ func ReadGeofences() error { statsFeatureCollection = fc } - statsTree = geo.LoadRtree(statsFeatureCollection) + newStatsTree := geo.LoadRtree(statsFeatureCollection) + var newStatsS2Lookup *geo.S2CellLookup if config.Config.Tuning.S2CellLookup { - statsS2Lookup = geo.BuildS2LookupFromFeatures(statsFeatureCollection) + newStatsS2Lookup = geo.BuildS2LookupFromFeatures(statsFeatureCollection) } + statsTree.Store(newStatsTree) + statsS2Lookup.Store(newStatsS2Lookup) + return nil } @@ -122,12 +127,14 @@ func MatchStatsGeofence(lat, lon float64) []geo.AreaName { } func MatchStatsGeofenceWithCell(lat, lon float64, cellId uint64) []geo.AreaName { - if cellId != 0 && statsS2Lookup != nil { - if areas := statsS2Lookup.Lookup(s2.CellID(cellId)); len(areas) > 0 { + lookup, _ := statsS2Lookup.Load().(*geo.S2CellLookup) + if cellId != 0 && lookup != nil { + if areas := lookup.Lookup(s2.CellID(cellId)); len(areas) > 0 { return areas } } - return geo.MatchGeofencesRtree(statsTree, lat, lon) + tree, _ := statsTree.Load().(*rtree.RTreeG[*geojson.Feature]) + return geo.MatchGeofencesRtree(tree, lat, lon) } func MatchNestGeofence(lat, lon float64) []geo.AreaName { From 9bf73e8024d413cb031387dfa54f0bbf281a07de Mon Sep 17 00:00:00 2001 From: Jakub <10072920+lenisko@users.noreply.github.com> Date: Mon, 9 Mar 2026 10:59:21 +0100 Subject: [PATCH 77/78] Remove duplicated FortChangeWebhook struct --- decoder/fort.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index 4b304a1a..73a03b28 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -36,13 +36,6 @@ type FortChangeWebhook struct { New *FortWebhook `json:"new,omitempty"` } -type FortChangeWebhook struct { - ChangeType string `json:"change_type"` - EditTypes []string `json:"edit_types,omitempty"` - Old *FortWebhook `json:"old,omitempty"` - New *FortWebhook `json:"new,omitempty"` -} - type FortChange string type FortType int8 From 65dfb56468cc5e1eeed086a81ad0ebae1922d10b Mon Sep 17 00:00:00 2001 From: lenisko <10072920+lenisko@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:22:48 +0100 Subject: [PATCH 78/78] fix: small changes to please jabes claude --- config.toml.example | 2 +- config/config.go | 2 +- decoder/geography.go | 5 +++-- decoder/geography_test.go | 8 +++++--- geo/s2_lookup.go | 40 ++++++++++++++++++++++----------------- 5 files changed, 33 insertions(+), 24 deletions(-) diff --git a/config.toml.example b/config.toml.example index 70b5fd20..a3b10a15 100644 --- a/config.toml.example +++ b/config.toml.example @@ -77,7 +77,7 @@ write_behind_batch_size = 50 # Number of entries per batch write write_behind_batch_timeout = 100 # Max wait time in ms before flushing partial batch profile_routes = false # Turn on debugging endpoints profile_contention = false # Collect data for contention (use with above) - has a perf impact -s2_cell_lookup = false # Use S2 cell lookup before rtree for lat, lon > area matching (default: false) +s2_cell_lookup = false # Pre-compute S2 cell lookup for faster geofence matching. Trades memory (~60x geofence file size) for ~7x faster lookups. (default: false) # When enabled, reduce_updates will make fort update debounce windows much longer # to reduce database churn. Specifically, gym/pokestop/station debounce will be diff --git a/config/config.go b/config/config.go index 9ee6f274..b9af097e 100644 --- a/config/config.go +++ b/config/config.go @@ -133,7 +133,7 @@ type tuning struct { WriteBehindWorkerCount int `koanf:"write_behind_worker_count"` // concurrent writers, default: 50 WriteBehindBatchSize int `koanf:"write_behind_batch_size"` // entries per batch, default: 50 WriteBehindBatchTimeoutMs int `koanf:"write_behind_batch_timeout"` // max wait for batch in ms, default: 100 - S2CellLookup bool `koanf:"s2_cell_lookup"` // Use S2 cell lookup for geofence matching, default: false + S2CellLookup bool `koanf:"s2_cell_lookup"` // Pre-compute S2 cell lookup for faster geofence matching. Trades memory (~60x geofence file size) for ~7x faster lookups, default: false } type scanRule struct { diff --git a/decoder/geography.go b/decoder/geography.go index 79409bf6..49c73812 100644 --- a/decoder/geography.go +++ b/decoder/geography.go @@ -24,7 +24,7 @@ type KojiResponse struct { } var statsTree atomic.Value -var nestTree *rtree.RTreeG[*geojson.Feature] +var nestTree atomic.Value var statsS2Lookup atomic.Value var kojiUrl = "" var kojiBearerToken = "" @@ -138,5 +138,6 @@ func MatchStatsGeofenceWithCell(lat, lon float64, cellId uint64) []geo.AreaName } func MatchNestGeofence(lat, lon float64) []geo.AreaName { - return geo.MatchGeofencesRtree(nestTree, lat, lon) + tree, _ := nestTree.Load().(*rtree.RTreeG[*geojson.Feature]) + return geo.MatchGeofencesRtree(tree, lat, lon) } diff --git a/decoder/geography_test.go b/decoder/geography_test.go index 6a04f674..f93ed7af 100644 --- a/decoder/geography_test.go +++ b/decoder/geography_test.go @@ -88,9 +88,11 @@ func TestMatchStatsGeofenceWithCellVsRtree(t *testing.T) { t.Logf("Rtree fallback: %d (%.2f%%)", rtreeHits, float64(rtreeHits)/float64(numPoints)*100) t.Logf("Mismatches: %d (%.2f%%)", mismatches, float64(mismatches)/float64(numPoints)*100) - // Note: Mismatches are expected for edge cells where S2 lookup returns - // fewer areas than rtree (cell fully inside some polygons but on edge of others) - t.Logf("Note: S2 lookup may return fewer areas for edge cells - this is expected") + // Mismatches should be 0 — edge cells are excluded from the S2 lookup, + // so the rtree fallback always provides the complete result. + if mismatches > 0 { + t.Errorf("Expected 0 mismatches, got %d. Edge cells should be excluded from S2 lookup.", mismatches) + } t.Logf("Timing: rtree only: %v (%.2f µs/call)", rtreeTime, float64(rtreeTime.Microseconds())/float64(numPoints)) t.Logf("Timing: lookup+fallback: %v (%.2f µs/call)", lookupTime, float64(lookupTime.Microseconds())/float64(numPoints)) diff --git a/geo/s2_lookup.go b/geo/s2_lookup.go index 9dead22d..61aeed23 100644 --- a/geo/s2_lookup.go +++ b/geo/s2_lookup.go @@ -14,7 +14,6 @@ import ( const S2LookupLevel = 15 type S2CellLookup struct { - mu sync.Mutex cells map[s2.CellID][]AreaName edgeCells map[s2.CellID]struct{} } @@ -26,18 +25,6 @@ func NewS2CellLookup() *S2CellLookup { } } -func (l *S2CellLookup) addArea(cellID s2.CellID, area AreaName) { - l.mu.Lock() - l.cells[cellID] = append(l.cells[cellID], area) - l.mu.Unlock() -} - -func (l *S2CellLookup) addEdgeCell(cellID s2.CellID) { - l.mu.Lock() - l.edgeCells[cellID] = struct{}{} - l.mu.Unlock() -} - func (l *S2CellLookup) removeEdgeCells() int { removed := 0 for cellID := range l.edgeCells { @@ -87,6 +74,20 @@ func BuildS2LookupFromFeatures(featureCollection *geojson.FeatureCollection) *S2 } lookup := NewS2CellLookup() + var mu sync.Mutex // Only used during build phase + + // Helper closures for thread-safe writes during build + addArea := func(cellID s2.CellID, area AreaName) { + mu.Lock() + lookup.cells[cellID] = append(lookup.cells[cellID], area) + mu.Unlock() + } + + addEdgeCell := func(cellID s2.CellID) { + mu.Lock() + lookup.edgeCells[cellID] = struct{}{} + mu.Unlock() + } numWorkers := max(runtime.NumCPU(), 4) @@ -96,7 +97,7 @@ func BuildS2LookupFromFeatures(featureCollection *geojson.FeatureCollection) *S2 for range numWorkers { wg.Go(func() { for work := range workChan { - processPolygon(lookup, work.polygon, work.area) + processPolygon(work.polygon, work.area, addArea, addEdgeCell) } }) } @@ -131,7 +132,12 @@ func BuildS2LookupFromFeatures(featureCollection *geojson.FeatureCollection) *S2 return lookup } -func processPolygon(lookup *S2CellLookup, polygon orb.Polygon, area AreaName) { +func processPolygon( + polygon orb.Polygon, + area AreaName, + addArea func(s2.CellID, AreaName), + addEdgeCell func(s2.CellID), +) { if len(polygon) == 0 || len(polygon[0]) == 0 { return } @@ -162,9 +168,9 @@ func processPolygon(lookup *S2CellLookup, polygon orb.Polygon, area AreaName) { } } if allInside { - lookup.addArea(cellID, area) + addArea(cellID, area) } else { - lookup.addEdgeCell(cellID) + addEdgeCell(cellID) } } }