Skip to content

Commit f44c4b6

Browse files
yhabteabyhabteab
committed
Use ctx everywhere & cache object only when the tx succeeds
1 parent cb15ebe commit f44c4b6

File tree

8 files changed

+183
-141
lines changed

8 files changed

+183
-141
lines changed

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ require (
1111
github.com/teambition/rrule-go v1.8.2
1212
go.uber.org/zap v1.23.0
1313
golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d
14+
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
1415
)
1516

1617
require (
@@ -30,7 +31,6 @@ require (
3031
github.com/ssgreg/journald v1.0.0 // indirect
3132
go.uber.org/atomic v1.7.0 // indirect
3233
go.uber.org/multierr v1.6.0 // indirect
33-
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
3434
golang.org/x/sys v0.1.0 // indirect
3535
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
3636
gopkg.in/yaml.v3 v3.0.1 // indirect

‎internal/incident/db_types.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package incident
22

33
import (
4+
"context"
45
"fmt"
56
"github.com/icinga/icinga-notifications/internal/event"
67
"github.com/icinga/icinga-notifications/internal/recipient"
@@ -33,15 +34,15 @@ func (i *IncidentRow) Upsert() interface{} {
3334
// Sync synchronizes incidents to the database.
3435
// Fetches the last inserted incident id and modifies this incident's id.
3536
// Returns an error on database failure.
36-
func (i *IncidentRow) Sync(tx *sqlx.Tx, db *icingadb.DB, upsert bool) error {
37+
func (i *IncidentRow) Sync(ctx context.Context, tx *sqlx.Tx, db *icingadb.DB, upsert bool) error {
3738
if upsert {
3839
stmt, _ := db.BuildUpsertStmt(i)
39-
_, err := tx.NamedExec(stmt, i)
40+
_, err := tx.NamedExecContext(ctx, stmt, i)
4041
if err != nil {
4142
return fmt.Errorf("failed to upsert incident: %s", err)
4243
}
4344
} else {
44-
incidentId, err := utils.InsertAndFetchId(tx, utils.BuildInsertStmtWithout(db, i, "id"), i)
45+
incidentId, err := utils.InsertAndFetchId(ctx, tx, utils.BuildInsertStmtWithout(db, i, "id"), i)
4546
if err != nil {
4647
return err
4748
}

‎internal/incident/incident.go‎

Lines changed: 81 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package incident
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
67
"github.com/icinga/icinga-notifications/internal/config"
@@ -75,14 +76,14 @@ func (i *Incident) HasManager() bool {
7576
}
7677

7778
// ProcessEvent processes the given event for the current incident.
78-
func (i *Incident) ProcessEvent(tx *sqlx.Tx, ev event.Event, created bool) error {
79+
func (i *Incident) ProcessEvent(ctx context.Context, tx *sqlx.Tx, ev event.Event, created bool) error {
7980
i.Lock()
8081
defer i.Unlock()
8182

8283
i.runtimeConfig.RLock()
8384
defer i.runtimeConfig.RUnlock()
8485

85-
err := i.Object.UpdateMetadata(tx, ev.SourceId, ev.Name, utils.ToDBString(ev.URL), ev.ExtraTags)
86+
err := i.Object.UpdateMetadata(ctx, tx, ev.SourceId, ev.Name, utils.ToDBString(ev.URL), ev.ExtraTags)
8687
if err != nil {
8788
i.logger.Errorw("can't update object metadata", zap.String("object", i.Object.DisplayName()), zap.Error(err))
8889

@@ -91,7 +92,7 @@ func (i *Incident) ProcessEvent(tx *sqlx.Tx, ev event.Event, created bool) error
9192

9293
if ev.ID == 0 {
9394
eventRow := event.NewEventRow(ev, i.Object.ID)
94-
eventID, err := utils.InsertAndFetchId(tx, utils.BuildInsertStmtWithout(i.db, eventRow, "id"), eventRow)
95+
eventID, err := utils.InsertAndFetchId(ctx, tx, utils.BuildInsertStmtWithout(i.db, eventRow, "id"), eventRow)
9596
if err != nil {
9697
i.logger.Errorw(
9798
"failed insert event and fetch its ID", zap.String("object", i.ObjectDisplayName()),
@@ -105,13 +106,13 @@ func (i *Incident) ProcessEvent(tx *sqlx.Tx, ev event.Event, created bool) error
105106
}
106107

107108
if created {
108-
err := i.processIncidentOpenedEvent(tx, ev)
109+
err := i.processIncidentOpenedEvent(ctx, tx, ev)
109110
if err != nil {
110111
return err
111112
}
112113
}
113114

114-
if err := i.AddEvent(tx, &ev); err != nil {
115+
if err := i.AddEvent(ctx, tx, &ev); err != nil {
115116
i.logger.Errorw(
116117
"can't insert incident event to the database", zap.String("object", i.ObjectDisplayName()),
117118
zap.String("incident", i.String()), zap.Error(err),
@@ -121,28 +122,39 @@ func (i *Incident) ProcessEvent(tx *sqlx.Tx, ev event.Event, created bool) error
121122
}
122123

123124
if ev.Type == event.TypeAcknowledgement {
124-
return i.processAcknowledgementEvent(tx, ev)
125+
// The current request has been processed successfully, so update the in-memory objects cache.
126+
i.Object.UpdateCache()
127+
128+
return i.processAcknowledgementEvent(ctx, tx, ev)
125129
}
126130

127-
causedBy, err := i.processSeverityChangedEvent(tx, ev)
131+
causedBy, err := i.processSeverityChangedEvent(ctx, tx, ev)
128132
if err != nil {
129133
return err
130134
}
131135

132136
// Check if any (additional) rules match this object. Filters of rules that already have a state don't have
133137
// to be checked again, these rules already matched and stay effective for the ongoing incident.
134-
causedBy, err = i.evaluateRules(tx, ev.ID, causedBy)
138+
causedBy, err = i.evaluateRules(ctx, tx, ev.ID, causedBy)
135139
if err != nil {
136140
return err
137141
}
138142

139143
// Re-evaluate escalations based on the newly evaluated rules.
140144
i.evaluateEscalations()
141145

142-
return i.notifyContacts(tx, &ev, causedBy)
146+
err = i.notifyContacts(ctx, tx, &ev, causedBy)
147+
if err != nil {
148+
return err
149+
}
150+
151+
// The current request has been processed successfully, so update the in-memory objects cache.
152+
i.Object.UpdateCache()
153+
154+
return nil
143155
}
144156

145-
func (i *Incident) processSeverityChangedEvent(tx *sqlx.Tx, ev event.Event) (types.Int, error) {
157+
func (i *Incident) processSeverityChangedEvent(ctx context.Context, tx *sqlx.Tx, ev event.Event) (types.Int, error) {
146158
oldIncidentSeverity := i.Severity()
147159
oldSourceSeverity := i.SeverityBySource[ev.SourceId]
148160
if oldSourceSeverity == event.SeverityNone {
@@ -169,7 +181,7 @@ func (i *Incident) processSeverityChangedEvent(tx *sqlx.Tx, ev event.Event) (typ
169181
OldSeverity: oldSourceSeverity,
170182
Message: utils.ToDBString(ev.Message),
171183
}
172-
causedByHistoryId, err := i.AddHistory(tx, history, true)
184+
causedByHistoryId, err := i.AddHistory(ctx, tx, history, true)
173185
if err != nil {
174186
i.logger.Errorw(
175187
"can't insert source severity changed history to the database", zap.String("object", i.ObjectDisplayName()),
@@ -179,7 +191,7 @@ func (i *Incident) processSeverityChangedEvent(tx *sqlx.Tx, ev event.Event) (typ
179191
return types.Int{}, errors.New("can't insert source severity changed history to the database")
180192
}
181193

182-
if err = i.AddSourceSeverity(tx, ev.Severity, ev.SourceId); err != nil {
194+
if err = i.AddSourceSeverity(ctx, tx, ev.Severity, ev.SourceId); err != nil {
183195
i.logger.Errorw(
184196
"failed to upsert source severity", zap.String("object", i.ObjectDisplayName()), zap.String("incident", i.String()),
185197
zap.Error(err),
@@ -199,7 +211,7 @@ func (i *Incident) processSeverityChangedEvent(tx *sqlx.Tx, ev event.Event) (typ
199211
zap.String("object", i.ObjectDisplayName()), zap.String("incident", i.String()),
200212
)
201213

202-
if err = i.Sync(tx); err != nil {
214+
if err = i.Sync(ctx, tx); err != nil {
203215
i.logger.Errorw(
204216
"failed to update incident severity", zap.String("object", i.ObjectDisplayName()), zap.String("incident", i.String()),
205217
zap.Error(err),
@@ -216,7 +228,7 @@ func (i *Incident) processSeverityChangedEvent(tx *sqlx.Tx, ev event.Event) (typ
216228
OldSeverity: oldIncidentSeverity,
217229
CausedByIncidentHistoryID: causedByHistoryId,
218230
}
219-
if causedByHistoryId, err = i.AddHistory(tx, history, true); err != nil {
231+
if causedByHistoryId, err = i.AddHistory(ctx, tx, history, true); err != nil {
220232
i.logger.Errorw(
221233
"failed to insert incident severity changed history", zap.String("object", i.ObjectDisplayName()),
222234
zap.String("incident", i.String()), zap.Error(err),
@@ -233,7 +245,7 @@ func (i *Incident) processSeverityChangedEvent(tx *sqlx.Tx, ev event.Event) (typ
233245
RemoveCurrent(i.Object)
234246

235247
incidentRow := &IncidentRow{ID: i.incidentRowID, RecoveredAt: types.UnixMilli(i.RecoveredAt)}
236-
_, err = tx.NamedExec(`UPDATE "incident" SET "recovered_at" = :recovered_at WHERE id = :id`, incidentRow)
248+
_, err = tx.NamedExecContext(ctx, `UPDATE "incident" SET "recovered_at" = :recovered_at WHERE id = :id`, incidentRow)
237249
if err != nil {
238250
i.logger.Errorw(
239251
"failed to close incident", zap.String("object", i.ObjectDisplayName()), zap.String("incident", i.String()),
@@ -249,7 +261,7 @@ func (i *Incident) processSeverityChangedEvent(tx *sqlx.Tx, ev event.Event) (typ
249261
Type: Closed,
250262
}
251263

252-
_, err = i.AddHistory(tx, history, false)
264+
_, err = i.AddHistory(ctx, tx, history, false)
253265
if err != nil {
254266
i.logger.Errorw(
255267
"can't insert incident closed history to the database", zap.String("object", i.ObjectDisplayName()),
@@ -263,9 +275,9 @@ func (i *Incident) processSeverityChangedEvent(tx *sqlx.Tx, ev event.Event) (typ
263275
return causedByHistoryId, nil
264276
}
265277

266-
func (i *Incident) processIncidentOpenedEvent(tx *sqlx.Tx, ev event.Event) error {
278+
func (i *Incident) processIncidentOpenedEvent(ctx context.Context, tx *sqlx.Tx, ev event.Event) error {
267279
i.StartedAt = ev.Time
268-
if err := i.Sync(tx); err != nil {
280+
if err := i.Sync(ctx, tx); err != nil {
269281
i.logger.Errorw(
270282
"can't insert incident to the database", zap.String("incident", i.String()),
271283
zap.String("object", i.ObjectDisplayName()), zap.Error(err),
@@ -281,7 +293,7 @@ func (i *Incident) processIncidentOpenedEvent(tx *sqlx.Tx, ev event.Event) error
281293
EventID: utils.ToDBInt(ev.ID),
282294
}
283295

284-
if _, err := i.AddHistory(tx, historyRow, false); err != nil {
296+
if _, err := i.AddHistory(ctx, tx, historyRow, false); err != nil {
285297
i.logger.Errorw(
286298
"can't insert incident opened history event", zap.String("object", i.ObjectDisplayName()),
287299
zap.String("incident", i.String()), zap.Error(err),
@@ -296,7 +308,7 @@ func (i *Incident) processIncidentOpenedEvent(tx *sqlx.Tx, ev event.Event) error
296308
// evaluateRules evaluates all the configured rules for this *incident.Object and
297309
// generates history entries for each matched rule.
298310
// Returns error on database failure.
299-
func (i *Incident) evaluateRules(tx *sqlx.Tx, eventID int64, causedBy types.Int) (types.Int, error) {
311+
func (i *Incident) evaluateRules(ctx context.Context, tx *sqlx.Tx, eventID int64, causedBy types.Int) (types.Int, error) {
300312
if i.Rules == nil {
301313
i.Rules = make(map[int64]struct{})
302314
}
@@ -333,7 +345,7 @@ func (i *Incident) evaluateRules(tx *sqlx.Tx, eventID int64, causedBy types.Int)
333345
Type: RuleMatched,
334346
CausedByIncidentHistoryID: causedBy,
335347
}
336-
insertedID, err := i.AddRuleMatchedHistory(tx, r, history)
348+
insertedID, err := i.AddRuleMatchedHistory(ctx, tx, r, history)
337349
if err != nil {
338350
i.logger.Errorw(
339351
"failed to add incident rule matched history", zap.String("rule", r.Name), zap.String("object", i.ObjectDisplayName()),
@@ -403,7 +415,7 @@ func (i *Incident) evaluateEscalations() {
403415
// notifyContacts evaluates the incident.EscalationState and checks if escalations need to be triggered
404416
// as well as builds the incident recipients along with their channel types and sends notifications based on that.
405417
// Returns error on database failure.
406-
func (i *Incident) notifyContacts(tx *sqlx.Tx, ev *event.Event, causedBy types.Int) error {
418+
func (i *Incident) notifyContacts(ctx context.Context, tx *sqlx.Tx, ev *event.Event, causedBy types.Int) error {
407419
managed := i.HasManager()
408420

409421
contactChannels := make(map[*recipient.Contact]map[string]struct{})
@@ -438,7 +450,7 @@ func (i *Incident) notifyContacts(tx *sqlx.Tx, ev *event.Event, causedBy types.I
438450
CausedByIncidentHistoryID: causedBy,
439451
}
440452

441-
causedByHistoryId, err := i.AddEscalationTriggered(tx, state, history)
453+
causedByHistoryId, err := i.AddEscalationTriggered(ctx, tx, state, history)
442454
if err != nil {
443455
i.logger.Errorw(
444456
"failed to add escalation triggered history", zap.String("incident", i.String()),
@@ -451,7 +463,7 @@ func (i *Incident) notifyContacts(tx *sqlx.Tx, ev *event.Event, causedBy types.I
451463

452464
causedBy = causedByHistoryId
453465

454-
err = i.AddRecipient(tx, escalation, ev.ID)
466+
err = i.AddRecipient(ctx, tx, escalation, ev.ID)
455467
if err != nil {
456468
i.logger.Errorw(
457469
"failed to add incident recipients", zap.String("object", i.ObjectDisplayName()), zap.String("incident", i.String()),
@@ -512,44 +524,53 @@ func (i *Incident) notifyContacts(tx *sqlx.Tx, ev *event.Event, causedBy types.I
512524
CausedByIncidentHistoryID: causedBy,
513525
}
514526

515-
for chType := range channels {
516-
i.logger.Infow(
517-
fmt.Sprintf("Notify contact %q via %q", contact.FullName, chType), zap.String("object", i.ObjectDisplayName()),
518-
zap.String("incident", i.String()),
519-
)
527+
select {
528+
case <-ctx.Done():
529+
return ctx.Err()
530+
default:
531+
for chType := range channels {
532+
i.logger.Infow(
533+
fmt.Sprintf("Notify contact %q via %q", contact.FullName, chType), zap.String("object", i.ObjectDisplayName()),
534+
zap.String("incident", i.String()),
535+
)
520536

521-
hr.ChannelType = utils.ToDBString(chType)
537+
hr.ChannelType = utils.ToDBString(chType)
522538

523-
_, err := i.AddHistory(tx, hr, false)
524-
if err != nil {
525-
i.logger.Errorln(err)
526-
}
539+
_, err := i.AddHistory(ctx, tx, hr, false)
540+
if err != nil {
541+
i.logger.Errorw(
542+
"failed to insert contact notified incident history", zap.String("type", chType),
543+
zap.String("contact", contact.String()), zap.String("object", i.ObjectDisplayName()),
544+
zap.String("incident", i.String()),
545+
)
546+
}
527547

528-
chConf := i.runtimeConfig.Channels[chType]
529-
if chConf == nil {
530-
i.logger.Errorw(
531-
"could not find config for channel", zap.String("type", chType), zap.String("object", i.ObjectDisplayName()),
532-
zap.String("incident", i.String()),
533-
)
534-
continue
535-
}
548+
chConf := i.runtimeConfig.Channels[chType]
549+
if chConf == nil {
550+
i.logger.Errorw(
551+
"could not find config for channel", zap.String("type", chType), zap.String("object", i.ObjectDisplayName()),
552+
zap.String("incident", i.String()),
553+
)
554+
continue
555+
}
536556

537-
plugin, err := chConf.GetPlugin()
538-
if err != nil {
539-
i.logger.Errorw(
540-
"couldn't initialize channel", zap.String("type", chType), zap.String("object", i.ObjectDisplayName()),
541-
zap.String("incident", i.String()), zap.Error(err),
542-
)
543-
continue
544-
}
557+
plugin, err := chConf.GetPlugin()
558+
if err != nil {
559+
i.logger.Errorw(
560+
"couldn't initialize channel", zap.String("type", chType), zap.String("object", i.ObjectDisplayName()),
561+
zap.String("incident", i.String()), zap.Error(err),
562+
)
563+
continue
564+
}
545565

546-
err = plugin.Send(contact, i, ev, i.configFile.Icingaweb2URL)
547-
if err != nil {
548-
i.logger.Errorw(
549-
"failed to send via channel", zap.String("type", chType), zap.String("object", i.ObjectDisplayName()),
550-
zap.String("incident", i.String()), zap.Error(err),
551-
)
552-
continue
566+
err = plugin.Send(contact, i, ev, i.configFile.Icingaweb2URL)
567+
if err != nil {
568+
i.logger.Errorw(
569+
"failed to send via channel", zap.String("type", chType), zap.String("object", i.ObjectDisplayName()),
570+
zap.String("incident", i.String()), zap.Error(err),
571+
)
572+
continue
573+
}
553574
}
554575
}
555576
}
@@ -560,7 +581,7 @@ func (i *Incident) notifyContacts(tx *sqlx.Tx, ev *event.Event, causedBy types.I
560581
// processAcknowledgementEvent processes the given ack event.
561582
// Promotes the ack author to incident.RoleManager if it's not already the case and generates a history entry.
562583
// Returns error on database failure.
563-
func (i *Incident) processAcknowledgementEvent(tx *sqlx.Tx, ev event.Event) error {
584+
func (i *Incident) processAcknowledgementEvent(ctx context.Context, tx *sqlx.Tx, ev event.Event) error {
564585
contact := i.runtimeConfig.GetContact(ev.Username)
565586
if contact == nil {
566587
i.logger.Warnw(
@@ -601,7 +622,7 @@ func (i *Incident) processAcknowledgementEvent(tx *sqlx.Tx, ev event.Event) erro
601622
Message: utils.ToDBString(ev.Message),
602623
}
603624

604-
_, err := i.AddHistory(tx, hr, false)
625+
_, err := i.AddHistory(ctx, tx, hr, false)
605626
if err != nil {
606627
i.logger.Errorw(
607628
"failed to add recipient role changed history", zap.String("recipient", contact.String()),
@@ -614,7 +635,7 @@ func (i *Incident) processAcknowledgementEvent(tx *sqlx.Tx, ev event.Event) erro
614635
cr := &ContactRow{IncidentID: hr.IncidentID, Key: recipientKey, Role: newRole}
615636

616637
stmt, _ := i.db.BuildUpsertStmt(cr)
617-
_, err = tx.NamedExec(stmt, cr)
638+
_, err = tx.NamedExecContext(ctx, stmt, cr)
618639
if err != nil {
619640
i.logger.Errorw(
620641
"failed to upsert incident contact", zap.String("contact", contact.String()), zap.String("object", i.ObjectDisplayName()),

0 commit comments

Comments
 (0)