Skip to content

Commit 478ceb0

Browse files
committed
feat: merge PD incidents automatically
Signed-off-by: Chris Waldon <cwaldon@redhat.com>
1 parent 525c499 commit 478ceb0

File tree

4 files changed

+126
-6
lines changed

4 files changed

+126
-6
lines changed

go.sum

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
371371
github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
372372
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
373373
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
374-
github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
375374
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
376375
github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs=
377376
github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
@@ -927,7 +926,6 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
927926
github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
928927
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
929928
github.com/onsi/ginkgo v1.12.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
930-
github.com/onsi/ginkgo v1.13.0 h1:M76yO2HkZASFjXL0HSoZJ1AYEmQxNJmY41Jx1zNUq1Y=
931929
github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
932930
github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
933931
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
@@ -941,7 +939,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
941939
github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
942940
github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
943941
github.com/onsi/gomega v1.10.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
944-
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
945942
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
946943
github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
947944
github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
@@ -1170,7 +1167,6 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
11701167
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
11711168
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
11721169
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
1173-
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
11741170
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
11751171
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
11761172
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=

pkg/common/config/config.go

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -429,9 +429,14 @@ var Alert = struct {
429429
// PagerDutyAPIToken is a pagerduty token
430430
// Env: PAGERDUTY_API_TOKEN
431431
PagerDutyAPIToken string
432+
433+
// PagerDutyUserToken is a pagerduty token for a user account with full access to the v2 API
434+
// Env: PAGERDUTY_API_TOKEN
435+
PagerDutyUserToken string
432436
}{
433-
SlackAPIToken: "alert.slackAPIToken",
434-
PagerDutyAPIToken: "alert.pagerDutyAPIToken",
437+
SlackAPIToken: "alert.slackAPIToken",
438+
PagerDutyAPIToken: "alert.pagerDutyAPIToken",
439+
PagerDutyUserToken: "alert.pagerDutyUserToken",
435440
}
436441

437442
func init() {
@@ -646,6 +651,9 @@ func init() {
646651

647652
viper.BindEnv(Alert.PagerDutyAPIToken, "PAGERDUTY_API_TOKEN")
648653
RegisterSecret(Alert.PagerDutyAPIToken, "pagerduty-api-token")
654+
655+
viper.BindEnv(Alert.PagerDutyAPIToken, "PAGERDUTY_USER_TOKEN")
656+
RegisterSecret(Alert.PagerDutyAPIToken, "pagerduty-user-token")
649657
}
650658

651659
// PostProcess is a variety of post-processing commands that is intended to be run after a config is loaded.

pkg/common/pagerduty/pagerduty.go

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package pagerduty
22

33
import (
44
"fmt"
5+
"log"
6+
"sort"
57

68
pd "github.com/PagerDuty/go-pagerduty"
79
)
@@ -25,3 +27,110 @@ func (pdc Config) FireAlert(details pd.V2Payload) (*pd.V2EventResponse, error) {
2527
}
2628
return e, nil
2729
}
30+
31+
// MergeCICDIncidents merges all incidents for the CI Watcher by their title.
32+
func MergeCICDIncidents(client *pd.Client) error {
33+
options := pd.ListIncidentsOptions{
34+
ServiceIDs: []string{"P7VT2V5"},
35+
Statuses: []string{"triggered", "acknowledged"},
36+
APIListObject: pd.APIListObject{
37+
Limit: 100,
38+
},
39+
}
40+
var incidents []pd.Incident
41+
if err := Incidents(client, options, func(i pd.Incident) error {
42+
incidents = append(incidents, i)
43+
return nil
44+
}); err != nil {
45+
return fmt.Errorf("failed collecting incidents: %w", err)
46+
}
47+
if err := MergeIncidentsByTitle(client, incidents); err != nil {
48+
return fmt.Errorf("failed merging incidents: %w", err)
49+
}
50+
return nil
51+
}
52+
53+
// MergeIncidentsByTitle will combine all incidents in the provided slice that share the
54+
// same title into a single incident with multiple alerts.
55+
func MergeIncidentsByTitle(c *pd.Client, incidents []pd.Incident) error {
56+
titleToIncident := make(map[string][]pd.Incident)
57+
58+
for _, incident := range incidents {
59+
titleToIncident[incident.Title] = append(titleToIncident[incident.Title], incident)
60+
}
61+
62+
for _, incidents := range titleToIncident {
63+
sort.Slice(incidents, func(i, j int) bool {
64+
return incidents[i].Id < incidents[j].Id
65+
})
66+
if len(incidents) < 2 {
67+
continue
68+
}
69+
first := incidents[0]
70+
mergeOptions := []pd.MergeIncidentsOptions{}
71+
for _, toMerge := range incidents[1:] {
72+
mergeOptions = append(mergeOptions, pd.MergeIncidentsOptions{
73+
ID: toMerge.Id,
74+
Type: toMerge.APIObject.Type,
75+
})
76+
}
77+
log.Printf("Merging into %s: %v", first.Id, mergeOptions)
78+
_, err := c.MergeIncidents("", first.Id, mergeOptions)
79+
if err != nil {
80+
return fmt.Errorf("Failed merging %d incidents into %s: %w", len(incidents)-1, first.Id, err)
81+
}
82+
}
83+
return nil
84+
}
85+
86+
// Incidents uses the provided client to retrieve all Incidents matching the provided
87+
// list options and calls the handler function on each one.
88+
func Incidents(c *pd.Client, options pd.ListIncidentsOptions, handler func(pd.Incident) error) error {
89+
var (
90+
il = new(pd.ListIncidentsResponse)
91+
err error
92+
previousLen int
93+
)
94+
firstIteration := true
95+
for il.APIListObject.More || firstIteration {
96+
firstIteration = false
97+
options.APIListObject.Offset = il.APIListObject.Offset + uint(previousLen)
98+
il, err = c.ListIncidents(options)
99+
if err != nil {
100+
return fmt.Errorf("failed listing incidents: %w", err)
101+
}
102+
previousLen = len(il.Incidents)
103+
for _, incident := range il.Incidents {
104+
if err := handler(incident); err != nil {
105+
return fmt.Errorf("handler failed: %w", err)
106+
}
107+
}
108+
}
109+
return nil
110+
}
111+
112+
// Alerts uses the provided client to retrieve all Alerts associated with the provided
113+
// incident, calling the provided handler function on each alert.
114+
func Alerts(c *pd.Client, incident pd.Incident, options pd.ListIncidentAlertsOptions, handler func(pd.IncidentAlert) error) error {
115+
var (
116+
il = new(pd.ListAlertsResponse)
117+
err error
118+
previousLen int
119+
)
120+
firstIteration := true
121+
for il.APIListObject.More || firstIteration {
122+
firstIteration = false
123+
options.APIListObject.Offset = il.APIListObject.Offset + uint(previousLen)
124+
il, err = c.ListIncidentAlertsWithOpts(incident.Id, options)
125+
if err != nil {
126+
return fmt.Errorf("failed listing alerts: %w", err)
127+
}
128+
previousLen = len(il.Alerts)
129+
for _, alert := range il.Alerts {
130+
if err := handler(alert); err != nil {
131+
return fmt.Errorf("handler failed: %w", err)
132+
}
133+
}
134+
}
135+
return nil
136+
}

pkg/e2e/e2e.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -699,6 +699,13 @@ func runTestsInPhase(phase string, description string, dryrun bool) bool {
699699
}
700700
}
701701
}
702+
// If we could have opened new alerts, consolidate them
703+
if os.Getenv("JOB_TYPE") == "periodic" {
704+
err := pagerduty.MergeCICDIncidents(pd.NewClient(viper.GetString(config.Alert.PagerDutyUserToken)))
705+
if err != nil {
706+
log.Printf("Failed merging PD incidents: %v", err)
707+
}
708+
}
702709

703710
passRate := float64(numPassingTests) / float64(numTests)
704711

0 commit comments

Comments
 (0)