diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index 317f03bc558..78e2033ceaf 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -386,29 +386,58 @@ func TestAlertmanagerSharding(t *testing.T) { assert.Equal(t, s3, ids[id3].Status.State) } - // Endpoint: GET /silences + // Endpoint: GET /v1/silences { for _, c := range clients { - list, err := c.GetSilences(context.Background()) + list, err := c.GetSilencesV1(context.Background()) require.NoError(t, err) assertSilences(list, types.SilenceStateActive, types.SilenceStateActive, types.SilenceStateActive) } } - // Endpoint: GET /silence/{id} + // Endpoint: GET /v2/silences { for _, c := range clients { - sil1, err := c.GetSilence(context.Background(), id1) + list, err := c.GetSilencesV2(context.Background()) + require.NoError(t, err) + assertSilences(list, types.SilenceStateActive, types.SilenceStateActive, types.SilenceStateActive) + } + } + + // Endpoint: GET /v1/silence/{id} + { + for _, c := range clients { + sil1, err := c.GetSilenceV1(context.Background(), id1) + require.NoError(t, err) + assert.Equal(t, comment(1), sil1.Comment) + assert.Equal(t, types.SilenceStateActive, sil1.Status.State) + + sil2, err := c.GetSilenceV1(context.Background(), id2) + require.NoError(t, err) + assert.Equal(t, comment(2), sil2.Comment) + assert.Equal(t, types.SilenceStateActive, sil2.Status.State) + + sil3, err := c.GetSilenceV1(context.Background(), id3) + require.NoError(t, err) + assert.Equal(t, comment(3), sil3.Comment) + assert.Equal(t, types.SilenceStateActive, sil3.Status.State) + } + } + + // Endpoint: GET /v2/silence/{id} + { + for _, c := range clients { + sil1, err := c.GetSilenceV2(context.Background(), id1) require.NoError(t, err) assert.Equal(t, comment(1), sil1.Comment) assert.Equal(t, types.SilenceStateActive, sil1.Status.State) - sil2, err := c.GetSilence(context.Background(), id2) + sil2, err := c.GetSilenceV2(context.Background(), id2) require.NoError(t, err) assert.Equal(t, comment(2), sil2.Comment) assert.Equal(t, types.SilenceStateActive, sil2.Status.State) - sil3, err := c.GetSilence(context.Background(), id3) + sil3, err := c.GetSilenceV2(context.Background(), id3) require.NoError(t, err) assert.Equal(t, comment(3), sil3.Comment) assert.Equal(t, types.SilenceStateActive, sil3.Status.State) @@ -445,7 +474,7 @@ func TestAlertmanagerSharding(t *testing.T) { require.NoError(t, waitForSilences("expired", 1*testCfg.replicationFactor)) for _, c := range clients { - list, err := c.GetSilences(context.Background()) + list, err := c.GetSilencesV2(context.Background()) require.NoError(t, err) assertSilences(list, types.SilenceStateActive, types.SilenceStateExpired, types.SilenceStateActive) } @@ -455,7 +484,7 @@ func TestAlertmanagerSharding(t *testing.T) { require.NoError(t, waitForSilences("expired", 2*testCfg.replicationFactor)) for _, c := range clients { - list, err := c.GetSilences(context.Background()) + list, err := c.GetSilencesV2(context.Background()) require.NoError(t, err) assertSilences(list, types.SilenceStateActive, types.SilenceStateExpired, types.SilenceStateExpired) } @@ -465,7 +494,7 @@ func TestAlertmanagerSharding(t *testing.T) { require.NoError(t, waitForSilences("expired", 3*testCfg.replicationFactor)) for _, c := range clients { - list, err := c.GetSilences(context.Background()) + list, err := c.GetSilencesV2(context.Background()) require.NoError(t, err) assertSilences(list, types.SilenceStateExpired, types.SilenceStateExpired, types.SilenceStateExpired) } diff --git a/integration/e2ecortex/client.go b/integration/e2ecortex/client.go index b4197f5812b..f09e9a59c57 100644 --- a/integration/e2ecortex/client.go +++ b/integration/e2ecortex/client.go @@ -651,7 +651,7 @@ func (c *Client) CreateSilence(ctx context.Context, silence types.Silence) (stri return decoded.Data.SilenceID, nil } -func (c *Client) GetSilences(ctx context.Context) ([]types.Silence, error) { +func (c *Client) GetSilencesV1(ctx context.Context) ([]types.Silence, error) { u := c.alertmanagerClient.URL("api/prom/api/v1/silences", nil) req, err := http.NewRequest(http.MethodGet, u.String(), nil) @@ -689,7 +689,36 @@ func (c *Client) GetSilences(ctx context.Context) ([]types.Silence, error) { return decoded.Data, nil } -func (c *Client) GetSilence(ctx context.Context, id string) (types.Silence, error) { +func (c *Client) GetSilencesV2(ctx context.Context) ([]types.Silence, error) { + u := c.alertmanagerClient.URL("api/prom/api/v2/silences", nil) + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + + resp, body, err := c.alertmanagerClient.Do(ctx, req) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusNotFound { + return nil, ErrNotFound + } + + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("getting silences failed with status %d and error %v", resp.StatusCode, string(body)) + } + + decoded := []types.Silence{} + if err := json.Unmarshal(body, &decoded); err != nil { + return nil, err + } + + return decoded, nil +} + +func (c *Client) GetSilenceV1(ctx context.Context, id string) (types.Silence, error) { u := c.alertmanagerClient.URL(fmt.Sprintf("api/prom/api/v1/silence/%s", url.PathEscape(id)), nil) req, err := http.NewRequest(http.MethodGet, u.String(), nil) @@ -727,6 +756,35 @@ func (c *Client) GetSilence(ctx context.Context, id string) (types.Silence, erro return decoded.Data, nil } +func (c *Client) GetSilenceV2(ctx context.Context, id string) (types.Silence, error) { + u := c.alertmanagerClient.URL(fmt.Sprintf("api/prom/api/v2/silence/%s", url.PathEscape(id)), nil) + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return types.Silence{}, fmt.Errorf("error creating request: %v", err) + } + + resp, body, err := c.alertmanagerClient.Do(ctx, req) + if err != nil { + return types.Silence{}, err + } + + if resp.StatusCode == http.StatusNotFound { + return types.Silence{}, ErrNotFound + } + + if resp.StatusCode/100 != 2 { + return types.Silence{}, fmt.Errorf("getting silence failed with status %d and error %v", resp.StatusCode, string(body)) + } + + decoded := types.Silence{} + if err := json.Unmarshal(body, &decoded); err != nil { + return types.Silence{}, err + } + + return decoded, nil +} + func (c *Client) DeleteSilence(ctx context.Context, id string) error { u := c.alertmanagerClient.URL(fmt.Sprintf("api/prom/api/v1/silence/%s", url.PathEscape(id)), nil) diff --git a/pkg/alertmanager/distributor.go b/pkg/alertmanager/distributor.go index 1d7d8f382dc..dba0dcf0589 100644 --- a/pkg/alertmanager/distributor.go +++ b/pkg/alertmanager/distributor.go @@ -94,12 +94,18 @@ func (d *Distributor) isQuorumReadPath(p string) (bool, merger.Merger) { if strings.HasSuffix(p, "/v2/alerts/groups") { return true, merger.V2AlertGroups{} } + if strings.HasSuffix(p, "/v2/silences") { + return true, merger.V2Silences{} + } + if strings.HasSuffix(path.Dir(p), "/v2/silence") { + return true, merger.V2SilenceID{} + } return false, nil } func (d *Distributor) isUnaryReadPath(p string) bool { - return strings.HasSuffix(p, "/silences") || - strings.HasSuffix(path.Dir(p), "/silence") || + return strings.HasSuffix(p, "/v1/silences") || + strings.HasSuffix(path.Dir(p), "/v1/silence") || strings.HasSuffix(p, "/status") || strings.HasSuffix(p, "/receivers") } diff --git a/pkg/alertmanager/distributor_test.go b/pkg/alertmanager/distributor_test.go index 4c2c2af6e4c..05457e299d9 100644 --- a/pkg/alertmanager/distributor_test.go +++ b/pkg/alertmanager/distributor_test.go @@ -119,14 +119,24 @@ func TestDistributor_DistributeRequest(t *testing.T) { headersNotPreserved: true, route: "/alerts/groups", }, { - name: "Read /silences is sent to only 1 AM", + name: "Read /v1/silences is sent to only 1 AM", numAM: 5, numHappyAM: 5, replicationFactor: 3, isRead: true, expStatusCode: http.StatusOK, expectedTotalCalls: 1, - route: "/silences", + route: "/v1/silences", + }, { + name: "Read /v2/silences is sent to 3 AMs", + numAM: 5, + numHappyAM: 5, + replicationFactor: 3, + isRead: true, + expStatusCode: http.StatusOK, + expectedTotalCalls: 3, + route: "/v2/silences", + responseBody: []byte(`[]`), }, { name: "Write /silences is sent to only 1 AM", numAM: 5, @@ -136,15 +146,26 @@ func TestDistributor_DistributeRequest(t *testing.T) { expectedTotalCalls: 1, route: "/silences", }, { - name: "Read /silence/id is sent to only 1 AM", + name: "Read /v1/silence/id is sent to only 1 AM", numAM: 5, numHappyAM: 5, replicationFactor: 3, isRead: true, expStatusCode: http.StatusOK, expectedTotalCalls: 1, - route: "/silence/id", + route: "/v1/silence/id", }, { + name: "Read /v2/silence/id is sent to 3 AMs", + numAM: 5, + numHappyAM: 5, + replicationFactor: 3, + isRead: true, + expStatusCode: http.StatusOK, + expectedTotalCalls: 3, + route: "/v2/silence/id", + responseBody: []byte(`{"id":"aaa","updatedAt":"2020-01-01T00:00:00Z"}`), + }, + { name: "Write /silence/id not supported", numAM: 5, numHappyAM: 5, diff --git a/pkg/alertmanager/merger/v2_silence_id.go b/pkg/alertmanager/merger/v2_silence_id.go new file mode 100644 index 00000000000..7718cb99e20 --- /dev/null +++ b/pkg/alertmanager/merger/v2_silence_id.go @@ -0,0 +1,34 @@ +package merger + +import ( + "errors" + + "github.com/go-openapi/swag" + v2_models "github.com/prometheus/alertmanager/api/v2/models" +) + +// V2SilenceID implements the Merger interface for GET /v2/silence/{id}. It returns the most +// recently updated silence (newest UpdatedAt timestamp). +type V2SilenceID struct{} + +func (V2SilenceID) MergeResponses(in [][]byte) ([]byte, error) { + silences := make(v2_models.GettableSilences, 0) + for _, body := range in { + parsed := &v2_models.GettableSilence{} + if err := swag.ReadJSON(body, parsed); err != nil { + return nil, err + } + silences = append(silences, parsed) + } + + merged, err := mergeV2Silences(silences) + if err != nil { + return nil, err + } + + if len(merged) != 1 { + return nil, errors.New("unexpected mismatched silence ids") + } + + return swag.WriteJSON(merged[0]) +} diff --git a/pkg/alertmanager/merger/v2_silence_id_test.go b/pkg/alertmanager/merger/v2_silence_id_test.go new file mode 100644 index 00000000000..6968e278ba0 --- /dev/null +++ b/pkg/alertmanager/merger/v2_silence_id_test.go @@ -0,0 +1,61 @@ +package merger + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestV2SilenceId_ReturnsNewestSilence(t *testing.T) { + + // We re-use MergeV2Silences so we rely on that being primarily tested elsewhere. + + in := [][]byte{ + []byte(`{"id":"77b580dd-1d9c-4b7e-9bba-13ac173cb4e5","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.215Z","comment":"This is the newest silence",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.215Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.725Z"}`), + []byte(`{"id":"77b580dd-1d9c-4b7e-9bba-13ac173cb4e5","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.000Z","comment":"Silence Comment #1",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.215Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.725Z"}`), + []byte(`{"id":"77b580dd-1d9c-4b7e-9bba-13ac173cb4e5","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.000Z","comment":"Silence Comment #1",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.215Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.725Z"}`), + } + + expected := []byte(`{"id":"77b580dd-1d9c-4b7e-9bba-13ac173cb4e5","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.215Z","comment":"This is the newest silence",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.215Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.725Z"}`) + + out, err := V2SilenceID{}.MergeResponses(in) + require.NoError(t, err) + require.Equal(t, string(expected), string(out)) +} + +func TestV2SilenceID_InvalidDifferentIDs(t *testing.T) { + + // Responses containing silences with different IDs is invalid input. + + in := [][]byte{ + []byte(`{"id":"77b580dd-1d9c-4b7e-9bba-13ac173cb4e5","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.215Z","comment":"Silence Comment #1",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.215Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.725Z"}`), + []byte(`{"id":"261248d1-4ff7-4cf1-9957-850c65f4e48b","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.082Z","comment":"Silence Comment #3",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.082Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.735Z"}`), + } + + _, err := V2SilenceID{}.MergeResponses(in) + require.Error(t, err) +} diff --git a/pkg/alertmanager/merger/v2_silences.go b/pkg/alertmanager/merger/v2_silences.go new file mode 100644 index 00000000000..f268e0b983d --- /dev/null +++ b/pkg/alertmanager/merger/v2_silences.go @@ -0,0 +1,65 @@ +package merger + +import ( + "errors" + "time" + + "github.com/go-openapi/swag" + v2 "github.com/prometheus/alertmanager/api/v2" + v2_models "github.com/prometheus/alertmanager/api/v2/models" +) + +// V2Silences implements the Merger interface for GET /v2/silences. It returns the union of silences +// over all the responses. When a silence with the same ID exists in multiple responses, the silence +// most recently updated silence is returned (newest UpdatedAt timestamp). +type V2Silences struct{} + +func (V2Silences) MergeResponses(in [][]byte) ([]byte, error) { + silences := make(v2_models.GettableSilences, 0) + for _, body := range in { + parsed := make(v2_models.GettableSilences, 0) + if err := swag.ReadJSON(body, &parsed); err != nil { + return nil, err + } + silences = append(silences, parsed...) + } + + merged, err := mergeV2Silences(silences) + if err != nil { + return nil, err + } + + return swag.WriteJSON(merged) +} + +func mergeV2Silences(in v2_models.GettableSilences) (v2_models.GettableSilences, error) { + // Select the most recently updated silences for each silence ID. + silences := make(map[string]*v2_models.GettableSilence) + for _, silence := range in { + if silence.ID == nil { + return nil, errors.New("unexpected nil id") + } + if silence.UpdatedAt == nil { + return nil, errors.New("unexpected nil updatedAt") + } + + key := *silence.ID + if current, ok := silences[key]; ok { + if time.Time(*silence.UpdatedAt).After(time.Time(*current.UpdatedAt)) { + silences[key] = silence + } + } else { + silences[key] = silence + } + } + + result := make(v2_models.GettableSilences, 0, len(silences)) + for _, silence := range silences { + result = append(result, silence) + } + + // Re-use Alertmanager sorting for silences. + v2.SortSilences(result) + + return result, nil +} diff --git a/pkg/alertmanager/merger/v2_silences_test.go b/pkg/alertmanager/merger/v2_silences_test.go new file mode 100644 index 00000000000..97b248077c6 --- /dev/null +++ b/pkg/alertmanager/merger/v2_silences_test.go @@ -0,0 +1,172 @@ +package merger + +import ( + "testing" + + v2_models "github.com/prometheus/alertmanager/api/v2/models" + "github.com/stretchr/testify/require" +) + +func TestV2Silences(t *testing.T) { + + // This test is to check the parsing round-trip is working as expected, the merging logic is + // tested in TestMergeV2Silences. The test data is based on captures from an actual Alertmanager. + + in := [][]byte{ + []byte(`[` + + `{"id":"77b580dd-1d9c-4b7e-9bba-13ac173cb4e5","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.215Z","comment":"Silence Comment #1",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.215Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.725Z"},` + + `{"id":"261248d1-4ff7-4cf1-9957-850c65f4e48b","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.082Z","comment":"Silence Comment #3",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.082Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.735Z"}` + + `]`), + []byte(`[` + + `{"id":"17526003-c745-4464-a355-4f06de26a236","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:01.953Z","comment":"Silence Comment #2",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:01.953Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.731Z"}` + + `]`), + []byte(`[]`), + } + + expected := []byte(`[` + + `{"id":"77b580dd-1d9c-4b7e-9bba-13ac173cb4e5","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.215Z","comment":"Silence Comment #1",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.215Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.725Z"},` + + `{"id":"261248d1-4ff7-4cf1-9957-850c65f4e48b","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:02.082Z","comment":"Silence Comment #3",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:02.082Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.735Z"},` + + `{"id":"17526003-c745-4464-a355-4f06de26a236","status":{"state":"expired"},` + + `"updatedAt":"2021-04-28T17:31:01.953Z","comment":"Silence Comment #2",` + + `"createdBy":"","endsAt":"2021-04-28T17:31:01.953Z","matchers":` + + `[{"isEqual":true,"isRegex":false,"name":"instance","value":"prometheus-one"}],` + + `"startsAt":"2021-04-28T17:31:01.731Z"}` + + `]`) + + out, err := V2Silences{}.MergeResponses(in) + require.NoError(t, err) + require.Equal(t, string(expected), string(out)) +} + +// v2silence is a convenience function to create silence structures with certain important fields set +// and with sensible defaults for the remaining fields to test they are passed through. +func v2silence(id, endsAt, updatedAt string) *v2_models.GettableSilence { + var ( + active = v2_models.SilenceStatusStateActive + comment = "test" + createdBy = "someone" + isEqual = true + isRegex = false + name = "foo" + value = "bar" + ) + return &v2_models.GettableSilence{ + ID: &id, + Status: &v2_models.SilenceStatus{ + State: &active, + }, + UpdatedAt: v2ParseTime(updatedAt), + Silence: v2_models.Silence{ + Comment: &comment, + CreatedBy: &createdBy, + EndsAt: v2ParseTime(endsAt), + Matchers: v2_models.Matchers{ + &v2_models.Matcher{ + IsEqual: &isEqual, + IsRegex: &isRegex, + Name: &name, + Value: &value, + }, + }, + StartsAt: v2ParseTime("2020-01-01T12:00:00.000Z"), + }, + } +} + +func v2silences(silences ...*v2_models.GettableSilence) v2_models.GettableSilences { + return silences +} + +func TestMergeV2Silences(t *testing.T) { + var ( + silence1 = v2silence("id1", "2020-01-01T12:11:11.000Z", "2020-01-01T12:00:00.000Z") + newerSilence1 = v2silence("id1", "2020-01-01T12:11:11.000Z", "2020-01-01T12:00:00.001Z") + silence2 = v2silence("id2", "2020-01-01T12:22:22.000Z", "2020-01-01T12:00:00.000Z") + silence3 = v2silence("id3", "2020-01-01T12:33:33.000Z", "2020-01-01T12:00:00.000Z") + ) + cases := []struct { + name string + in v2_models.GettableSilences + err error + out v2_models.GettableSilences + }{ + { + name: "no silences, should return an empty list", + in: v2silences(), + out: v2_models.GettableSilences{}, + }, + { + name: "one silence, should return the silence", + in: v2silences(silence1), + out: v2silences(silence1), + }, + { + name: "two silences, should return two silences", + in: v2silences(silence1, silence2), + out: v2silences(silence1, silence2), + }, + { + name: "three silences, should return three silences", + in: v2silences(silence1, silence2, silence3), + out: v2silences(silence1, silence2, silence3), + }, + { + name: "three active silences out of order, should return three silences in expiry order", + in: v2silences(silence3, silence2, silence1), + out: v2silences(silence1, silence2, silence3), + }, + { + name: "two identical silences, should return one silence", + in: v2silences(silence1, silence1), + out: v2silences(silence1), + }, + { + name: "two identical silences plus another, should return two silences", + in: v2silences(silence1, silence1, silence2), + out: v2silences(silence1, silence2), + }, + { + name: "two duplicates out of sync silences, should return newer silence", + in: v2silences(silence1, newerSilence1), + out: v2silences(newerSilence1), + }, + { + name: "two duplicates out of sync silences (newer first), should return newer silence", + in: v2silences(newerSilence1, silence1), + out: v2silences(newerSilence1), + }, + { + name: "two duplicates plus others, should return newer silence and others", + in: v2silences(newerSilence1, silence3, silence1, silence2), + out: v2silences(newerSilence1, silence2, silence3), + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + out, err := mergeV2Silences(c.in) + require.Equal(t, c.err, err) + require.Equal(t, c.out, out) + }) + } +}