Disable access to Alertmanager (user, or sending alerts), if it is not running.

CHANGELOG.md

@@ -54,6 +54,7 @@
 * [BUGFIX] Alertmanager: disable access to `/alertmanager/metrics` (which exposes all Cortex metrics), `/alertmanager/-/reload` and `/alertmanager/debug/*`, which were available to any authenticated user with enabled AlertManager. #3678
 * [BUGFIX] Query-Frontend: avoid creating many small sub-queries by discarding cache extents under 5 minutes #3653
 * [BUGFIX] Ruler: Ensure the stale markers generated for evaluated rules respect the configured `-ruler.evaluation-delay-duration`. This will avoid issues with samples with NaN be persisted with timestamps set ahead of the next rule evaluation. #3687
+* [BUGFIX] Alertmanager: don't serve HTTP requests until Alertmanager has fully started. Serving HTTP requests earlier may result in loss of configuration for the user. #3679
 
 ## 1.6.0
 

pkg/alertmanager/multitenant.go

@@ -283,9 +283,11 @@ func (am *MultitenantAlertmanager) stopping(_ error) error {
 		am.Stop()
 	}
 	am.alertmanagersMtx.Unlock()
-	err := am.peer.Leave(am.cfg.PeerTimeout)
-	if err != nil {
-		level.Warn(am.logger).Log("msg", "failed to leave the cluster", "err", err)
+	if am.peer != nil { // Tests don't setup any peer.
+		err := am.peer.Leave(am.cfg.PeerTimeout)
+		if err != nil {
+			level.Warn(am.logger).Log("msg", "failed to leave the cluster", "err", err)
+		}
 	}
 	level.Debug(am.logger).Log("msg", "stopping")
 	return nil
@@ -470,6 +472,11 @@ func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amco
 
 // ServeHTTP serves the Alertmanager's web UI and API.
 func (am *MultitenantAlertmanager) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	if am.State() != services.Running {
+		http.Error(w, "Alertmanager not ready", http.StatusServiceUnavailable)
+		return
+	}
+
 	userID, err := tenant.TenantID(req.Context())
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusUnauthorized)

pkg/alertmanager/multitenant_test.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/go-kit/kit/log"
 	"github.com/prometheus/client_golang/prometheus"
@@ -21,6 +22,7 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/alertmanager/alerts"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
+	"github.com/cortexproject/cortex/pkg/util/services"
 )
 
 var (
@@ -210,10 +212,14 @@ func TestAlertmanager_ServeHTTP(t *testing.T) {
 	// Create the Multitenant Alertmanager.
 	reg := prometheus.NewPedanticRegistry()
 	am := createMultitenantAlertmanager(&MultitenantAlertmanagerConfig{
-		ExternalURL: externalURL,
-		DataDir:     tempDir,
+		ExternalURL:  externalURL,
+		DataDir:      tempDir,
+		PollInterval: time.Minute,
 	}, nil, nil, mockStore, log.NewNopLogger(), reg)
 
+	require.NoError(t, services.StartAndAwaitRunning(context.Background(), am))
+	defer services.StopAndAwaitTerminated(context.Background(), am) //nolint:errcheck
+
 	// Request when no user configuration is present.
 	req := httptest.NewRequest("GET", externalURL.String(), nil)
 	ctx := user.InjectOrgID(req.Context(), "user1")
@@ -321,11 +327,15 @@ receivers:
 
 	// Create the Multitenant Alertmanager.
 	am := createMultitenantAlertmanager(&MultitenantAlertmanagerConfig{
-		ExternalURL: externalURL,
-		DataDir:     tempDir,
+		ExternalURL:  externalURL,
+		DataDir:      tempDir,
+		PollInterval: time.Minute,
 	}, nil, nil, mockStore, log.NewNopLogger(), nil)
 	am.fallbackConfig = fallbackCfg
 
+	require.NoError(t, services.StartAndAwaitRunning(context.Background(), am))
+	defer services.StopAndAwaitTerminated(context.Background(), am) //nolint:errcheck
+
 	// Request when no user configuration is present.
 	req := httptest.NewRequest("GET", externalURL.String()+"/api/v1/status", nil)
 	ctx := user.InjectOrgID(req.Context(), "user1")