Support Alertmanager HA

With this, we can now support increasing the number of replicas for a
Cortex AM thus enabling HA.

 Please note that Alerts themselves are not gossiped between
Alertmanagers. Each Ruler needs to send the alert to every Alertmanager
available thus the reason why a headless service gets created when the
number of replicas is more than 1.

Author: gotjosh

cortex/alertmanager.libsonnet

@@ -4,15 +4,23 @@
   local container = $.core.v1.container,
   local statefulSet = $.apps.v1.statefulSet,
   local service = $.core.v1.service,
+  local isGossiping = $._config.alertmanager.replicas > 1,
+  local peers = if isGossiping then
+    [
+      'alertmanager-%d.alertmanager.%s.svc.%s.local:%s' % [i, $._config.namespace, $._config.cluster, $._config.alertmanager_gossip_port]
+      for i in std.range(0, $._config.alertmanager.replicas - 1)
+    ]
+  else [],
 
   alertmanager_args::
     {
       target: 'alertmanager',
       'log.level': 'debug',
 
+      'experimental.alertmanager.enable-api': 'true',
       'alertmanager.storage.type': 'gcs',
       'alertmanager.storage.path': '/data',
-      'alertmanager.gcs.bucketname': '%(cluster)s-cortex-configdb-%(namespace)s' % $._config,
+      'alertmanager.storage.gcs.bucketname': '%(cluster)s-cortex-%(namespace)s' % $._config,
       'alertmanager.web.external-url': '%s/alertmanager' % $._config.external_url,
     },
 
@@ -27,8 +35,22 @@
   alertmanager_container::
     if $._config.alertmanager_enabled then
       container.new('alertmanager', $._images.alertmanager) +
-      container.withPorts($.util.defaultPorts) +
-      container.withArgsMixin($.util.mapToFlags($.alertmanager_args)) +
+      container.withPorts(
+        $.util.defaultPorts +
+        if isGossiping then [
+          $.core.v1.containerPort.newUDP('gossip-udp', $._config.alertmanager_gossip_port),
+          $.core.v1.containerPort.new('gossip-tcp', $._config.alertmanager_gossip_port),
+        ]
+        else [],
+      ) +
+      container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) +
+      container.withArgsMixin(
+        $.util.mapToFlags($.alertmanager_args) +
+        if isGossiping then
+          ['--cluster.listen-address=[$(POD_IP)]:%s' % $._config.alertmanager_gossip_port] +
+          ['--cluster.peer=%s' % peer for peer in peers]
+        else [],
+      ) +
       container.withVolumeMountsMixin([volumeMount.new('alertmanager-data', '/data')]) +
       $.util.resourcesRequests('100m', '1Gi') +
       $.util.readinessProbe +
@@ -37,7 +59,7 @@
 
   alertmanager_statefulset:
     if $._config.alertmanager_enabled then
-      statefulSet.new('alertmanager', 1, [$.alertmanager_container], $.alertmanager_pvc) +
+      statefulSet.new('alertmanager', $._config.alertmanager.replicas, [$.alertmanager_container], $.alertmanager_pvc) +
       statefulSet.mixin.spec.withServiceName('alertmanager') +
       statefulSet.mixin.metadata.withNamespace($._config.namespace) +
       statefulSet.mixin.metadata.withLabels({ name: 'alertmanager' }) +
@@ -50,6 +72,10 @@
 
   alertmanager_service:
     if $._config.alertmanager_enabled then
-      $.util.serviceFor($.alertmanager_statefulset)
+      if $._config.alertmanager.replicas > 1 then
+        $.util.serviceFor($.alertmanager_statefulset) +
+        service.mixin.spec.withClusterIp('None')
+      else
+        $.util.serviceFor($.alertmanager_statefulset)
     else {},
 }

cortex/config.libsonnet

@@ -227,6 +227,10 @@
         },
       }[$._config.ruler_client_type],
 
+    alertmanager: {
+      replicas: 1,
+    },
+
     overrides: {
       // === Per-tenant usage limits. ===
       //

cortex/images.libsonnet

@@ -18,8 +18,7 @@
     store_gateway: self.cortex,
 
     query_tee: 'quay.io/cortexproject/query-tee:master-5d7b05c3',
-    // TODO(gouthamve/jtlisi): Upstream the ruler and AM configs.
-    alertmanager: 'jtlisi/cortex:20190819_alertmanager_update-faa66aa43',
+    alertmanager: 'quay.io/cortexproject/cortex:master-2b41aa38d',
     testExporter: 'cortexproject/test-exporter:master-be013707',
   },
 }