diff --git a/CHANGELOG.md b/CHANGELOG.md index 449dcd342d11..f0555e54eb24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,11 @@ `loki_tag` is now `logs_instance_tag`, and `backend: loki` is now `backend: logs_instance`. (@rfratto) + +# v0.18.1 (2021-08-09) + +- [BUGFIX] Reduce number of consul calls when ran in scrape service mode (@mattdurham) + # v0.18.0 (2021-07-29) - [FEATURE] Added [Github exporter](https://github.com/infinityworks/github-exporter) integration. (@rgeyer) diff --git a/docs/configuration/integrations/process-exporter-config.md b/docs/configuration/integrations/process-exporter-config.md index ad5a43a9ef00..00411a5953c1 100644 --- a/docs/configuration/integrations/process-exporter-config.md +++ b/docs/configuration/integrations/process-exporter-config.md @@ -35,7 +35,7 @@ metadata: name: agent spec: containers: - - image: grafana/agent:v0.16.1 + - image: grafana/agent:v0.18.1 name: agent args: - --config.file=/etc/agent-config/agent.yaml diff --git a/docs/getting-started/_index.md b/docs/getting-started/_index.md index b401f05db752..80ed1d0fa09e 100644 --- a/docs/getting-started/_index.md +++ b/docs/getting-started/_index.md @@ -24,7 +24,7 @@ Currently, there are six ways to install the agent: docker run \ -v /tmp/agent:/etc/agent/data \ -v /path/to/config.yaml:/etc/agent/agent.yaml \ - grafana/agent:v0.16.1 + grafana/agent:v0.18.1 ``` Replace `/tmp/agent` with the folder you wish to store WAL data in. WAL data is diff --git a/docs/operator/getting-started.md b/docs/operator/getting-started.md index 2d71dd5f5995..5a583fffd26c 100644 --- a/docs/operator/getting-started.md +++ b/docs/operator/getting-started.md @@ -62,7 +62,7 @@ spec: serviceAccountName: grafana-agent-operator containers: - name: operator - image: grafana/agent-operator:v0.16.1 + image: grafana/agent-operator:v0.18.1 --- apiVersion: v1 diff --git a/go.mod b/go.mod index 6e75fb565329..e0623fb1d0ee 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/grafana/loki v1.6.2-0.20210429132126-d88f3996eaa2 github.com/hashicorp/consul/api v1.8.1 + github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-getter v1.5.3 github.com/infinityworks/github-exporter v0.0.0-20201016091012-831b72461034 github.com/jsternberg/zap-logfmt v1.2.0 diff --git a/pkg/operator/defaults.go b/pkg/operator/defaults.go index 7ae99d639de1..ac0a7a8db3f5 100644 --- a/pkg/operator/defaults.go +++ b/pkg/operator/defaults.go @@ -7,6 +7,9 @@ var ( "v0.15.0", // "v0.16.0", // Pulled due to critical bug fixed in v0.16.1. "v0.16.1", + "v0.17.0", + "v0.18.0", + "v0.18.1", // NOTE(rfratto): when performing an upgrade, add the newest version above instead of changing the existing reference. } diff --git a/pkg/prom/instance/configstore/remote.go b/pkg/prom/instance/configstore/remote.go index 7b29cb0faca6..1e583abad9d2 100644 --- a/pkg/prom/instance/configstore/remote.go +++ b/pkg/prom/instance/configstore/remote.go @@ -2,10 +2,20 @@ package configstore import ( "context" + "errors" "fmt" + "net/http" "strings" "sync" + "github.com/weaveworks/common/instrument" + + "github.com/cortexproject/cortex/pkg/ring/kv/consul" + + "github.com/hashicorp/go-cleanhttp" + + "github.com/hashicorp/consul/api" + "github.com/cortexproject/cortex/pkg/ring/kv" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -14,6 +24,20 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +/*********************************************************************************************************************** +The consul code skipping the cortex handler is due to performance issue with a large number of configs and overloading +consul. See issue https://github.com/grafana/agent/issues/789. The long term method will be to refactor and extract +the cortex code so other stores can also benefit from this. @mattdurham +***********************************************************************************************************************/ + +// This is copied from cortex code so that stats stay the same +var consulRequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "consul_request_duration_seconds", + Help: "Time spent on consul requests.", + Buckets: prometheus.DefBuckets, +}, []string{"operation", "status_code"})) + // Remote loads instance files from a remote KV store. The KV store // can be swapped out in real time. type Remote struct { @@ -21,7 +45,7 @@ type Remote struct { reg *util.Unregisterer kvMut sync.RWMutex - kv kv.Client + kv *agentRemoteClient reloadKV chan struct{} cancelCtx context.Context @@ -31,6 +55,14 @@ type Remote struct { configsCh chan WatchEvent } +// agentRemoteClient is a simple wrapper to allow the shortcircuit of consul, while being backwards compatible with non +// consul kv stores +type agentRemoteClient struct { + kv.Client + consul *api.Client + consulConfig consul.Config +} + // NewRemote creates a new Remote store that uses a Key-Value client to store // and retrieve configs. If enable is true, the store will be immediately // connected to. Otherwise, it can be lazily loaded by enabling later through @@ -70,23 +102,48 @@ func (r *Remote) ApplyConfig(cfg kv.Config, enable bool) error { r.reg.UnregisterAll() if !enable { - r.setClient(nil) + r.setClient(nil, nil) return nil } cli, err := kv.NewClient(cfg, GetCodec(), kv.RegistererWithKVName(r.reg, "agent_configs")) + // This is a hack to get a consul client, the client above has it embedded but its not exposed + var consulClient *api.Client + if cfg.Store == "consul" { + consulClient, err = api.NewClient(&api.Config{ + Address: cfg.Consul.Host, + Token: cfg.Consul.ACLToken, + Scheme: "http", + HttpClient: &http.Client{ + Transport: cleanhttp.DefaultPooledTransport(), + // See https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/ + Timeout: cfg.Consul.HTTPClientTimeout, + }, + }) + if err != nil { + return err + } + + } if err != nil { return fmt.Errorf("failed to create kv client: %w", err) } - r.setClient(cli) + r.setClient(cli, consulClient) return nil } // setClient sets the active client and notifies run to restart the // kv watcher. -func (r *Remote) setClient(client kv.Client) { - r.kv = client +func (r *Remote) setClient(client kv.Client, consulClient *api.Client) { + if client == nil && consulClient == nil { + r.kv = nil + } else { + r.kv = &agentRemoteClient{ + Client: client, + consul: consulClient, + } + } r.reloadKV <- struct{}{} } @@ -119,7 +176,7 @@ Outer: } } -func (r *Remote) watchKV(ctx context.Context, client kv.Client) { +func (r *Remote) watchKV(ctx context.Context, client *agentRemoteClient) { // Edge case: client was unset, nothing to do here. if client == nil { level.Info(r.log).Log("msg", "not watching the KV, none set") @@ -162,6 +219,36 @@ func (r *Remote) List(ctx context.Context) ([]string, error) { return r.kv.List(ctx, "") } +// listConsul returns Key Value Pairs instead of []string +func (r *Remote) listConsul(ctx context.Context) (api.KVPairs, error) { + if r.kv == nil { + return nil, ErrNotConnected + } + + var pairs api.KVPairs + options := &api.QueryOptions{ + AllowStale: !r.kv.consulConfig.ConsistentReads, + RequireConsistent: r.kv.consulConfig.ConsistentReads, + } + // This is copied from cortex list so that stats stay the same + err := instrument.CollectedRequest(ctx, "List", consulRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { + var err error + // This mirrors the default prefix in cortex + pairs, _, err = r.kv.consul.KV().List("configurations/", options.WithContext(ctx)) + return err + }) + + if err != nil { + return nil, err + } + // This mirrors the previous behavior of returning a blank array as opposed to nil. + if pairs == nil { + blankPairs := make(api.KVPairs, 0) + return blankPairs, nil + } + return pairs, nil +} + // Get retrieves an individual config from the KV store. func (r *Remote) Get(ctx context.Context, key string) (instance.Config, error) { r.kvMut.RLock() @@ -260,6 +347,69 @@ func (r *Remote) all(ctx context.Context, keep func(key string) bool) (<-chan in return nil, ErrNotConnected } + // If we are using a consul client then do the short circuit way, this is done so that we receive all the key value pairs + // in one call then, operate on them in memory. Previously we retrieved the list (which stripped the values) + // then ran a goroutine to get each individual value from consul. In situations with an extremely large number of + // configs this overloaded the consul instances. This reduces that to one call, that was being made anyways. + if r.kv.consul != nil { + return r.allConsul(ctx, keep) + } + return r.allOther(ctx, keep) + +} + +// allConsul is ONLY usable when consul is the keystore. This is a performance improvement in using the client directly +// instead of the cortex multi store kv interface. That interface returns the list then each value must be retrieved +// individually. This returns all the keys and values in one call and works on them in memory +func (r *Remote) allConsul(ctx context.Context, keep func(key string) bool) (<-chan instance.Config, error) { + if r.kv.consul == nil { + level.Error(r.log).Log("err", "allConsul called but consul client nil") + return nil, errors.New("allConsul called but consul client nil") + } + var configs []*instance.Config + c := GetCodec() + + pairs, err := r.listConsul(ctx) + + if err != nil { + return nil, err + } + for _, kvp := range pairs { + if keep != nil && !keep(kvp.Key) { + level.Debug(r.log).Log("msg", "skipping key that was filtered out", "key", kvp.Key) + continue + } + value, err := c.Decode(kvp.Value) + if err != nil { + level.Error(r.log).Log("msg", "failed to decode config from store", "key", kvp.Key, "err", err) + continue + } + if value == nil { + // Config was deleted since we called list, skip it. + level.Debug(r.log).Log("msg", "skipping key that was deleted after list was called", "key", kvp.Key) + continue + } + + cfg, err := instance.UnmarshalConfig(strings.NewReader(value.(string))) + if err != nil { + level.Error(r.log).Log("msg", "failed to unmarshal config from store", "key", kvp.Key, "err", err) + continue + } + configs = append(configs, cfg) + } + ch := make(chan instance.Config, len(configs)) + for _, cfg := range configs { + ch <- *cfg + } + close(ch) + return ch, nil +} + +func (r *Remote) allOther(ctx context.Context, keep func(key string) bool) (<-chan instance.Config, error) { + if r.kv == nil { + return nil, ErrNotConnected + } + keys, err := r.kv.List(ctx, "") if err != nil { return nil, fmt.Errorf("failed to list configs: %w", err) diff --git a/production/README.md b/production/README.md index 0a62b6a5471a..82b37fa3994c 100644 --- a/production/README.md +++ b/production/README.md @@ -26,7 +26,7 @@ directory on your host that you want the agent to store its WAL. docker run \ -v /tmp/agent:/etc/agent/data \ -v /path/to/config.yaml:/etc/agent/agent.yaml \ - grafana/agent:v0.16.1 + grafana/agent:v0.18.1 ``` ## Running the Agent locally diff --git a/production/grafanacloud-install.sh b/production/grafanacloud-install.sh index 8fe9b5096443..433b8718e78a 100755 --- a/production/grafanacloud-install.sh +++ b/production/grafanacloud-install.sh @@ -47,7 +47,7 @@ PACKAGE_SYSTEM=${PACKAGE_SYSTEM:=} # # Global constants. # -RELEASE_VERSION="0.18.0" +RELEASE_VERSION="0.18.1" RELEASE_URL="https://github.com/grafana/agent/releases/download/v${RELEASE_VERSION}" DEB_URL="${RELEASE_URL}/grafana-agent-${RELEASE_VERSION}-1.${ARCH}.deb" diff --git a/production/kubernetes/agent-bare.yaml b/production/kubernetes/agent-bare.yaml index 63e970d83040..61f4fa881555 100644 --- a/production/kubernetes/agent-bare.yaml +++ b/production/kubernetes/agent-bare.yaml @@ -66,7 +66,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: grafana/agent:v0.16.1 + image: grafana/agent:v0.18.1 imagePullPolicy: IfNotPresent name: agent ports: diff --git a/production/kubernetes/agent-loki.yaml b/production/kubernetes/agent-loki.yaml index cddee33ae8a0..cd59a23f2f05 100644 --- a/production/kubernetes/agent-loki.yaml +++ b/production/kubernetes/agent-loki.yaml @@ -64,7 +64,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: grafana/agent:v0.16.1 + image: grafana/agent:v0.18.1 imagePullPolicy: IfNotPresent name: agent ports: diff --git a/production/kubernetes/agent-tempo.yaml b/production/kubernetes/agent-tempo.yaml index bab61d6fe8ee..80b11e4c46bf 100644 --- a/production/kubernetes/agent-tempo.yaml +++ b/production/kubernetes/agent-tempo.yaml @@ -109,7 +109,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: grafana/agent:v0.16.1 + image: grafana/agent:v0.18.1 imagePullPolicy: IfNotPresent name: agent ports: diff --git a/production/kubernetes/build/lib/version.libsonnet b/production/kubernetes/build/lib/version.libsonnet index 97cf3e42de1f..e49b3e4875ec 100644 --- a/production/kubernetes/build/lib/version.libsonnet +++ b/production/kubernetes/build/lib/version.libsonnet @@ -1 +1 @@ -'grafana/agent:v0.16.1' +'grafana/agent:v0.18.1' diff --git a/production/kubernetes/install-bare.sh b/production/kubernetes/install-bare.sh index e46310e895db..b8701407d031 100644 --- a/production/kubernetes/install-bare.sh +++ b/production/kubernetes/install-bare.sh @@ -25,7 +25,7 @@ check_installed() { check_installed curl check_installed envsubst -MANIFEST_BRANCH=v0.16.1 +MANIFEST_BRANCH=v0.18.1 MANIFEST_URL=${MANIFEST_URL:-https://raw.githubusercontent.com/grafana/agent/${MANIFEST_BRANCH}/production/kubernetes/agent-bare.yaml} NAMESPACE=${NAMESPACE:-default} diff --git a/production/tanka/grafana-agent/v1/main.libsonnet b/production/tanka/grafana-agent/v1/main.libsonnet index de541e61a334..920c1ba2f633 100644 --- a/production/tanka/grafana-agent/v1/main.libsonnet +++ b/production/tanka/grafana-agent/v1/main.libsonnet @@ -15,8 +15,8 @@ local service = k.core.v1.service; (import './lib/tempo.libsonnet') + { _images:: { - agent: 'grafana/agent:v0.16.1', - agentctl: 'grafana/agentctl:v0.16.1', + agent: 'grafana/agent:v0.18.1', + agentctl: 'grafana/agentctl:v0.18.1', }, // new creates a new DaemonSet deployment of the grafana-agent. By default,