Skip to content

Commit

Permalink
Squashed commit of the following:
Browse files Browse the repository at this point in the history
commit 0bfd0ad
Merge: 68aa188 efdae3d
Author: Trevor Whitney <[email protected]>
Date:   Thu May 23 17:04:32 2024 -0600

    Merge branch 'main' into sample-count-and-bytes

commit 68aa188
Author: Trevor Whitney <[email protected]>
Date:   Thu May 23 17:03:32 2024 -0600

    feat: guard aggregation behavior behind a feature flag

commit efdae3d
Author: hayden <[email protected]>
Date:   Thu May 23 16:25:50 2024 -0400

    feat(helm): Support for PVC Annotations for Non-Distributed Modes (#12023)

    Signed-off-by: hfuss <[email protected]>
    Co-authored-by: J Stickler <[email protected]>
    Co-authored-by: Trevor Whitney <[email protected]>

commit f0d6a92
Author: Trevor Whitney <[email protected]>
Date:   Thu May 23 14:03:32 2024 -0600

    feat: reject filter queries to /patterns endpoint

commit dc620e7
Author: Trevor Whitney <[email protected]>
Date:   Wed May 8 14:08:44 2024 -0600

    feat: collect and serve pre-agg bytes and count

    * pre-aggregate bytes and count per stream in the pattern ingester
    * serve bytes_over_time and count_over_time queries from the patterns
      endpoint

commit 97212ea
Author: Jay Clifford <[email protected]>
Date:   Thu May 23 12:10:48 2024 -0400

    feat: Added Interactive Sandbox to Quickstart tutorial (#12701)

commit 1111595
Author: Vladyslav Diachenko <[email protected]>
Date:   Thu May 23 13:18:16 2024 +0300

    feat: new stream count limiter (#13006)

    Signed-off-by: Vladyslav Diachenko <[email protected]>
    Co-authored-by: JordanRushing <[email protected]>

commit 987e551
Author: Quentin Bisson <[email protected]>
Date:   Thu May 23 02:15:52 2024 +0200

    fix: allow cluster label override in bloom dashboards (#13012)

    Signed-off-by: QuentinBisson <[email protected]>

commit d3c9cec
Author: Quentin Bisson <[email protected]>
Date:   Thu May 23 01:59:28 2024 +0200

    fix: upgrade old plugin for the loki-operational dashboard. (#13016)

    Signed-off-by: QuentinBisson <[email protected]>

commit 8d9fb68
Author: Quentin Bisson <[email protected]>
Date:   Wed May 22 22:00:08 2024 +0200

    fix: remove unneccessary disk panels for ssd read path (#13014)

    Signed-off-by: QuentinBisson <[email protected]>

commit 1948899
Author: Quentin Bisson <[email protected]>
Date:   Wed May 22 15:16:29 2024 +0200

    fix: Mixins - Add missing log datasource on loki-deletion (#13011)

commit efd8f5d
Author: Salva Corts <[email protected]>
Date:   Wed May 22 10:43:32 2024 +0200

    refactor(blooms): Add queue to bloom planner and enqueue tasks (#13005)

commit d6f29fc
Author: Vitor Gomes <[email protected]>
Date:   Wed May 22 04:34:42 2024 +1200

    docs: update otlp ingestion with correct endpoint and add endpoint to reference api docs (#12996)

commit 3195036
Author: Salva Corts <[email protected]>
Date:   Tue May 21 13:12:24 2024 +0200

    refactor(bloom planner): Compute gaps and build tasks from metas and TSDBs  (#12994)

commit 7a3338e
Author: Jonathan Davies <[email protected]>
Date:   Tue May 21 10:41:42 2024 +0100

    feat: loki/main.go: Log which config file path is used on startup (#12985)

    Co-authored-by: Michel Hollands <[email protected]>

commit bf8a278
Author: Ashwanth <[email protected]>
Date:   Tue May 21 12:56:07 2024 +0530

    chore: remove duplicate imports (#13001)

commit 1f5291a
Author: Ashwanth <[email protected]>
Date:   Tue May 21 12:38:02 2024 +0530

    fix(indexstats): do not collect stats from "IndexStats" lookups for other query types (#12978)

commit 8442dca
Author: Jay Clifford <[email protected]>
Date:   Mon May 20 17:52:17 2024 -0400

    feat: Added getting started video (#12975)

commit 75ccf21
Author: Christian Haudum <[email protected]>
Date:   Mon May 20 17:14:40 2024 +0200

    feat(blooms): Separate page buffer pools for series pages and bloom pages (#12992)

    Series pages are much smaller than bloom pages and therefore can make use of a separate buffer pool with different buckets.

    The second commit fixes a possible panic.

    Signed-off-by: Christian Haudum <[email protected]>

commit 94d610e
Author: Yarden Shoham <[email protected]>
Date:   Mon May 20 18:05:50 2024 +0300

    docs: Fix broken link in the release notes (#12990)

    Co-authored-by: J Stickler <[email protected]>

commit 31a1314
Author: choeffer <[email protected]>
Date:   Mon May 20 16:39:25 2024 +0200

    docs(install-monolithic): add quotation marks (#12982)

    Co-authored-by: Michel Hollands <[email protected]>

commit 8978ecf
Author: Salva Corts <[email protected]>
Date:   Mon May 20 12:36:22 2024 +0200

    feat: Boilerplate for new bloom build planner and worker components. (#12989)
  • Loading branch information
trevorwhitney committed May 24, 2024
1 parent 5b80095 commit d9e8b48
Show file tree
Hide file tree
Showing 94 changed files with 6,356 additions and 793 deletions.
3 changes: 3 additions & 0 deletions cmd/loki/loki-local-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ schema_config:
prefix: index_
period: 24h

pattern_ingester:
enabled: true

ruler:
alertmanager_url: http://localhost:9093

Expand Down
1 change: 1 addition & 0 deletions cmd/loki/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ func main() {
}

level.Info(util_log.Logger).Log("msg", "Starting Loki", "version", version.Info())
level.Info(util_log.Logger).Log("msg", "Loading configuration file", "filename", config.ConfigFile)

err = t.Run(loki.RunOpts{StartTime: startTime})
util_log.CheckFatal("running loki", err, util_log.Logger)
Expand Down
1 change: 1 addition & 0 deletions docs/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ include docs.mk
PODMAN := $(shell if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)
BUILD_IN_CONTAINER ?= true

.PHONY: sources/setup/install/helm/reference.md
sources/setup/install/helm/reference.md: ../production/helm/loki/reference.md.gotmpl ../production/helm/loki/values.yaml
ifeq ($(BUILD_IN_CONTAINER),true)
$(PODMAN) run --rm --volume "$(realpath ..):/helm-docs" -u "$$(id -u)" "docker.io/jnorwood/helm-docs:v1.11.0" \
Expand Down
2 changes: 2 additions & 0 deletions docs/sources/get-started/_index.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ description: Provides an overview of the steps for implementing Grafana Loki to

# Get started with Grafana Loki

{{< youtube id="1uk8LtQqsZQ" >}}

Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is designed to be very cost effective and easy to operate. It does not index the contents of the logs, but rather a set of labels for each log stream.

Because all Loki implementations are unique, the installation process is
Expand Down
12 changes: 12 additions & 0 deletions docs/sources/get-started/quick-start.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,18 @@ The Docker Compose configuration instantiates the following components, each in

{{< figure max-width="75%" src="/media/docs/loki/get-started-flog-v3.png" caption="Getting started sample application" alt="Getting started sample application">}}

## Interactive Learning Environment

{{< admonition type="note" >}}
The Interactive Learning Environment is currently in trial. Please provide feedback, report bugs, and raise issues in the [Grafana Killercoda Repository](https://github.com/grafana/killercoda).
{{< /admonition >}}

Try out this demo within our interactive learning environment: [Loki Quickstart Sandbox](https://killercoda.com/grafana-labs/course/loki/loki-quickstart)

- A free Killercoda account is required to verify you are not a bot.
- Tutorial instructions are located on the left-hand side of the screen. Click to move on to the next section.
- All commands run inside the interactive terminal. Grafana can also be accessed via the URL links provided within the sandbox.

## Installing Loki and collecting sample logs

Prerequisites
Expand Down
11 changes: 11 additions & 0 deletions docs/sources/reference/loki-http-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ Authorization needs to be done separately, for example, using an open-source loa
These endpoints are exposed by the `distributor`, `write`, and `all` components:

- [`POST /loki/api/v1/push`](#ingest-logs)
- [`POST /otlp/v1/logs`](#ingest-logs-using-otlp)

A [list of clients]({{< relref "../send-data" >}}) can be found in the clients documentation.

Expand Down Expand Up @@ -260,6 +261,16 @@ curl -H "Content-Type: application/json" \
--data-raw '{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}'
```

## Ingest logs using OTLP

```bash
POST /otlp/v1/logs
```

`/otlp/v1/logs` lets the OpenTelemetry Collector send logs to Loki using `otlphttp` procotol.

For information on how to configure Loki, refer to the [OTel Collector topic](https://grafana.com/docs/loki/<LOKI_VERSION>/send-data/otel/).

## Query logs at a single point in time

```bash
Expand Down
2 changes: 1 addition & 1 deletion docs/sources/release-notes/_index.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ weight: 100
Release notes for Loki are in the CHANGELOG for the release and
listed here by version number.

- [V3.0 release notes](https://grafana.com/docs/loki/<LOKI_VERSION>/release-notes/v3.0/)
- [V3.0 release notes](https://grafana.com/docs/loki/<LOKI_VERSION>/release-notes/v3-0/)
- [V2.9 release notes](https://grafana.com/docs/loki/<LOKI_VERSION>/release-notes/v2-9/)
- [V2.8 release notes](https://grafana.com/docs/loki/<LOKI_VERSION>/release-notes/v2-8/)
- [V2.7 release notes](https://grafana.com/docs/loki/<LOKI_VERSION>/release-notes/v2-7/)
Expand Down
4 changes: 2 additions & 2 deletions docs/sources/send-data/otel/_index.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ You need to make the following changes to the [OpenTelemetry Collector config](h
```yaml
exporters:
otlphttp:
endpoint: http://<loki-addr>:3100/otlp
endpoint: http://<loki-addr>:3100/otlp/v1/logs
```
And enable it in `service.pipelines`:
Expand All @@ -57,7 +57,7 @@ exporters:
otlphttp:
auth:
authenticator: basicauth/otlp
endpoint: http://<loki-addr>:3100/otlp
endpoint: http://<loki-addr>:3100/otlp/v1/logs
service:
extensions: [basicauth/otlp]
Expand Down
4 changes: 2 additions & 2 deletions docs/sources/setup/install/helm/install-monolithic/_index.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ If you set the `singleBinary.replicas` value to 2 or more, this chart configures
type: 'filesystem'
schemaConfig:
configs:
- from: 2024-01-01
- from: "2024-01-01"
store: tsdb
index:
prefix: loki_index_
Expand All @@ -72,7 +72,7 @@ If you set the `singleBinary.replicas` value to 2 or more, this chart configures
replication_factor: 3
schemaConfig:
configs:
- from: 2024-01-01
- from: "2024-01-01"
store: tsdb
index:
prefix: loki_index_
Expand Down
39 changes: 39 additions & 0 deletions docs/sources/setup/install/helm/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,7 @@ This is the generated reference for the Loki Helm Chart values.
"initContainers": [],
"nodeSelector": {},
"persistence": {
"annotations": {},
"dataVolumeParameters": {
"emptyDir": {}
},
Expand Down Expand Up @@ -512,6 +513,15 @@ null
<td><pre lang="json">
{}
</pre>
</td>
</tr>
<tr>
<td>backend.persistence.annotations</td>
<td>object</td>
<td>Annotations for volume claim</td>
<td><pre lang="json">
{}
</pre>
</td>
</tr>
<tr>
Expand Down Expand Up @@ -6226,6 +6236,7 @@ false
"drivesPerNode": 2,
"enabled": false,
"persistence": {
"annotations": {},
"size": "5Gi"
},
"replicas": 1,
Expand Down Expand Up @@ -8442,6 +8453,7 @@ false
"lifecycle": {},
"nodeSelector": {},
"persistence": {
"annotations": {},
"enableStatefulSetAutoDeletePVC": true,
"selector": null,
"size": "10Gi",
Expand Down Expand Up @@ -8653,6 +8665,15 @@ false
<td><pre lang="json">
{}
</pre>
</td>
</tr>
<tr>
<td>read.persistence.annotations</td>
<td>object</td>
<td>Annotations for volume claim</td>
<td><pre lang="json">
{}
</pre>
</td>
</tr>
<tr>
Expand Down Expand Up @@ -9893,6 +9914,15 @@ null
<td><pre lang="json">
{}
</pre>
</td>
</tr>
<tr>
<td>singleBinary.persistence.annotations</td>
<td>object</td>
<td>Annotations for volume claim</td>
<td><pre lang="json">
{}
</pre>
</td>
</tr>
<tr>
Expand Down Expand Up @@ -10677,6 +10707,15 @@ null
<td><pre lang="json">
{}
</pre>
</td>
</tr>
<tr>
<td>write.persistence.annotations</td>
<td>object</td>
<td>Annotations for volume claim</td>
<td><pre lang="json">
{}
</pre>
</td>
</tr>
<tr>
Expand Down
51 changes: 51 additions & 0 deletions docs/sources/shared/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,37 @@ pattern_ingester:
# merging them as bloom blocks.
[bloom_compactor: <bloom_compactor>]

bloom_build:
# Flag to enable or disable the usage of the bloom-planner and bloom-builder
# components.
# CLI flag: -bloom-build.enabled
[enabled: <boolean> | default = false]

planner:
# Interval at which to re-run the bloom creation planning.
# CLI flag: -bloom-build.planner.interval
[planning_interval: <duration> | default = 8h]

# Newest day-table offset (from today, inclusive) to build blooms for.
# Increase to lower cost by not re-writing data to object storage too
# frequently since recent data changes more often at the cost of not having
# blooms available as quickly.
# CLI flag: -bloom-build.planner.min-table-offset
[min_table_offset: <int> | default = 1]

# Oldest day-table offset (from today, inclusive) to compact. This can be
# used to lower cost by not trying to compact older data which doesn't
# change. This can be optimized by aligning it with the maximum
# `reject_old_samples_max_age` setting of any tenant.
# CLI flag: -bloom-build.planner.max-table-offset
[max_table_offset: <int> | default = 2]

# Maximum number of tasks to queue per tenant.
# CLI flag: -bloom-build.planner.max-tasks-per-tenant
[max_queued_tasks_per_tenant: <int> | default = 30000]

builder:

# Experimental: The bloom_gateway block configures the Loki bloom gateway
# server, responsible for serving queries for filtering chunks based on filter
# expressions.
Expand Down Expand Up @@ -2947,6 +2978,11 @@ The `limits_config` block configures global and per-tenant limits in Loki. The v
# CLI flag: -validation.discover-log-levels
[discover_log_levels: <boolean> | default = true]

# When true an ingester takes into account only the streams that it owns
# according to the ring while applying the stream limit.
# CLI flag: -ingester.use-owned-stream-count
[use_owned_stream_count: <boolean> | default = false]

# Maximum number of active streams per user, per ingester. 0 to disable.
# CLI flag: -ingester.max-streams-per-user
[max_streams_per_user: <int> | default = 0]
Expand Down Expand Up @@ -3372,6 +3408,21 @@ shard_streams:
# CLI flag: -bloom-compactor.max-bloom-size
[bloom_compactor_max_bloom_size: <int> | default = 128MB]

# Experimental. Whether to create blooms for the tenant.
# CLI flag: -bloom-build.enable
[bloom_creation_enabled: <boolean> | default = false]

# Experimental. Number of splits to create for the series keyspace when building
# blooms. The series keyspace is split into this many parts to parallelize bloom
# creation.
# CLI flag: -bloom-build.split-keyspace-by
[bloom_split_series_keyspace_by: <int> | default = 256]

# Experimental. Maximum number of builders to use when building blooms. 0 allows
# unlimited builders.
# CLI flag: -bloom-build.max-builders
[bloom_build_max_builders: <int> | default = 0]

# Experimental. Length of the n-grams created when computing blooms from log
# lines.
# CLI flag: -bloom-compactor.ngram-length
Expand Down
50 changes: 50 additions & 0 deletions pkg/bloombuild/builder/builder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package builder

import (
"context"

"github.com/go-kit/log"
"github.com/grafana/dskit/services"
"github.com/prometheus/client_golang/prometheus"

utillog "github.com/grafana/loki/v3/pkg/util/log"
)

type Worker struct {
services.Service

cfg Config
metrics *Metrics
logger log.Logger
}

func New(
cfg Config,
logger log.Logger,
r prometheus.Registerer,
) (*Worker, error) {
utillog.WarnExperimentalUse("Bloom Builder", logger)

w := &Worker{
cfg: cfg,
metrics: NewMetrics(r),
logger: logger,
}

w.Service = services.NewBasicService(w.starting, w.running, w.stopping)
return w, nil
}

func (w *Worker) starting(_ context.Context) (err error) {
w.metrics.running.Set(1)
return err
}

func (w *Worker) stopping(_ error) error {
w.metrics.running.Set(0)
return nil
}

func (w *Worker) running(_ context.Context) error {
return nil
}
21 changes: 21 additions & 0 deletions pkg/bloombuild/builder/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package builder

import "flag"

// Config configures the bloom-builder component.
type Config struct {
// TODO: Add config
}

// RegisterFlagsWithPrefix registers flags for the bloom-planner configuration.
func (cfg *Config) RegisterFlagsWithPrefix(_ string, _ *flag.FlagSet) {
// TODO: Register flags with flagsPrefix
}

func (cfg *Config) Validate() error {
return nil
}

type Limits interface {
// TODO: Add limits
}
26 changes: 26 additions & 0 deletions pkg/bloombuild/builder/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package builder

import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)

const (
metricsNamespace = "loki"
metricsSubsystem = "bloombuilder"
)

type Metrics struct {
running prometheus.Gauge
}

func NewMetrics(r prometheus.Registerer) *Metrics {
return &Metrics{
running: promauto.With(r).NewGauge(prometheus.GaugeOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "running",
Help: "Value will be 1 if the bloom builder is currently running on this instance",
}),
}
}
Loading

0 comments on commit d9e8b48

Please sign in to comment.