diff --git a/CHANGELOG.md b/CHANGELOG.md index 317993ca..39b17c78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## master / unreleased * [FEATURE] Added "Cortex / Rollout progress" dashboard. #289 #290 +* [FEATURE] Added support for using query-scheduler component, which moves the queue out of query-frontend, and allows scaling up of query-frontend component. #295 * [ENHANCEMENT] Added `newCompactorStatefulSet()` function to create a custom statefulset for the compactor. #287 * [ENHANCEMENT] Added option to configure compactor job name used in dashboards and alerts. #287 * [ENHANCEMENT] Added `CortexCompactorHasNotSuccessfullyRunCompaction` alert. #292 #294 diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet index 14fc54f9..591b8b0c 100644 --- a/cortex/config.libsonnet +++ b/cortex/config.libsonnet @@ -434,6 +434,9 @@ alertmanager_enabled: false, + // Enables query-scheduler component, and reconfigures querier and query-frontend to use it. + query_scheduler_enabled: false, + // Enables streaming of chunks from ingesters using blocks. ingester_stream_chunks_when_using_blocks: true, }, diff --git a/cortex/cortex.libsonnet b/cortex/cortex.libsonnet index 5341b7c0..b8716d19 100644 --- a/cortex/cortex.libsonnet +++ b/cortex/cortex.libsonnet @@ -13,6 +13,7 @@ (import 'table-manager.libsonnet') + (import 'ruler.libsonnet') + (import 'alertmanager.libsonnet') + +(import 'query-scheduler.libsonnet') + // Supporting services (import 'etcd.libsonnet') + diff --git a/cortex/images.libsonnet b/cortex/images.libsonnet index 7546fb6f..a80baded 100644 --- a/cortex/images.libsonnet +++ b/cortex/images.libsonnet @@ -17,6 +17,7 @@ flusher: self.cortex, ruler: self.cortex, store_gateway: self.cortex, + query_scheduler: self.cortex, cortex_tools: 'grafana/cortex-tools:v0.4.0', query_tee: 'quay.io/cortexproject/query-tee:v1.8.0', diff --git a/cortex/query-scheduler.libsonnet b/cortex/query-scheduler.libsonnet new file mode 100644 index 00000000..531bb1c3 --- /dev/null +++ b/cortex/query-scheduler.libsonnet @@ -0,0 +1,52 @@ +// Query-scheduler is optional service. When query-scheduler.libsonnet is added to Cortex, querier and frontend +// are reconfigured to use query-scheduler service. +{ + local container = $.core.v1.container, + local deployment = $.apps.v1.deployment, + local service = $.core.v1.service, + + query_scheduler_args+:: + $._config.grpcConfig + { + target: 'query-scheduler', + 'log.level': 'debug', + 'query-scheduler.max-outstanding-requests-per-tenant': 100, + }, + + query_scheduler_container:: + container.new('query-scheduler', $._images.query_scheduler) + + container.withPorts($.util.defaultPorts) + + container.withArgsMixin($.util.mapToFlags($.query_scheduler_args)) + + $.jaeger_mixin + + $.util.readinessProbe + + $.util.resourcesRequests('2', '1Gi') + + $.util.resourcesLimits(null, '2Gi'), + + + query_scheduler_deployment: if !$._config.query_scheduler_enabled then {} else + deployment.new('query-scheduler', 2, [$.query_scheduler_container]) + + $.util.configVolumeMount('overrides', '/etc/cortex') + + $.util.antiAffinity, + + query_scheduler_service: if !$._config.query_scheduler_enabled then {} else + $.util.serviceFor($.query_scheduler_deployment), + + // Headless to make sure resolution gets IP address of target pods, and not service IP. + query_scheduler_discovery_service: if !$._config.query_scheduler_enabled then {} else + $.util.serviceFor($.query_scheduler_deployment) + + service.mixin.spec.withPublishNotReadyAddresses(true) + + service.mixin.spec.withClusterIp('None') + + service.mixin.metadata.withName('query-scheduler-discovery'), + + // Reconfigure querier and query-frontend to use scheduler. + querier_args+:: if !$._config.query_scheduler_enabled then {} else { + 'querier.worker-match-max-concurrent': 'true', + 'querier.worker-parallelism': null, // Disabled since we set worker-match-max-concurrent. + 'querier.frontend-address': null, + 'querier.scheduler-address': 'query-scheduler-discovery.%(namespace)s.svc.cluster.local:9095' % $._config, + }, + + query_frontend_args+:: if !$._config.query_scheduler_enabled then {} else { + 'frontend.scheduler-address': 'query-scheduler-discovery.%(namespace)s.svc.cluster.local:9095' % $._config, + }, +}