From bc12597f6dd971f559d024c43868ab2d2f881e58 Mon Sep 17 00:00:00 2001 From: akvlad Date: Wed, 25 Sep 2024 23:02:03 +0300 Subject: [PATCH 1/4] fix #574 --- pyroscope/json_parsers.js | 5 +- pyroscope/pyroscope.js | 217 ++++++++++++++++++++++++++++++++------ pyroscope/render.js | 4 +- 3 files changed, 188 insertions(+), 38 deletions(-) diff --git a/pyroscope/json_parsers.js b/pyroscope/json_parsers.js index a1b8ed23..f7fa9d0a 100644 --- a/pyroscope/json_parsers.js +++ b/pyroscope/json_parsers.js @@ -33,7 +33,8 @@ const labelNames = async (req, payload) => { return { getStart: () => body.start, getEnd: () => body.end, - getName: () => body.name + getName: () => body.name, + getMatchersList: () => body.matchers } } @@ -43,7 +44,7 @@ const labelValues = async (req, payload) => { body = JSON.parse(body.toString()) return { getName: () => body.name, - getMatchers: () => body.matchers, + getMatchersList: () => body.matchers, getStart: () => body.start, getEnd: () => body.end } diff --git a/pyroscope/pyroscope.js b/pyroscope/pyroscope.js index b62beeea..502cff59 100644 --- a/pyroscope/pyroscope.js +++ b/pyroscope/pyroscope.js @@ -53,6 +53,7 @@ WHERE date >= toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)})) AND date <= toDa } const labelNames = async (req, res) => { + const body = req.body const dist = clusterName ? '_dist' : '' const fromTimeSec = Math.floor(req.body && req.body.getStart ? parseInt(req.body.getStart()) / 1000 @@ -60,17 +61,52 @@ const labelNames = async (req, res) => { const toTimeSec = Math.floor(req.body && req.body.getEnd ? parseInt(req.body.getEnd()) / 1000 : Date.now() / 1000) - const labelNames = await clickhouse.rawRequest(`SELECT DISTINCT key -FROM profiles_series_keys${dist} -WHERE date >= toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)})) AND date <= toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)})) FORMAT JSON`, - null, DATABASE_NAME()) + if (!body.getMatchersList || body.getMatchersList().length === 0) { + const q = `SELECT DISTINCT key + FROM profiles_series_keys ${dist} + WHERE date >= toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)})) + AND date <= toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)})) FORMAT JSON` + console.log(q) + const labelNames = await clickhouse.rawRequest(q, null, DATABASE_NAME()) + const resp = new types.LabelNamesResponse() + resp.setNamesList(labelNames.data.data.map(label => label.key)) + return resp + } + const promises = [] + for (const matcher of body.getMatchersList()) { + const specialMatchers = getSpecialMatchers(matcher) + const idxReq = matcherIdxRequest(matcher, specialMatchers, fromTimeSec, toTimeSec) + const withIdxReq = new Sql.With('idx', idxReq) + const specialClauses = specialMatchersQuery(specialMatchers.matchers, + 'sample_types_units') + const serviceNameSelector = serviceNameSelectorQuery(matcher) + const req = (new Sql.Select()).with(withIdxReq) + .select('key') + .distinct(true) + .from(`profiles_series_gin${dist}`) + .where(Sql.And( + specialClauses, + serviceNameSelector, + Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)), + Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)), + new Sql.In('fingerprint', 'IN', new Sql.WithReference(withIdxReq)) + )) + promises.push(clickhouse.rawRequest(req.toString() + ' FORMAT JSON', null, DATABASE_NAME())) + } + const labelNames = await Promise.all(promises) + const labelNamesDedup = Object.fromEntries( + labelNames.flatMap(val => { + return val.data.data.map(row => [row.key, true]) + }) + ) const resp = new types.LabelNamesResponse() - resp.setNamesList(labelNames.data.data.map(label => label.key)) + resp.setNamesList([...Object.keys(labelNamesDedup)]) return resp } const labelValues = async (req, res) => { const dist = clusterName ? '_dist' : '' + const body = req.body; const name = req.body && req.body.getName ? req.body.getName() : '' @@ -83,13 +119,45 @@ const labelValues = async (req, res) => { if (!name) { throw new Error('No name provided') } - const labelValues = await clickhouse.rawRequest(`SELECT DISTINCT val + if (!body.getMatchersList || body.getMatchersList().length === 0) { + const labelValues = await clickhouse.rawRequest(`SELECT DISTINCT val FROM profiles_series_gin${dist} WHERE key = ${Sql.quoteVal(name)} AND date >= toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)})) AND date <= toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)})) FORMAT JSON`, null, DATABASE_NAME()) + const resp = new types.LabelValuesResponse() + resp.setNamesList(labelValues.data.data.map(label => label.val)) + return resp + } + const promises = [] + for (const matcher of body.getMatchersList()) { + const specialMatchers = getSpecialMatchers(matcher) + const idxReq = matcherIdxRequest(matcher, specialMatchers, fromTimeSec, toTimeSec) + const withIdxReq = new Sql.With('idx', idxReq) + const specialClauses = specialMatchersQuery(specialMatchers.matchers, + 'sample_types_units') + const serviceNameSelector = serviceNameSelectorQuery(matcher) + const req = (new Sql.Select()).with(withIdxReq) + .select('val') + .distinct(true) + .from(`profiles_series_gin${dist}`) + .where(Sql.And( + specialClauses, + serviceNameSelector, + Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)), + Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)), + Sql.Eq('key', name), + new Sql.In('fingerprint', 'IN', new Sql.WithReference(withIdxReq)) + )) + console.log(req.toString()) + promises.push(clickhouse.rawRequest(req.toString() + ' FORMAT JSON', null, DATABASE_NAME())) + } + const labelValues = await Promise.all(promises) + const labelValuesDedup = Object.fromEntries( + labelValues.flatMap(val => val.data.data.map(row => [row.val, true])) + ) const resp = new types.LabelValuesResponse() - resp.setNamesList(labelValues.data.data.map(label => label.val)) + resp.setNamesList([...Object.keys(labelValuesDedup)]) return resp } @@ -244,6 +312,36 @@ const selectMergeProfile = async (req, res) => { } } +/** + * + * @param labelSelector {string} + * @param specialMatchers {object || undefined} + * @param fromTimeSec {number} + * @param toTimeSec {number} + * @returns {Sql.Select} + */ +const matcherIdxRequest = (labelSelector, specialMatchers, fromTimeSec, toTimeSec) => { + specialMatchers = specialMatchers || getSpecialMatchers(labelSelector) + const specialClauses = specialMatchersQuery(specialMatchers.matchers, + 'sample_types_units') + const serviceNameSelector = serviceNameSelectorQuery(labelSelector) + const idxReq = (new Sql.Select()) + .select(new Sql.Raw('fingerprint')) + .from(`${DATABASE_NAME()}.profiles_series_gin`) + .where( + Sql.And( + specialClauses, + serviceNameSelector, + Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)), + Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)) + ) + ) + if (!specialMatchers.query.match(/^[{} ]*$/)) { + labelSelectorQuery(idxReq, specialMatchers.query) + } + return idxReq +} + const series = async (req, res) => { const _req = req.body const fromTimeSec = Math.floor(_req.getStart && _req.getStart() @@ -256,19 +354,39 @@ const series = async (req, res) => { const promises = [] for (const labelSelector of _req.getMatchersList() || []) { const specialMatchers = getSpecialMatchers(labelSelector) - const specialClauses = specialMatchersQuery(specialMatchers.matchers) + // Special matchers -> query clauses + const sampleTypesUnitsFieldName = '_sample_types_units' + const clauses = [] + if (specialMatchers.__name__) { + clauses.push(matcherClause("splitByChar(':', type_id)[1]", specialMatchers.__name__)) + } + if (specialMatchers.__period_type__) { + clauses.push(matcherClause("splitByChar(':', type_id)[2]", specialMatchers.__period_type__)) + } + if (specialMatchers.__period_unit__) { + clauses.push(matcherClause("splitByChar(':', type_id)[3]", specialMatchers.__period_unit__)) + } + if (specialMatchers.__sample_type__) { + clauses.push(matcherClause(`${sampleTypesUnitsFieldName}.1`, specialMatchers.__sample_type__)) + } + if (specialMatchers.__sample_unit__) { + clauses.push(matcherClause(`${sampleTypesUnitsFieldName}.2`, specialMatchers.__sample_unit__)) + } + if (specialMatchers.__profile_type__) { + clauses.push(matcherClause( + `format('{}:{}:{}:{}:{}', (splitByChar(':', type_id) as _parts)[1], ${sampleTypesUnitsFieldName}.1, ${sampleTypesUnitsFieldName}.2, _parts[2], _parts[3])`, + specialMatchers.__profile_type__)) + } + if (clauses.length === 0) { + return Sql.Eq(new Sql.Raw('1'), 1) + } + if (clauses.length === 1) { + return clauses[0] + } + const specialClauses = Sql.And(...clauses) + // const serviceNameSelector = serviceNameSelectorQuery(labelSelector) - const idxReq = (new Sql.Select()) - .select(new Sql.Raw('fingerprint')) - .from(`${DATABASE_NAME()}.profiles_series_gin`) - .where( - Sql.And( - serviceNameSelector, - Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)), - Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)) - ) - ) - labelSelectorQuery(idxReq, specialMatchers.query) + const idxReq = matcherIdxRequest(labelSelector, specialMatchers, fromTimeSec, toTimeSec) const withIdxReq = (new Sql.With('idx', idxReq, !!clusterName)) const labelsReq = (new Sql.Select()) .with(withIdxReq) @@ -349,8 +467,21 @@ const series = async (req, res) => { } /** + * returns special matchers and sanitized query without them as following: + * @example + * { + * "matchers": { + * "__name__": ["=", "foo"], + * "__period_type__": ["=~", "bar"], + * }, + * "query": "{service_name=\"abc\", job=\"def\"}" + * } * * @param query {string} + * @returns {{ + * matchers: { [fieldName: string]: [operator: string, value: string] }, + * query: string + * }} */ const getSpecialMatchers = (query) => { if (query.length <= 2) { @@ -395,27 +526,45 @@ const matcherClause = (field, matcher) => { return valRul } -const specialMatchersQuery = (matchers) => { +/** + * @example + * specialMatchersQuery({ + * "__name__": ["=", "foo"], + * "__period_type__": ["=~", "bar"], + * }) + * + * @param specialMatchers {Object} + * @returns {Sql.Condition} + */ +const specialMatchersQuery = (specialMatchers) => { + const sampleTypesUnitsFieldName = 'sample_types_units' const clauses = [] - if (matchers.__name__) { - clauses.push(matcherClause("splitByChar(':', type_id)[1]", matchers.__name__)) + if (specialMatchers.__name__) { + clauses.push(matcherClause("splitByChar(':', type_id)[1]", specialMatchers.__name__)) } - if (matchers.__period_type__) { - clauses.push(matcherClause("splitByChar(':', type_id)[2]", matchers.__period_type__)) + if (specialMatchers.__period_type__) { + clauses.push(matcherClause("splitByChar(':', type_id)[2]", specialMatchers.__period_type__)) } - if (matchers.__period_unit__) { - clauses.push(matcherClause("splitByChar(':', type_id)[3]", matchers.__period_unit__)) + if (specialMatchers.__period_unit__) { + clauses.push(matcherClause("splitByChar(':', type_id)[3]", specialMatchers.__period_unit__)) + } + const arrayExists = (field) => { + const arrayExists = Sql.Condition(null, null, null) + arrayExists.toString = () => { + return `arrayExists(x -> ${field}, ${sampleTypesUnitsFieldName})` + } + return arrayExists } - if (matchers.__sample_type__) { - clauses.push(matcherClause('_sample_types_units.1', matchers.__sample_type__)) + if (specialMatchers.__sample_type__) { + clauses.push(arrayExists(matcherClause('x.1', specialMatchers.__sample_type__))) } - if (matchers.__sample_unit__) { - clauses.push(matcherClause('_sample_types_units.2', matchers.__sample_unit__)) + if (specialMatchers.__sample_unit__) { + clauses.push(arrayExists(matcherClause('x.2', specialMatchers.__sample_unit__))) } - if (matchers.__profile_type__) { - clauses.push(matcherClause( - "format('{}:{}:{}:{}:{}', (splitByChar(':', type_id) as _parts)[1], _sample_types_units.1, _sample_types_units.2, _parts[2], _parts[3])", - matchers.__profile_type__)) + if (specialMatchers.__profile_type__) { + clauses.push(arrayExists(matcherClause( + "format('{}:{}:{}:{}:{}', (splitByChar(':', type_id) as _parts)[1], x.1, x.2, _parts[2], _parts[3])", + specialMatchers.__profile_type__))) } if (clauses.length === 0) { return Sql.Eq(new Sql.Raw('1'), 1) diff --git a/pyroscope/render.js b/pyroscope/render.js index ee3a2e94..1e637552 100644 --- a/pyroscope/render.js +++ b/pyroscope/render.js @@ -13,7 +13,7 @@ const render = async (req, res) => { ? Math.floor(parseInt(req.query.until) / 1000) : Math.floor((Date.now() - 1000 * 60 * 60 * 48) / 1000) if (!parsedQuery) { - return res.sendStatus(400).send('Invalid query') + return res.code(400).send('Invalid query') } const groupBy = req.query.groupBy || [] let agg = '' @@ -26,7 +26,7 @@ const render = async (req, res) => { break } if (req.query.format === 'dot') { - return res.sendStatus(400).send('Dot format is not supported') + return res.code(400).send('Dot format is not supported') } const promises = [] promises.push(mergeStackTraces( From 57961e93a094cf7bd91734e9638d6c600ae8c1b3 Mon Sep 17 00:00:00 2001 From: akvlad Date: Wed, 25 Sep 2024 23:02:21 +0300 Subject: [PATCH 2/4] e2e upd --- test/e2e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e b/test/e2e index 376a7dbb..7d77675d 160000 --- a/test/e2e +++ b/test/e2e @@ -1 +1 @@ -Subproject commit 376a7dbbc46109a1bb568b23ad8d27477b6ca91f +Subproject commit 7d77675d19889b833fcd00c78003ee814fbc0e34 From ea1941887705af828a448383d638d9c19c01d4c7 Mon Sep 17 00:00:00 2001 From: akvlad Date: Wed, 25 Sep 2024 23:35:02 +0300 Subject: [PATCH 3/4] debug --- pyroscope/pyroscope.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyroscope/pyroscope.js b/pyroscope/pyroscope.js index 502cff59..e2788b1b 100644 --- a/pyroscope/pyroscope.js +++ b/pyroscope/pyroscope.js @@ -377,13 +377,14 @@ const series = async (req, res) => { `format('{}:{}:{}:{}:{}', (splitByChar(':', type_id) as _parts)[1], ${sampleTypesUnitsFieldName}.1, ${sampleTypesUnitsFieldName}.2, _parts[2], _parts[3])`, specialMatchers.__profile_type__)) } + let specialClauses = null if (clauses.length === 0) { - return Sql.Eq(new Sql.Raw('1'), 1) + specialClauses = Sql.Eq(new Sql.Raw('1'), 1) + } else if (clauses.length === 1) { + specialClauses = clauses[0] + } else { + specialClauses = Sql.And(...clauses) } - if (clauses.length === 1) { - return clauses[0] - } - const specialClauses = Sql.And(...clauses) // const serviceNameSelector = serviceNameSelectorQuery(labelSelector) const idxReq = matcherIdxRequest(labelSelector, specialMatchers, fromTimeSec, toTimeSec) From 08814373e63f62b20fd1451afa7f7aa4a978eaa2 Mon Sep 17 00:00:00 2001 From: akvlad Date: Thu, 26 Sep 2024 14:34:36 +0300 Subject: [PATCH 4/4] fix gzip on accept-encoding --- lib/bun_wrapper.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/bun_wrapper.js b/lib/bun_wrapper.js index edc82a3a..2ce2eeeb 100644 --- a/lib/bun_wrapper.js +++ b/lib/bun_wrapper.js @@ -1,6 +1,7 @@ const { Transform } = require('stream') const log = require('./logger') const { EventEmitter } = require('events') +const zlib = require('zlib') class BodyStream extends Transform { _transform (chunk, encoding, callback) { @@ -121,6 +122,16 @@ const wrapper = (handler, parsers) => { headers['Content-Type'] = 'application/json' response = JSON.stringify(response) } + if (response && (ctx.headers.get('accept-encoding') || '').indexOf('gzip') !== -1) { + if (response.on) { + const _r = zlib.createGzip() + response.pipe(_r) + response = _r + } else { + response = Bun.gzipSync(response) + } + headers['Content-Encoding'] = 'gzip' + } return new Response(response, { status: status, headers: headers }) } return res