diff --git a/NOTICE.txt b/NOTICE.txt index 46d244d8f08b..4be2a9742050 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21125,6 +21125,61 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/shopspring/decimal +Version: v1.4.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/shopspring/decimal@v1.4.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2015 Spring, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +- Based on https://github.com/oguzbilgic/fpd, which has the following license: +""" +The MIT License (MIT) + +Copyright (c) 2013 Oguz Bilgic + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + + -------------------------------------------------------------------------------- Dependency : github.com/spf13/cobra Version: v1.10.2 @@ -70090,61 +70145,6 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice --------------------------------------------------------------------------------- -Dependency : github.com/shopspring/decimal -Version: v1.4.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/shopspring/decimal@v1.4.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2015 Spring, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -- Based on https://github.com/oguzbilgic/fpd, which has the following license: -""" -The MIT License (MIT) - -Copyright (c) 2013 Oguz Bilgic - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" - - -------------------------------------------------------------------------------- Dependency : github.com/sirupsen/logrus Version: v1.9.3 diff --git a/changelog/fragments/1770358526-add-cursor-based-incremental-fetching-to-sql-module.yaml b/changelog/fragments/1770358526-add-cursor-based-incremental-fetching-to-sql-module.yaml new file mode 100644 index 000000000000..13b006881395 --- /dev/null +++ b/changelog/fragments/1770358526-add-cursor-based-incremental-fetching-to-sql-module.yaml @@ -0,0 +1,49 @@ +# REQUIRED +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user's deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# REQUIRED for all kinds +# Change summary; a 80ish characters long description of the change. +summary: Add cursor-based incremental data fetching to the SQL module query metricset. + +# REQUIRED for breaking-change, deprecation, known-issue +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +description: | + Add a cursor feature to the SQL query metricset that enables incremental data + fetching by tracking the last fetched row value. Supports integer, timestamp, + date, float, and decimal cursor types with ascending and descending scan + directions. State is persisted via libbeat statestore (memlog backend). + +# REQUIRED for breaking-change, deprecation, known-issue +# impact: + +# REQUIRED for breaking-change, deprecation, known-issue +# action: + +# REQUIRED for all kinds +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: metricbeat + +# AUTOMATED +# OPTIONAL to manually add other PR URLs +# PR URL: A link the PR that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/beats/pull/48722 + +# AUTOMATED +# OPTIONAL to manually add other issue URLs +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +# issue: https://github.com/owner/repo/1234 diff --git a/docs/reference/metricbeat/metricbeat-metricset-sql-query.md b/docs/reference/metricbeat/metricbeat-metricset-sql-query.md index 9d2dbdee613f..cda45f4add8f 100644 --- a/docs/reference/metricbeat/metricbeat-metricset-sql-query.md +++ b/docs/reference/metricbeat/metricbeat-metricset-sql-query.md @@ -14,6 +14,331 @@ The sql `query` metricset collects rows returned by a query. Field names (columns) are returned as lowercase strings. Values are returned as numeric or string. +## Cursor-based incremental data fetching + +```{applies_to} +stack: beta 9.4 +``` + +The cursor feature enables incremental data fetching by tracking the last fetched row value +and using it to retrieve only new data on subsequent collection cycles. This is particularly +useful for: + +- Fetching audit logs or events that are continuously appended +- Reducing database load by avoiding full table scans +- Preventing duplicate data ingestion + +### Configuration + +To enable cursor-based fetching, add a `cursor` configuration block to your metricset: + +```yaml +- module: sql + metricsets: [query] + hosts: ["postgres://user:pass@localhost:5432/mydb"] + driver: postgres + sql_query: "SELECT id, event_data, created_at FROM events WHERE id > :cursor ORDER BY id ASC LIMIT 1000" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "0" +``` + +:::{note} + +`raw_data.enabled: true` in the examples above is optional and controls the event output format, not the cursor. It is shown here because raw mode is commonly used with cursor-based fetching. + +::: + +### Cursor configuration options + +| Option | Required | Description | +|--------|----------|-------------| +| `cursor.enabled` | No | Set to `true` to enable cursor-based fetching. Default: `false` | +| `cursor.column` | Yes (when enabled) | The column name to track for cursor state. Must be present in query results. | +| `cursor.type` | No | Optional cursor type. If omitted, it is inferred from `cursor.default` and refined from result rows. Allowed values: `integer`, `timestamp`, `date`, `float`, `decimal` | +| `cursor.state_id` | No | Optional stable state identity. When set, cursor state keys use this value instead of DSN, allowing continuity across DSN credential/parameter changes. Use a unique value per logical source. | +| `cursor.default` | Yes (when enabled) | Initial cursor value used on first run (before any state is persisted) | +| `cursor.direction` | No | Scan direction: `asc` (default, tracks max value) or `desc` (tracks min value) | + +::::{tip} +For best performance, ensure the `cursor.column` has a database index. Without an index, the `WHERE column > :cursor ORDER BY column` clause will trigger a full table scan on every collection cycle, which can be slow on large tables. +:::: + +### Supported cursor types + +| Type | Description | Default Format Example | +|------|-------------|----------------------| +| `integer` | Integer values (auto-incrementing IDs, sequence numbers) | `"0"` | +| `timestamp` | Timestamp values (TIMESTAMP, DATETIME). Accepts RFC3339, `YYYY-MM-DD HH:MM:SS[.nnnnnnnnn]`, and date-only formats. Stored internally as nanoseconds in UTC. | `"2024-01-01T00:00:00Z"` | +| `date` | Date values (YYYY-MM-DD format) | `"2024-01-01"` | +| `float` | Floating-point values (FLOAT, DOUBLE, REAL). IEEE 754 precision limits apply. | `"0.0"` | +| `decimal` | Exact decimal values (DECIMAL, NUMERIC). Arbitrary precision, no data loss. | `"0.00"` | + +### Scan direction + +By default, the cursor tracks the maximum value from each batch (ascending scan). For descending scans, set `cursor.direction: desc`: + +| Direction | Operator | ORDER BY | Cursor Tracks | +|-----------|----------|----------|---------------| +| `asc` (default) | `>` | `ASC` | Maximum value | +| `desc` | `<` | `DESC` | Minimum value | + +### Query requirements + +When cursor is enabled, your SQL query must: + +1. **Include the `:cursor` placeholder** exactly once in the query WHERE clause +2. **Include an ORDER BY clause** on the cursor column matching the configured direction +3. **Use `sql_response_format: table`** — cursor requires table mode +4. **Use `sql_query` (single query mode)** — cursor is not supported with `sql_queries` (multiple queries) + +Cursor is also not compatible with `fetch_from_all_databases`. Use a separate module block for each database if you need both features. + +### Example configurations + +#### Integer cursor (auto-increment ID) + +```yaml +- module: sql + metricsets: [query] + hosts: ["mysql://user:pass@localhost:3306/mydb"] + driver: mysql + sql_query: "SELECT id, event_type, payload FROM audit_log WHERE id > :cursor ORDER BY id ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "0" +``` + +#### Timestamp cursor (event timestamps) + +```yaml +- module: sql + metricsets: [query] + hosts: ["postgres://user:pass@localhost:5432/mydb"] + driver: postgres + sql_query: "SELECT id, message, created_at FROM logs WHERE created_at > :cursor ORDER BY created_at ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: created_at + type: timestamp + default: "2024-01-01T00:00:00Z" +``` + +#### Date cursor (daily partitioned data) + +```yaml +- module: sql + metricsets: [query] + hosts: ["oracle://user:pass@localhost:1521/MYDB"] + driver: oracle + sql_query: "SELECT report_date, metrics FROM daily_reports WHERE report_date > :cursor ORDER BY report_date ASC FETCH FIRST 500 ROWS ONLY" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: report_date + type: date + default: "2024-01-01" +``` + +#### Decimal cursor (exact numeric, financial data) + +```yaml +- module: sql + metricsets: [query] + hosts: ["postgres://user:pass@localhost:5432/mydb"] + driver: postgres + sql_query: "SELECT id, amount, description FROM transactions WHERE amount > :cursor ORDER BY amount ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: amount + type: decimal + default: "0.00" +``` + +#### Float cursor (approximate numeric) + +```yaml +- module: sql + metricsets: [query] + hosts: ["mysql://user:pass@localhost:3306/mydb"] + driver: mysql + sql_query: "SELECT id, score FROM scores WHERE score > :cursor ORDER BY score ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: score + type: float + default: "0.0" +``` + +:::{note} + +Float cursors use IEEE 754 `float64` representation. For exact precision at boundaries (for example, financial data), use the `decimal` type instead. + +::: + +#### MSSQL cursor (with TOP instead of LIMIT) + +MSSQL does not support `LIMIT`. Use `TOP` to restrict the number of rows per cycle: + +```yaml +- module: sql + metricsets: [query] + hosts: ["sqlserver://sa:YourPassword@localhost:1433?database=mydb"] + driver: mssql + sql_query: "SELECT TOP 500 id, event_type, payload FROM audit_log WHERE id > :cursor ORDER BY id ASC" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "0" +``` + +#### Descending scan (processing historical data backwards) + +```yaml +- module: sql + metricsets: [query] + hosts: ["postgres://user:pass@localhost:5432/mydb"] + driver: postgres + sql_query: "SELECT id, event_data FROM events WHERE id < :cursor ORDER BY id DESC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "999999999" + direction: desc +``` + +With `direction: desc`, the cursor tracks the minimum value from each batch, suitable for scanning data in reverse chronological order. + +### State persistence + +Cursor state is persisted to disk using Metricbeat's statestore at: +`{data.path}/sql-cursor/` + +The state persists across Metricbeat restarts, allowing incremental fetching to continue +from where it left off. State is keyed by a hash of: +- State identity: + - Full database URI/DSN (default behavior) + - `cursor.state_id` when configured +- Query string +- Cursor column name +- Cursor direction (`asc` or `desc`) + +This ensures that different query configurations maintain separate cursor states, including +different databases on the same server. + +**Important:** Changing any of these components (DSN, query, cursor column, or direction) produces a +different state key, which effectively resets the cursor to its `default` value. This is by design — +if you modify the query, the old cursor position might no longer be valid for the new query. + +### Cursor reset scenarios + +The cursor falls back to `cursor.default` when: +- Any state-key component changes: state identity (DSN by default or `cursor.state_id`), query text, `cursor.column`, or `cursor.direction` +- Persisted state is invalid (for example: unsupported state version, corrupted value, unsupported stored type, or state-load failure) +- `cursor.type` is explicitly configured and does not match the persisted state type + +The cursor does **not** reset when only these settings change: +- `cursor.default` (used only when state is missing/invalid) +- `period` or `timeout` (runtime scheduling/timeout settings) +- DSN credentials/parameters when `cursor.state_id` is set and unchanged + +When `cursor.type` is omitted (auto mode), a valid persisted state type is reused on restart. +If no valid state exists, type is inferred from `cursor.default`. + +### Choosing comparison operators for queries + +The choice of comparison operator in your WHERE clause affects data completeness: + +**Use `>` (greater than) when:** +- The cursor column has unique, monotonically increasing values (auto-increment IDs, sequences) +- No two rows can share the same cursor value +- Example: `WHERE id > :cursor ORDER BY id ASC` + +**Use `>=` (greater than or equal) when:** +- The cursor column can have duplicate values (timestamps, dates, scores) +- Late-arriving rows might be inserted with the same value as the current cursor +- Example: `WHERE created_at >= :cursor ORDER BY created_at ASC` + +The `>=` operator causes the last row from each batch to be re-fetched on the next cycle (a duplicate), but ensures no data is lost when multiple rows share the same cursor value. If using `>=`, configure Elasticsearch document IDs or use an ingest pipeline to deduplicate. + +```yaml +# Safe for timestamps -- accepts duplicates, prevents data loss +sql_query: "SELECT id, data, created_at FROM events WHERE created_at >= :cursor ORDER BY created_at ASC LIMIT 500" + +# Safe for unique IDs -- no duplicates possible +sql_query: "SELECT id, data FROM events WHERE id > :cursor ORDER BY id ASC LIMIT 500" +``` + +### Error handling + +The cursor feature follows an "at-least-once" delivery model: +- Events are emitted **before** the cursor state is updated +- If a failure occurs after emitting events but before updating state, those events will be + re-fetched on the next cycle +- This ensures no data loss, but can result in occasional duplicates + +### Driver-specific notes + +**MySQL:** When using timestamp cursors, include `parseTime=true` in your DSN to ensure the driver +correctly handles `time.Time` parameters: +``` +hosts: ["root:pass@tcp(localhost:3306)/mydb?parseTime=true"] +``` + +**Oracle:** Set the session timezone to UTC for timestamp cursors. The `godror` driver can convert +Go UTC timestamps to the Oracle session timezone, causing incorrect comparisons. Use the +`alterSession` DSN parameter or consult the Oracle integration documentation. + +**MSSQL:** Use `TOP` instead of `LIMIT` to restrict results per cycle. The driver uses `@p1` as the +parameter placeholder. Use `driver: mssql` in your configuration. It is automatically mapped to the +modern `sqlserver` driver internally. + +**Decimal columns:** The `decimal` cursor type passes the cursor value as a string to the database +driver. Most drivers (PostgreSQL, MySQL, MSSQL) implicitly cast strings to DECIMAL for comparison. +If your driver doesn't, use an explicit cast: `WHERE price > CAST(:cursor AS DECIMAL(10,2))`. + +### Limitations + +- Only one `:cursor` placeholder is allowed per query +- Placeholder detection skips common quoted strings, quoted identifiers, and SQL comments. + Limitation: MySQL backslash-escaped strings (for example, `'it\\'s :cursor'`) can still + mis-detect `:cursor` inside the literal +- The cursor column **must** be included in the SELECT clause. If omitted, the cursor will not + advance and an error will be logged on the first fetch +- NULL cursor values are skipped (only non-NULL values contribute to cursor progression) +- String, UUID, and ULID columns are not supported as cursor types. Workaround: add an integer + or timestamp column for cursor tracking, or use a database function to convert to a sortable value +- All matching rows are loaded into memory before events are emitted. Use LIMIT to control memory + usage (recommended: 500-5000 rows per cycle). For wide rows with large text columns, use a lower LIMIT +- Float cursors are subject to IEEE 754 precision limits. For exact boundary comparisons + (for example, financial data), use the `decimal` type instead +- Each cursor-based fetch is protected by the module's `timeout` setting (which defaults to + `period`). Hung queries are cancelled after the timeout expires, the cursor remains unchanged, + and the next collection cycle can proceed normally +- If a cursor collection cycle takes longer than `period` but completes within `timeout`, + subsequent cycles are skipped until the current one completes + ## Fields [_fields] For a description of each field in the metricset, see the [exported fields](/reference/metricbeat/exported-fields-sql.md) section. diff --git a/docs/reference/metricbeat/metricbeat-module-sql.md b/docs/reference/metricbeat/metricbeat-module-sql.md index 6309d0ec5823..88fa1b360cf4 100644 --- a/docs/reference/metricbeat/metricbeat-module-sql.md +++ b/docs/reference/metricbeat/metricbeat-module-sql.md @@ -61,6 +61,30 @@ Use `sql_queries` or `sql_query` depending on the use-case. `sql_query` (`Backward Compatibility`) : Single query you want to run. Also, provide corresponding `sql_response_format` (value: `variables` or `table`) similar to `sql_queries`'s `response_format`. +`cursor` +: Optional configuration block for cursor-based incremental data fetching. When `cursor.enabled` is set to `true`, the module tracks the last fetched row value and retrieves only new data on subsequent collection cycles. The query must use `sql_query` (not `sql_queries`), `sql_response_format: table`, and include a `:cursor` placeholder. Supported sub-fields: + + `cursor.enabled` + : Set to `true` to enable cursor-based fetching. Default: `false`. + + `cursor.column` + : The column name to track for cursor state. Must be present in query results. + + `cursor.type` + : Optional cursor type. If omitted, it is inferred from `cursor.default` and refined from result rows. Allowed values: `integer`, `timestamp`, `date`, `float`, `decimal`. + + `cursor.state_id` + : Optional stable state identity. When set, cursor state keys use this value instead of DSN, allowing continuity across DSN credential/parameter changes. Use a unique value per logical source. + + `cursor.default` + : Initial cursor value used on first run (before any state is persisted). + + `cursor.direction` + : Scan direction: `asc` (default, tracks max value) or `desc` (tracks min value). + + Cursor is not compatible with `sql_queries` (multiple queries) or `fetch_from_all_databases`. Each cursor-based fetch is protected by the module's `timeout` setting (which defaults to `period`) to prevent hung queries from blocking indefinitely. See the [query metricset documentation](/reference/metricbeat/metricbeat-metricset-sql-query.md) for full details. + Cursor reset summary: state resets to `cursor.default` when state identity (DSN by default or `cursor.state_id`), query, column, or direction changes, when persisted state is invalid, or when an explicit `cursor.type` mismatches persisted state. Changing only `cursor.default`, `period`, or `timeout` does not reset state. With `cursor.state_id` set, DSN credential/parameter changes do not reset state. + ## Example [_example_4] @@ -803,6 +827,31 @@ For an mssql instance, by default only four databases are present namely — `ma } ``` +### Example: Use cursor for incremental data fetching + +The cursor feature enables incremental data fetching by tracking the last fetched row value. This is useful for fetching audit logs or events that are continuously appended: + +```yaml +- module: sql + metricsets: + - query + period: 30s + hosts: ["postgres://user:pass@localhost:5432/mydb?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT id, event_type, payload, created_at FROM audit_events WHERE id > :cursor ORDER BY id ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "0" +``` + +The cursor tracks the maximum (or minimum, for descending scans) value of the specified column across all fetched rows and uses it as a filter for subsequent queries. Supported cursor types are `integer`, `timestamp`, `date`, `float`, and `decimal`. + +**Important:** Use `>` with unique columns (auto-increment IDs) and `>=` with columns that may have duplicate values (timestamps) to prevent data loss. See the query metricset documentation for full cursor configuration details, including driver-specific notes and boundary handling guidance. + ### Host Setup Some drivers require additional configuration to work. Find here instructions for these drivers. @@ -956,6 +1005,18 @@ metricbeat.modules: sql_query: "select now()" sql_response_format: table + # Cursor-based incremental data fetching. When enabled, the cursor tracks the + # last fetched row value and uses it to retrieve only new data on subsequent + # collection cycles. Requires sql_response_format: table and a single sql_query + # with exactly one :cursor placeholder. + #cursor: + # enabled: true + # column: id + # type: integer + # #state_id: payments-prod # optional: stable identity across DSN changes + # default: "0" + # #direction: asc + # List of root certificates for SSL/TLS server verification # ssl.certificate_authorities: ["/path/to/ca.pem"] diff --git a/go.mod b/go.mod index 0ed44e3c0300..e05ab4cc6425 100644 --- a/go.mod +++ b/go.mod @@ -110,7 +110,7 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e // indirect github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54 - github.com/shopspring/decimal v1.4.0 // indirect + github.com/shopspring/decimal v1.4.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 diff --git a/metricbeat/helper/sql/sql.go b/metricbeat/helper/sql/sql.go index 8757a6c9100a..3fa9db08517f 100644 --- a/metricbeat/helper/sql/sql.go +++ b/metricbeat/helper/sql/sql.go @@ -24,9 +24,8 @@ import ( "net/url" "regexp" "strconv" - "time" - "strings" + "time" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -65,11 +64,24 @@ func NewDBClient(driver, uri string, l *logp.Logger) (*DbClient, error) { } // FetchTableMode scan the rows and publishes the event for querys that return the response in a table format. -func (d *DbClient) FetchTableMode(ctx context.Context, q string) ([]mapstr.M, error) { - rows, err := d.QueryContext(ctx, q) +func (d *DbClient) FetchTableMode(ctx context.Context, query string) ([]mapstr.M, error) { + rows, err := d.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("query execution failed: %w", err) + } + defer rows.Close() + return d.fetchTableMode(rows) +} + +// FetchTableModeWithParams executes a parameterized query and returns results in table format. +// This is similar to FetchTableMode but accepts query parameters for safe parameter substitution. +// Use this for cursor-based queries where the cursor value is passed as a parameter. +func (d *DbClient) FetchTableModeWithParams(ctx context.Context, query string, args ...interface{}) ([]mapstr.M, error) { + rows, err := d.QueryContext(ctx, query, args...) if err != nil { - return nil, err + return nil, fmt.Errorf("parameterized query failed: %w", err) } + defer rows.Close() return d.fetchTableMode(rows) } @@ -117,11 +129,12 @@ func (d *DbClient) fetchTableMode(rows sqlRow) ([]mapstr.M, error) { } // FetchVariableMode executes the provided SQL query and returns the results in a key/value format. -func (d *DbClient) FetchVariableMode(ctx context.Context, q string) (mapstr.M, error) { - rows, err := d.QueryContext(ctx, q) +func (d *DbClient) FetchVariableMode(ctx context.Context, query string) (mapstr.M, error) { + rows, err := d.QueryContext(ctx, query) if err != nil { - return nil, err + return nil, fmt.Errorf("query execution failed: %w", err) } + defer rows.Close() return d.fetchVariableMode(rows) } @@ -206,6 +219,10 @@ func SwitchDriverName(d string) string { return "postgres" case "postgresql": return "postgres" + case "mssql": + // Use the modern sqlserver driver instead of the deprecated mssql driver. + // The sqlserver driver uses native @Name or @p1..@pN parameter syntax. + return "sqlserver" } return d diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 25a6a1307b59..1c06d415877e 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1655,6 +1655,18 @@ metricbeat.modules: sql_query: "select now()" sql_response_format: table + # Cursor-based incremental data fetching. When enabled, the cursor tracks the + # last fetched row value and uses it to retrieve only new data on subsequent + # collection cycles. Requires sql_response_format: table and a single sql_query + # with exactly one :cursor placeholder. + #cursor: + # enabled: true + # column: id + # type: integer + # #state_id: payments-prod # optional: stable identity across DSN changes + # default: "0" + # #direction: asc + # List of root certificates for SSL/TLS server verification # ssl.certificate_authorities: ["/path/to/ca.pem"] diff --git a/x-pack/metricbeat/module/sql/_meta/config.yml b/x-pack/metricbeat/module/sql/_meta/config.yml index fe082011846e..4b44683c371b 100644 --- a/x-pack/metricbeat/module/sql/_meta/config.yml +++ b/x-pack/metricbeat/module/sql/_meta/config.yml @@ -13,6 +13,18 @@ sql_query: "select now()" sql_response_format: table + # Cursor-based incremental data fetching. When enabled, the cursor tracks the + # last fetched row value and uses it to retrieve only new data on subsequent + # collection cycles. Requires sql_response_format: table and a single sql_query + # with exactly one :cursor placeholder. + #cursor: + # enabled: true + # column: id + # type: integer + # #state_id: payments-prod # optional: stable identity across DSN changes + # default: "0" + # #direction: asc + # List of root certificates for SSL/TLS server verification # ssl.certificate_authorities: ["/path/to/ca.pem"] diff --git a/x-pack/metricbeat/module/sql/_meta/docs.md b/x-pack/metricbeat/module/sql/_meta/docs.md index e8fa2ef07972..e09b481375aa 100644 --- a/x-pack/metricbeat/module/sql/_meta/docs.md +++ b/x-pack/metricbeat/module/sql/_meta/docs.md @@ -49,6 +49,30 @@ Use `sql_queries` or `sql_query` depending on the use-case. `sql_query` (`Backward Compatibility`) : Single query you want to run. Also, provide corresponding `sql_response_format` (value: `variables` or `table`) similar to `sql_queries`'s `response_format`. +`cursor` +: Optional configuration block for cursor-based incremental data fetching. When `cursor.enabled` is set to `true`, the module tracks the last fetched row value and retrieves only new data on subsequent collection cycles. The query must use `sql_query` (not `sql_queries`), `sql_response_format: table`, and include a `:cursor` placeholder. Supported sub-fields: + + `cursor.enabled` + : Set to `true` to enable cursor-based fetching. Default: `false`. + + `cursor.column` + : The column name to track for cursor state. Must be present in query results. + + `cursor.type` + : Optional cursor type. If omitted, it is inferred from `cursor.default` and refined from result rows. Allowed values: `integer`, `timestamp`, `date`, `float`, `decimal`. + + `cursor.state_id` + : Optional stable state identity. When set, cursor state keys use this value instead of DSN, allowing continuity across DSN credential/parameter changes. Use a unique value per logical source. + + `cursor.default` + : Initial cursor value used on first run (before any state is persisted). + + `cursor.direction` + : Scan direction: `asc` (default, tracks max value) or `desc` (tracks min value). + + Cursor is not compatible with `sql_queries` (multiple queries) or `fetch_from_all_databases`. Each cursor-based fetch is protected by the module's `timeout` setting (which defaults to `period`) to prevent hung queries from blocking indefinitely. See the [query metricset documentation](/reference/metricbeat/metricbeat-metricset-sql-query.md) for full details. + Cursor reset summary: state resets to `cursor.default` when state identity (DSN by default or `cursor.state_id`), query, column, or direction changes, when persisted state is invalid, or when an explicit `cursor.type` mismatches persisted state. Changing only `cursor.default`, `period`, or `timeout` does not reset state. With `cursor.state_id` set, DSN credential/parameter changes do not reset state. + ## Example [_example_4] @@ -791,6 +815,31 @@ For an mssql instance, by default only four databases are present namely — `ma } ``` +### Example: Use cursor for incremental data fetching + +The cursor feature enables incremental data fetching by tracking the last fetched row value. This is useful for fetching audit logs or events that are continuously appended: + +```yaml +- module: sql + metricsets: + - query + period: 30s + hosts: ["postgres://user:pass@localhost:5432/mydb?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT id, event_type, payload, created_at FROM audit_events WHERE id > :cursor ORDER BY id ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "0" +``` + +The cursor tracks the maximum (or minimum, for descending scans) value of the specified column across all fetched rows and uses it as a filter for subsequent queries. Supported cursor types are `integer`, `timestamp`, `date`, `float`, and `decimal`. + +**Important:** Use `>` with unique columns (auto-increment IDs) and `>=` with columns that may have duplicate values (timestamps) to prevent data loss. See the query metricset documentation for full cursor configuration details, including driver-specific notes and boundary handling guidance. + ### Host Setup Some drivers require additional configuration to work. Find here instructions for these drivers. diff --git a/x-pack/metricbeat/module/sql/module.go b/x-pack/metricbeat/module/sql/module.go new file mode 100644 index 000000000000..82a65d8fc36e --- /dev/null +++ b/x-pack/metricbeat/module/sql/module.go @@ -0,0 +1,143 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sql + +import ( + "fmt" + "sync" + + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/beats/v7/libbeat/statestore/backend/memlog" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/paths" +) + +// Module extends mb.Module with shared resources for the SQL module. +// MetricSets that need cursor state should type-assert base.Module() to this +// interface to access the shared statestore registry. +// +// Registry Lifetime: The statestore.Registry returned by GetCursorRegistry() is +// created lazily on first call and lives for the duration of the Beat process. +// It is never explicitly closed — the registry cleanup happens automatically when +// the Beat process exits. This is safe because: +// 1. The registry uses memlog backend which flushes writes synchronously +// 2. Beat shutdown sequence stops all modules before process exit +// 3. Individual Store handles (from registry.Get) are closed by MetricSets via mb.Closer +// +// This design matches the Filebeat pattern and prevents multiple independent stores +// from operating on the same files, which would cause file lock conflicts. +type Module interface { + mb.Module + // GetCursorRegistry returns the shared statestore registry for cursor + // persistence. The registry is created lazily on first call — no disk I/O + // occurs until a MetricSet actually enables cursor. Returns an error if + // the registry could not be created. + GetCursorRegistry() (*statestore.Registry, error) +} + +// sharedRegistryState holds the shared statestore registry and tracks which +// data path it was created for. All SQL module instances created by the same +// ModuleBuilder share a single sharedRegistryState via pointer. +// +// Path-Aware Caching: In production, paths.Resolve(paths.Data, "sql-cursor") +// never changes, so the registry is created once and reused for the entire +// lifetime of the Beat process — identical behaviour to sync.Once. +// In integration tests, each test overrides paths.Paths.Data to a unique +// t.TempDir(), causing the resolved path to change. When a path change is +// detected, a new registry is created at the new location, giving each test +// an isolated cursor store without any cross-test state leakage. +type sharedRegistryState struct { + mu sync.Mutex + registry *statestore.Registry + err error + dataPath string // resolved path the current registry was created for +} + +type module struct { + mb.BaseModule + + // Shared across all module instances via the ModuleBuilder closure. + shared *sharedRegistryState +} + +func init() { + if err := mb.Registry.AddModule("sql", ModuleBuilder()); err != nil { + panic(err) + } +} + +// ModuleBuilder returns a ModuleFactory that shares a single statestore.Registry +// across all SQL module instances. The registry is created lazily on first call +// to GetCursorRegistry, so no disk I/O occurs for non-cursor configurations. +// +// Closure Pattern: This function creates a closure containing a shared +// sharedRegistryState that persists across all SQL module instances. +// Each module instance receives a pointer to this shared state, ensuring that: +// - sync.Mutex guarantees thread-safe initialization and access +// - All modules sharing the same data path access the exact same registry +// - No file conflicts occur from multiple independent registries +// +// This function is called ONCE during init() and registered in mb.Registry. +// The returned factory function is called MULTIPLE TIMES (once per module instance). +func ModuleBuilder() mb.ModuleFactory { + shared := &sharedRegistryState{} + return func(base mb.BaseModule) (mb.Module, error) { + return &module{ + BaseModule: base, + shared: shared, + }, nil + } +} + +// GetCursorRegistry returns the shared statestore registry for cursor persistence. +// The registry and its underlying memlog backend are created on the first call +// and then reused as long as the resolved data path has not changed. Each caller +// should use registry.Get("cursor-state") to obtain a ref-counted Store handle. +// +// Path-aware caching: The method resolves the current data path via +// paths.Resolve(paths.Data, "sql-cursor"). If the resolved path matches the +// path used to create the cached registry, the cached registry is returned +// immediately. If the path has changed (which only happens in tests), a new +// registry is created at the new location. +// +// The registry itself is never closed (lives with the Beat process). This is by +// design — all Store handles must be closed (via mb.Closer), but the registry +// persists to allow new MetricSets to be created dynamically. Registry cleanup +// happens automatically when the Beat process exits. +// +// Thread-safety: This method uses sync.Mutex to ensure thread-safe access +// even when called concurrently from multiple MetricSet instances. +func (m *module) GetCursorRegistry() (*statestore.Registry, error) { + s := m.shared + s.mu.Lock() + defer s.mu.Unlock() + + dataPath := paths.Resolve(paths.Data, "sql-cursor") + + // Return the cached registry if it was created for the same data path. + if s.registry != nil && s.dataPath == dataPath { + return s.registry, s.err + } + + logger := logp.NewLogger("sql.cursor") + + reg, err := memlog.New(logger.Named("memlog"), memlog.Settings{ + Root: dataPath, + FileMode: 0o600, + }) + if err != nil { + s.err = fmt.Errorf("failed to create memlog registry: %w", err) + s.registry = nil + s.dataPath = dataPath + return nil, s.err + } + + s.registry = statestore.NewRegistry(reg) + s.err = nil + s.dataPath = dataPath + logger.Debugf("Created shared SQL cursor registry at %s (ptr=%p)", dataPath, s.registry) + return s.registry, nil +} diff --git a/x-pack/metricbeat/module/sql/module_test.go b/x-pack/metricbeat/module/sql/module_test.go new file mode 100644 index 000000000000..22ccf68a761a --- /dev/null +++ b/x-pack/metricbeat/module/sql/module_test.go @@ -0,0 +1,186 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sql + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-libs/paths" +) + +// TestModuleBuilderSharesState verifies that all module instances created by the +// same ModuleBuilder share the same sharedRegistryState pointer. +func TestModuleBuilderSharesState(t *testing.T) { + factory := ModuleBuilder() + + mod1, err := factory(mb.BaseModule{}) + require.NoError(t, err) + + mod2, err := factory(mb.BaseModule{}) + require.NoError(t, err) + + m1, ok := mod1.(*module) + require.True(t, ok, "mod1 should be *module") + m2, ok := mod2.(*module) + require.True(t, ok, "mod2 should be *module") + + assert.Same(t, m1.shared, m2.shared, + "All module instances from the same ModuleBuilder must share the same sharedRegistryState") +} + +// TestGetCursorRegistryReturnsSamePointer verifies that repeated calls to +// GetCursorRegistry return the exact same *statestore.Registry pointer when +// the data path has not changed. +func TestGetCursorRegistryReturnsSamePointer(t *testing.T) { + tmpDir := t.TempDir() + origData := paths.Paths.Data + paths.Paths.Data = tmpDir + t.Cleanup(func() { paths.Paths.Data = origData }) + + factory := ModuleBuilder() + + mod1, err := factory(mb.BaseModule{}) + require.NoError(t, err) + + mod2, err := factory(mb.BaseModule{}) + require.NoError(t, err) + + sqlMod1, ok := mod1.(Module) + require.True(t, ok, "mod1 should implement sql.Module") + reg1, err := sqlMod1.GetCursorRegistry() + require.NoError(t, err) + require.NotNil(t, reg1) + + sqlMod2, ok := mod2.(Module) + require.True(t, ok, "mod2 should implement sql.Module") + reg2, err := sqlMod2.GetCursorRegistry() + require.NoError(t, err) + require.NotNil(t, reg2) + + assert.Same(t, reg1, reg2, + "GetCursorRegistry must return the same pointer when data path is unchanged") + + // Call again on the first module - still the same pointer (cached). + reg1again, err := sqlMod1.GetCursorRegistry() + require.NoError(t, err) + assert.Same(t, reg1, reg1again, + "Repeated calls on the same module must return cached registry") +} + +// TestGetCursorRegistryPathChange verifies that changing paths.Paths.Data +// causes GetCursorRegistry to create a new registry at the new location. +func TestGetCursorRegistryPathChange(t *testing.T) { + tmpDir1 := t.TempDir() + tmpDir2 := t.TempDir() + + origData := paths.Paths.Data + t.Cleanup(func() { paths.Paths.Data = origData }) + + factory := ModuleBuilder() + mod, err := factory(mb.BaseModule{}) + require.NoError(t, err) + + sqlMod, ok := mod.(Module) + require.True(t, ok, "mod should implement sql.Module") + + // First path + paths.Paths.Data = tmpDir1 + reg1, err := sqlMod.GetCursorRegistry() + require.NoError(t, err) + require.NotNil(t, reg1) + + // Same path - cached + reg1again, err := sqlMod.GetCursorRegistry() + require.NoError(t, err) + assert.Same(t, reg1, reg1again) + + // Change path - new registry expected + paths.Paths.Data = tmpDir2 + reg2, err := sqlMod.GetCursorRegistry() + require.NoError(t, err) + require.NotNil(t, reg2) + assert.NotSame(t, reg1, reg2, + "Changing the data path must produce a new registry") + + // Revert to first path - yet another new registry (previous one is not cached) + paths.Paths.Data = tmpDir1 + reg3, err := sqlMod.GetCursorRegistry() + require.NoError(t, err) + require.NotNil(t, reg3) + assert.NotSame(t, reg1, reg3, + "Reverting to a previous path creates a new registry (old cache was replaced)") + assert.NotSame(t, reg2, reg3) +} + +// TestGetCursorRegistryConcurrent verifies that concurrent calls to +// GetCursorRegistry from multiple goroutines are safe and all return +// the same pointer. +func TestGetCursorRegistryConcurrent(t *testing.T) { + tmpDir := t.TempDir() + origData := paths.Paths.Data + paths.Paths.Data = tmpDir + t.Cleanup(func() { paths.Paths.Data = origData }) + + factory := ModuleBuilder() + + const n = 20 + modules := make([]mb.Module, n) + for i := range modules { + mod, err := factory(mb.BaseModule{}) + require.NoError(t, err) + modules[i] = mod + } + + type result struct { + reg interface{} + err error + } + + // Call GetCursorRegistry concurrently from all modules. + var wg sync.WaitGroup + resultsCh := make(chan result, n) + + for i := 0; i < n; i++ { + wg.Add(1) + go func(mod mb.Module) { + defer wg.Done() + reg, err := mod.(Module).GetCursorRegistry() + resultsCh <- result{reg: reg, err: err} + }(modules[i]) + } + + wg.Wait() + close(resultsCh) + + var first interface{} + for r := range resultsCh { + require.NoError(t, r.err) + require.NotNil(t, r.reg) + if first == nil { + first = r.reg + } else { + assert.Same(t, first, r.reg, + "All concurrent calls must return the same registry pointer") + } + } +} + +// TestModuleImplementsInterface verifies the module type implements the Module interface. +func TestModuleImplementsInterface(t *testing.T) { + factory := ModuleBuilder() + mod, err := factory(mb.BaseModule{}) + require.NoError(t, err) + + _, ok := mod.(Module) + assert.True(t, ok, "module must implement the sql.Module interface") + + // mod is already mb.Module (the return type of factory), so no assertion needed. + assert.NotNil(t, mod, "module must not be nil") +} diff --git a/x-pack/metricbeat/module/sql/query/_meta/docs.md b/x-pack/metricbeat/module/sql/query/_meta/docs.md index 4d7ad3134b99..b2a5a477632b 100644 --- a/x-pack/metricbeat/module/sql/query/_meta/docs.md +++ b/x-pack/metricbeat/module/sql/query/_meta/docs.md @@ -1,3 +1,328 @@ The sql `query` metricset collects rows returned by a query. Field names (columns) are returned as lowercase strings. Values are returned as numeric or string. + +## Cursor-based incremental data fetching + +```{applies_to} +stack: beta 9.4 +``` + +The cursor feature enables incremental data fetching by tracking the last fetched row value +and using it to retrieve only new data on subsequent collection cycles. This is particularly +useful for: + +- Fetching audit logs or events that are continuously appended +- Reducing database load by avoiding full table scans +- Preventing duplicate data ingestion + +### Configuration + +To enable cursor-based fetching, add a `cursor` configuration block to your metricset: + +```yaml +- module: sql + metricsets: [query] + hosts: ["postgres://user:pass@localhost:5432/mydb"] + driver: postgres + sql_query: "SELECT id, event_data, created_at FROM events WHERE id > :cursor ORDER BY id ASC LIMIT 1000" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "0" +``` + +:::{note} + +`raw_data.enabled: true` in the examples above is optional and controls the event output format, not the cursor. It is shown here because raw mode is commonly used with cursor-based fetching. + +::: + +### Cursor configuration options + +| Option | Required | Description | +|--------|----------|-------------| +| `cursor.enabled` | No | Set to `true` to enable cursor-based fetching. Default: `false` | +| `cursor.column` | Yes (when enabled) | The column name to track for cursor state. Must be present in query results. | +| `cursor.type` | No | Optional cursor type. If omitted, it is inferred from `cursor.default` and refined from result rows. Allowed values: `integer`, `timestamp`, `date`, `float`, `decimal` | +| `cursor.state_id` | No | Optional stable state identity. When set, cursor state keys use this value instead of DSN, allowing continuity across DSN credential/parameter changes. Use a unique value per logical source. | +| `cursor.default` | Yes (when enabled) | Initial cursor value used on first run (before any state is persisted) | +| `cursor.direction` | No | Scan direction: `asc` (default, tracks max value) or `desc` (tracks min value) | + +::::{tip} +For best performance, ensure the `cursor.column` has a database index. Without an index, the `WHERE column > :cursor ORDER BY column` clause will trigger a full table scan on every collection cycle, which can be slow on large tables. +:::: + +### Supported cursor types + +| Type | Description | Default Format Example | +|------|-------------|----------------------| +| `integer` | Integer values (auto-incrementing IDs, sequence numbers) | `"0"` | +| `timestamp` | Timestamp values (TIMESTAMP, DATETIME). Accepts RFC3339, `YYYY-MM-DD HH:MM:SS[.nnnnnnnnn]`, and date-only formats. Stored internally as nanoseconds in UTC. | `"2024-01-01T00:00:00Z"` | +| `date` | Date values (YYYY-MM-DD format) | `"2024-01-01"` | +| `float` | Floating-point values (FLOAT, DOUBLE, REAL). IEEE 754 precision limits apply. | `"0.0"` | +| `decimal` | Exact decimal values (DECIMAL, NUMERIC). Arbitrary precision, no data loss. | `"0.00"` | + +### Scan direction + +By default, the cursor tracks the maximum value from each batch (ascending scan). For descending scans, set `cursor.direction: desc`: + +| Direction | Operator | ORDER BY | Cursor Tracks | +|-----------|----------|----------|---------------| +| `asc` (default) | `>` | `ASC` | Maximum value | +| `desc` | `<` | `DESC` | Minimum value | + +### Query requirements + +When cursor is enabled, your SQL query must: + +1. **Include the `:cursor` placeholder** exactly once in the query WHERE clause +2. **Include an ORDER BY clause** on the cursor column matching the configured direction +3. **Use `sql_response_format: table`** — cursor requires table mode +4. **Use `sql_query` (single query mode)** — cursor is not supported with `sql_queries` (multiple queries) + +Cursor is also not compatible with `fetch_from_all_databases`. Use a separate module block for each database if you need both features. + +### Example configurations + +#### Integer cursor (auto-increment ID) + +```yaml +- module: sql + metricsets: [query] + hosts: ["mysql://user:pass@localhost:3306/mydb"] + driver: mysql + sql_query: "SELECT id, event_type, payload FROM audit_log WHERE id > :cursor ORDER BY id ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "0" +``` + +#### Timestamp cursor (event timestamps) + +```yaml +- module: sql + metricsets: [query] + hosts: ["postgres://user:pass@localhost:5432/mydb"] + driver: postgres + sql_query: "SELECT id, message, created_at FROM logs WHERE created_at > :cursor ORDER BY created_at ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: created_at + type: timestamp + default: "2024-01-01T00:00:00Z" +``` + +#### Date cursor (daily partitioned data) + +```yaml +- module: sql + metricsets: [query] + hosts: ["oracle://user:pass@localhost:1521/MYDB"] + driver: oracle + sql_query: "SELECT report_date, metrics FROM daily_reports WHERE report_date > :cursor ORDER BY report_date ASC FETCH FIRST 500 ROWS ONLY" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: report_date + type: date + default: "2024-01-01" +``` + +#### Decimal cursor (exact numeric, financial data) + +```yaml +- module: sql + metricsets: [query] + hosts: ["postgres://user:pass@localhost:5432/mydb"] + driver: postgres + sql_query: "SELECT id, amount, description FROM transactions WHERE amount > :cursor ORDER BY amount ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: amount + type: decimal + default: "0.00" +``` + +#### Float cursor (approximate numeric) + +```yaml +- module: sql + metricsets: [query] + hosts: ["mysql://user:pass@localhost:3306/mydb"] + driver: mysql + sql_query: "SELECT id, score FROM scores WHERE score > :cursor ORDER BY score ASC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: score + type: float + default: "0.0" +``` + +:::{note} + +Float cursors use IEEE 754 `float64` representation. For exact precision at boundaries (for example, financial data), use the `decimal` type instead. + +::: + +#### MSSQL cursor (with TOP instead of LIMIT) + +MSSQL does not support `LIMIT`. Use `TOP` to restrict the number of rows per cycle: + +```yaml +- module: sql + metricsets: [query] + hosts: ["sqlserver://sa:YourPassword@localhost:1433?database=mydb"] + driver: mssql + sql_query: "SELECT TOP 500 id, event_type, payload FROM audit_log WHERE id > :cursor ORDER BY id ASC" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "0" +``` + +#### Descending scan (processing historical data backwards) + +```yaml +- module: sql + metricsets: [query] + hosts: ["postgres://user:pass@localhost:5432/mydb"] + driver: postgres + sql_query: "SELECT id, event_data FROM events WHERE id < :cursor ORDER BY id DESC LIMIT 500" + sql_response_format: table + raw_data.enabled: true + cursor: + enabled: true + column: id + type: integer + default: "999999999" + direction: desc +``` + +With `direction: desc`, the cursor tracks the minimum value from each batch, suitable for scanning data in reverse chronological order. + +### State persistence + +Cursor state is persisted to disk using Metricbeat's statestore at: +`{data.path}/sql-cursor/` + +The state persists across Metricbeat restarts, allowing incremental fetching to continue +from where it left off. State is keyed by a hash of: +- State identity: + - Full database URI/DSN (default behavior) + - `cursor.state_id` when configured +- Query string +- Cursor column name +- Cursor direction (`asc` or `desc`) + +This ensures that different query configurations maintain separate cursor states, including +different databases on the same server. + +**Important:** Changing any of these components (DSN, query, cursor column, or direction) produces a +different state key, which effectively resets the cursor to its `default` value. This is by design — +if you modify the query, the old cursor position might no longer be valid for the new query. + +### Cursor reset scenarios + +The cursor falls back to `cursor.default` when: +- Any state-key component changes: state identity (DSN by default or `cursor.state_id`), query text, `cursor.column`, or `cursor.direction` +- Persisted state is invalid (for example: unsupported state version, corrupted value, unsupported stored type, or state-load failure) +- `cursor.type` is explicitly configured and does not match the persisted state type + +The cursor does **not** reset when only these settings change: +- `cursor.default` (used only when state is missing/invalid) +- `period` or `timeout` (runtime scheduling/timeout settings) +- DSN credentials/parameters when `cursor.state_id` is set and unchanged + +When `cursor.type` is omitted (auto mode), a valid persisted state type is reused on restart. +If no valid state exists, type is inferred from `cursor.default`. + +### Choosing comparison operators for queries + +The choice of comparison operator in your WHERE clause affects data completeness: + +**Use `>` (greater than) when:** +- The cursor column has unique, monotonically increasing values (auto-increment IDs, sequences) +- No two rows can share the same cursor value +- Example: `WHERE id > :cursor ORDER BY id ASC` + +**Use `>=` (greater than or equal) when:** +- The cursor column can have duplicate values (timestamps, dates, scores) +- Late-arriving rows might be inserted with the same value as the current cursor +- Example: `WHERE created_at >= :cursor ORDER BY created_at ASC` + +The `>=` operator causes the last row from each batch to be re-fetched on the next cycle (a duplicate), but ensures no data is lost when multiple rows share the same cursor value. If using `>=`, configure Elasticsearch document IDs or use an ingest pipeline to deduplicate. + +```yaml +# Safe for timestamps -- accepts duplicates, prevents data loss +sql_query: "SELECT id, data, created_at FROM events WHERE created_at >= :cursor ORDER BY created_at ASC LIMIT 500" + +# Safe for unique IDs -- no duplicates possible +sql_query: "SELECT id, data FROM events WHERE id > :cursor ORDER BY id ASC LIMIT 500" +``` + +### Error handling + +The cursor feature follows an "at-least-once" delivery model: +- Events are emitted **before** the cursor state is updated +- If a failure occurs after emitting events but before updating state, those events will be + re-fetched on the next cycle +- This ensures no data loss, but can result in occasional duplicates + +### Driver-specific notes + +**MySQL:** When using timestamp cursors, include `parseTime=true` in your DSN to ensure the driver +correctly handles `time.Time` parameters: +``` +hosts: ["root:pass@tcp(localhost:3306)/mydb?parseTime=true"] +``` + +**Oracle:** Set the session timezone to UTC for timestamp cursors. The `godror` driver can convert +Go UTC timestamps to the Oracle session timezone, causing incorrect comparisons. Use the +`alterSession` DSN parameter or consult the Oracle integration documentation. + +**MSSQL:** Use `TOP` instead of `LIMIT` to restrict results per cycle. The driver uses `@p1` as the +parameter placeholder. Use `driver: mssql` in your configuration. It is automatically mapped to the +modern `sqlserver` driver internally. + +**Decimal columns:** The `decimal` cursor type passes the cursor value as a string to the database +driver. Most drivers (PostgreSQL, MySQL, MSSQL) implicitly cast strings to DECIMAL for comparison. +If your driver doesn't, use an explicit cast: `WHERE price > CAST(:cursor AS DECIMAL(10,2))`. + +### Limitations + +- Only one `:cursor` placeholder is allowed per query +- Placeholder detection skips common quoted strings, quoted identifiers, and SQL comments. + Limitation: MySQL backslash-escaped strings (for example, `'it\\'s :cursor'`) can still + mis-detect `:cursor` inside the literal +- The cursor column **must** be included in the SELECT clause. If omitted, the cursor will not + advance and an error will be logged on the first fetch +- NULL cursor values are skipped (only non-NULL values contribute to cursor progression) +- String, UUID, and ULID columns are not supported as cursor types. Workaround: add an integer + or timestamp column for cursor tracking, or use a database function to convert to a sortable value +- All matching rows are loaded into memory before events are emitted. Use LIMIT to control memory + usage (recommended: 500-5000 rows per cycle). For wide rows with large text columns, use a lower LIMIT +- Float cursors are subject to IEEE 754 precision limits. For exact boundary comparisons + (for example, financial data), use the `decimal` type instead +- Each cursor-based fetch is protected by the module's `timeout` setting (which defaults to + `period`). Hung queries are cancelled after the timeout expires, the cursor remains unchanged, + and the next collection cycle can proceed normally +- If a cursor collection cycle takes longer than `period` but completes within `timeout`, + subsequent cycles are skipped until the current one completes diff --git a/x-pack/metricbeat/module/sql/query/cursor/config.go b/x-pack/metricbeat/module/sql/query/cursor/config.go new file mode 100644 index 000000000000..e24d0e585ba0 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/config.go @@ -0,0 +1,128 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "errors" + "fmt" + "strings" +) + +// Cursor type constants +const ( + // CursorTypeInteger is the cursor type for integer columns (auto-increment IDs, etc.) + CursorTypeInteger = "integer" + + // CursorTypeTimestamp is the cursor type for timestamp columns + CursorTypeTimestamp = "timestamp" + + // CursorTypeDate is the cursor type for date columns + CursorTypeDate = "date" + + // CursorTypeFloat is the cursor type for floating-point columns (FLOAT, DOUBLE, REAL). + // Uses Go float64 internally. Subject to IEEE 754 precision limits — + // boundary rows may be duplicated or skipped at the 15th+ significant digit. + CursorTypeFloat = "float" + + // CursorTypeDecimal is the cursor type for exact decimal columns (DECIMAL, NUMERIC). + // Uses shopspring/decimal for arbitrary-precision arithmetic. No data loss at boundaries. + CursorTypeDecimal = "decimal" +) + +// Cursor direction constants +const ( + // CursorDirectionAsc tracks the maximum cursor value (for ascending ORDER BY). + CursorDirectionAsc = "asc" + + // CursorDirectionDesc tracks the minimum cursor value (for descending ORDER BY). + CursorDirectionDesc = "desc" +) + +// supportedCursorTypes lists all valid cursor types. +var supportedCursorTypes = []string{ + CursorTypeInteger, + CursorTypeTimestamp, + CursorTypeDate, + CursorTypeFloat, + CursorTypeDecimal, +} + +// Config holds the cursor configuration from user's metricbeat.yml +type Config struct { + Enabled bool `config:"enabled"` + Column string `config:"column"` + Type string `config:"type"` // Optional. "integer", "timestamp", "date", "float", or "decimal" + Default string `config:"default"` // Initial cursor value as string + Direction string `config:"direction"` // "asc" (default) or "desc" + StateID string `config:"state_id"` // Optional stable identity for state keying across DSN changes +} + +// Validate checks the configuration for errors. +// If cursor is disabled, no validation is performed. +// If cursor is enabled, column/default are required and type is optional. +func (c *Config) Validate() error { + if !c.Enabled { + return nil + } + + if c.Column == "" { + return errors.New("cursor.column is required when cursor is enabled") + } + + if c.StateID != "" { + c.StateID = strings.TrimSpace(c.StateID) + if c.StateID == "" { + return errors.New("cursor.state_id cannot be blank when set") + } + } + + if c.Type != "" && !isValidCursorType(c.Type) { + return fmt.Errorf("cursor.type must be one of [%s], got %q", + strings.Join(supportedCursorTypes, ", "), c.Type) + } + + if c.Default == "" { + return errors.New("cursor.default is required when cursor is enabled") + } + + // Validate default value when type is explicitly configured. + // If type is omitted, it will be inferred at runtime. + if c.Type != "" { + if _, err := ParseValue(c.Default, c.Type); err != nil { + return fmt.Errorf("cursor.default is invalid for type %q: %w", c.Type, err) + } + } + + // Validate direction (defaults to "asc" if empty) + if c.Direction == "" { + c.Direction = CursorDirectionAsc + } + if !isValidDirection(c.Direction) { + return fmt.Errorf("cursor.direction must be '%s' or '%s', got %q", + CursorDirectionAsc, CursorDirectionDesc, c.Direction) + } + + return nil +} + +// isValidCursorType checks if the given type is a supported cursor type +func isValidCursorType(t string) bool { + for _, valid := range supportedCursorTypes { + if t == valid { + return true + } + } + return false +} + +// isValidDirection checks if the given direction is valid +func isValidDirection(d string) bool { + switch d { + case CursorDirectionAsc, CursorDirectionDesc: + return true + default: + return false + } +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/config_test.go b/x-pack/metricbeat/module/sql/query/cursor/config_test.go new file mode 100644 index 000000000000..b29cda7decd4 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/config_test.go @@ -0,0 +1,304 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigValidate(t *testing.T) { + tests := []struct { + name string + config Config + wantErr bool + errMsg string + }{ + { + name: "disabled cursor - no validation", + config: Config{ + Enabled: false, + }, + wantErr: false, + }, + { + name: "valid integer cursor", + config: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + StateID: "payments-prod", + }, + wantErr: false, + }, + { + name: "valid timestamp cursor", + config: Config{ + Enabled: true, + Column: "created_at", + Type: CursorTypeTimestamp, + Default: "2024-01-01T00:00:00Z", + }, + wantErr: false, + }, + { + name: "valid date cursor", + config: Config{ + Enabled: true, + Column: "event_date", + Type: CursorTypeDate, + Default: "2024-01-01", + }, + wantErr: false, + }, + { + name: "missing column", + config: Config{ + Enabled: true, + Column: "", + Type: CursorTypeInteger, + Default: "0", + }, + wantErr: true, + errMsg: "cursor.column is required", + }, + { + name: "invalid type", + config: Config{ + Enabled: true, + Column: "id", + Type: "invalid", + Default: "0", + }, + wantErr: true, + errMsg: "cursor.type must be", + }, + { + name: "empty type - infer later", + config: Config{ + Enabled: true, + Column: "id", + Type: "", + Default: "0", + }, + wantErr: false, + }, + { + name: "missing default", + config: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "", + }, + wantErr: true, + errMsg: "cursor.default is required", + }, + { + name: "blank state_id", + config: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + StateID: " ", + }, + wantErr: true, + errMsg: "cursor.state_id cannot be blank", + }, + { + name: "invalid integer default", + config: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "not-an-integer", + }, + wantErr: true, + errMsg: "cursor.default is invalid", + }, + { + name: "invalid timestamp default", + config: Config{ + Enabled: true, + Column: "created_at", + Type: CursorTypeTimestamp, + Default: "not-a-timestamp", + }, + wantErr: true, + errMsg: "cursor.default is invalid", + }, + { + name: "invalid date default", + config: Config{ + Enabled: true, + Column: "event_date", + Type: CursorTypeDate, + Default: "not-a-date", + }, + wantErr: true, + errMsg: "cursor.default is invalid", + }, + // Float cursor tests + { + name: "valid float cursor", + config: Config{ + Enabled: true, + Column: "score", + Type: CursorTypeFloat, + Default: "0.0", + }, + wantErr: false, + }, + { + name: "invalid float default", + config: Config{ + Enabled: true, + Column: "score", + Type: CursorTypeFloat, + Default: "not-a-float", + }, + wantErr: true, + errMsg: "cursor.default is invalid", + }, + // Decimal cursor tests + { + name: "valid decimal cursor", + config: Config{ + Enabled: true, + Column: "price", + Type: CursorTypeDecimal, + Default: "99.95", + }, + wantErr: false, + }, + { + name: "invalid decimal default", + config: Config{ + Enabled: true, + Column: "price", + Type: CursorTypeDecimal, + Default: "not-a-decimal", + }, + wantErr: true, + errMsg: "cursor.default is invalid", + }, + // Direction tests + { + name: "valid direction asc", + config: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + Direction: CursorDirectionAsc, + }, + wantErr: false, + }, + { + name: "valid direction desc", + config: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "99999", + Direction: CursorDirectionDesc, + }, + wantErr: false, + }, + { + name: "empty direction defaults to asc", + config: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + Direction: "", + }, + wantErr: false, + }, + { + name: "invalid direction", + config: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + Direction: "sideways", + }, + wantErr: true, + errMsg: "cursor.direction must be", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMsg) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestConfigValidate_DirectionWithAllTypes(t *testing.T) { + // Verify direction works with every cursor type + types := []struct { + cursorType string + defaultValue string + }{ + {CursorTypeInteger, "0"}, + {CursorTypeTimestamp, "2024-01-01T00:00:00Z"}, + {CursorTypeDate, "2024-01-01"}, + {CursorTypeFloat, "0.0"}, + {CursorTypeDecimal, "0.00"}, + } + + for _, tt := range types { + for _, dir := range []string{CursorDirectionAsc, CursorDirectionDesc} { + name := tt.cursorType + "_" + dir + t.Run(name, func(t *testing.T) { + cfg := Config{ + Enabled: true, + Column: "col", + Type: tt.cursorType, + Default: tt.defaultValue, + Direction: dir, + } + err := cfg.Validate() + require.NoError(t, err, "direction=%s with type=%s should be valid", dir, tt.cursorType) + }) + } + } +} + +func TestIsValidCursorType(t *testing.T) { + tests := []struct { + cursorType string + valid bool + }{ + {CursorTypeInteger, true}, + {CursorTypeTimestamp, true}, + {CursorTypeDate, true}, + {CursorTypeFloat, true}, + {CursorTypeDecimal, true}, + {"", false}, + {"string", false}, + {"INTEGER", false}, // case sensitive + {"FLOAT", false}, // case sensitive + } + + for _, tt := range tests { + t.Run(tt.cursorType, func(t *testing.T) { + assert.Equal(t, tt.valid, isValidCursorType(tt.cursorType)) + }) + } +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/cursor.go b/x-pack/metricbeat/module/sql/query/cursor/cursor.go new file mode 100644 index 000000000000..8de1c099cfcb --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/cursor.go @@ -0,0 +1,440 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const nilValuePlaceholder = "" + +// Manager handles cursor state lifecycle including loading, updating, and persisting cursor values. +type Manager struct { + config Config + store *Store + stateKey string + cursorValue *Value + typeLocked bool + mu sync.Mutex + logger *logp.Logger +} + +// NewManager creates a new cursor manager. +// It validates the configuration, initializes the store, and loads any existing state. +// +// Parameters: +// - cfg: Cursor configuration from metricbeat.yml +// - store: State persistence store (memlog-backed) +// - dsn: Full database URI/DSN used for state key generation unless cfg.StateID is set +// - query: Original SQL query (before placeholder translation) +// - logger: Logger instance for this cursor +// +// The manager takes ownership of the store and will close it when Close() is called. +func NewManager(cfg Config, store *Store, dsn, query string, logger *logp.Logger) (*Manager, error) { + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("cursor config validation failed: %w", err) + } + + if err := ValidateQueryHasCursor(query); err != nil { + return nil, err + } + + stateIdentity := dsn + if cfg.StateID != "" { + // Prefix with namespace to reduce accidental collisions with DSN-shaped values. + stateIdentity = "state-id:" + cfg.StateID + } + + m := &Manager{ + config: cfg, + store: store, + stateKey: GenerateStateKey("sql", stateIdentity, query, cfg.Column, cfg.Direction), + // Explicit type from config is authoritative and cannot be auto-adjusted. + typeLocked: cfg.Type != "", + logger: logger, + } + + if err := m.loadState(); err != nil { + return nil, fmt.Errorf("failed to initialize cursor state: %w", err) + } + + return m, nil +} + +// Close releases resources held by the manager. +// This must be called when the MetricSet is closed to release statestore resources. +// Close is idempotent - calling it multiple times is safe. +func (m *Manager) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.store != nil { + err := m.store.Close() + m.store = nil + return err + } + return nil +} + +// loadState loads cursor state from the store or initializes with the default value. +func (m *Manager) loadState() error { + m.mu.Lock() + defer m.mu.Unlock() + + state, err := m.store.Load(m.stateKey) + if err != nil { + m.logger.Warnf("Failed to load cursor state, will use default: %v", err) + state = nil + } + + if state == nil || !m.isStateValid(state) { + return m.initDefault() + } + + // In auto mode (cursor.type omitted), state becomes authoritative once loaded. + if m.config.Type == "" { + m.config.Type = state.CursorType + m.typeLocked = true + } + + // Parse the stored value + val, err := ParseValue(state.CursorValue, state.CursorType) + if err != nil { + m.logger.Warnf("Invalid cursor state value, using default: %v", err) + return m.initDefault() + } + + m.cursorValue = val + m.logger.Infof("Cursor loaded: type=%s value=%s", state.CursorType, val.Raw) + return nil +} + +// isStateValid checks whether the loaded state is compatible with the current +// configuration. It returns false (and logs the reason) when the state is nil, +// has a version mismatch, or a cursor-type mismatch. +func (m *Manager) isStateValid(state *State) bool { + if state == nil { + return false + } + if state.Version != StateVersion { + m.logger.Warnf("Unsupported cursor state version %d (expected %d), using default", + state.Version, StateVersion) + return false + } + if !isValidCursorType(state.CursorType) { + m.logger.Warnf("Unsupported cursor type in state=%s, using default", state.CursorType) + return false + } + if state.CursorType != m.config.Type { + if m.config.Type != "" { + m.logger.Warnf("Cursor type mismatch (state=%s, config=%s), using default", + state.CursorType, m.config.Type) + return false + } + } + return true +} + +// initDefault initializes the cursor with the default value from config. +// Caller must hold m.mu. +func (m *Manager) initDefault() error { + if m.config.Type == "" { + inferredType, err := InferTypeFromDefaultValue(m.config.Default) + if err != nil { + return fmt.Errorf("failed to infer cursor type from default value %q: %w", m.config.Default, err) + } + m.config.Type = inferredType + m.logger.Infof("Cursor type inferred from default value: %s", inferredType) + } + + defaultVal, err := ParseValue(m.config.Default, m.config.Type) + if err != nil { + return fmt.Errorf("invalid default cursor value: %w", err) + } + + m.cursorValue = defaultVal + m.logger.Infof("Cursor initialized: column=%s, type=%s, default=%s", + m.config.Column, m.config.Type, defaultVal.Raw) + return nil +} + +// CursorValueForQuery returns the cursor value converted to a driver-compatible argument. +// The returned value is ready to be passed to db.QueryContext(). +func (m *Manager) CursorValueForQuery() interface{} { + m.mu.Lock() + defer m.mu.Unlock() + + if m.cursorValue == nil { + return nil + } + return m.cursorValue.ToDriverArg() +} + +// CursorValueString returns the cursor value as a string (for logging). +func (m *Manager) CursorValueString() string { + m.mu.Lock() + defer m.mu.Unlock() + + if m.cursorValue == nil { + return nilValuePlaceholder + } + return m.cursorValue.Raw +} + +// UpdateFromResults processes query results and updates the cursor. +// For ascending direction (default), it finds the maximum cursor value. +// For descending direction, it finds the minimum cursor value. +// The selected value is persisted as the new cursor state. +// +// The function is resilient to errors: +// - Missing cursor column: logs error, skips that row +// - NULL cursor value: logs warning, skips that row +// - Parse error: logs error, skips that row +// - If all rows have issues: cursor remains unchanged +// +// Returns an error only if state persistence fails (events are already emitted at this point). +func (m *Manager) UpdateFromResults(rows []mapstr.M) error { + m.mu.Lock() + defer m.mu.Unlock() + + if len(rows) == 0 { + m.logger.Debug("No rows returned, cursor unchanged") + return nil + } + + descending := m.config.Direction == CursorDirectionDesc + columnLower := strings.ToLower(m.config.Column) + var bestValue *Value + var processed int + var foundCount int + + // In auto mode, infer the best type from the returned rows before parsing. + if !m.typeLocked { + inferredType, err := inferTypeFromRows(rows, columnLower) + if err != nil { + m.logger.Debugf("Could not infer cursor type from result rows, using current type=%s: %v", + m.config.Type, err) + } else if inferredType != "" && inferredType != m.config.Type { + m.logger.Infof("Cursor type inferred from query results: %s", inferredType) + m.config.Type = inferredType + } + } + + for idx, row := range rows { + // Find the cursor column (case-insensitive) + var rawVal interface{} + var found bool + + for key, val := range row { + if strings.ToLower(key) == columnLower { + rawVal = val + found = true + break + } + } + + if !found { + // Don't spam logs per-row; a single prominent error is emitted below + // if the column is missing from all rows. + continue + } + foundCount++ + + if rawVal == nil { + m.logger.Warnf("NULL value in cursor column, row %d", idx+1) + continue + } + + val, err := FromDatabaseValue(rawVal, m.config.Type) + if err != nil && !m.typeLocked { + inferredType, inferErr := InferTypeFromDatabaseValue(rawVal) + if inferErr == nil && inferredType != "" && inferredType != m.config.Type { + fallbackVal, fallbackErr := FromDatabaseValue(rawVal, inferredType) + if fallbackErr == nil { + if bestValue != nil && bestValue.Type != inferredType { + convertedBest, convErr := ParseValue(bestValue.Raw, inferredType) + if convErr != nil { + m.logger.Errorf("Failed to convert existing cursor candidate from %s to %s in row %d: %v", + bestValue.Type, inferredType, idx+1, convErr) + continue + } + bestValue = convertedBest + } + m.logger.Infof("Cursor type adjusted from %s to %s based on row %d", + m.config.Type, inferredType, idx+1) + m.config.Type = inferredType + val = fallbackVal + err = nil + } + } + } + if err != nil { + m.logger.Errorf("Failed to parse cursor value in row %d: %v", idx+1, err) + continue + } + + processed++ + + // Track the best value (max for ascending, min for descending) + if bestValue == nil { + bestValue = val + continue + } + + cmp, err := val.Compare(bestValue) + if err != nil { + m.logger.Errorf("Failed to compare cursor values (a=%s, b=%s): %v", val.Raw, bestValue.Raw, err) + continue + } + + if descending { + if cmp < 0 { + bestValue = val + } + } else { + if cmp > 0 { + bestValue = val + } + } + } + + if bestValue == nil { + // If cursor column was not found in any row, emit a single prominent error + // explaining the likely misconfiguration. Otherwise, the column exists but + // all values were NULL or invalid. + if foundCount == 0 { + m.logger.Errorf("Cursor column %q was not found in any of the %d result rows. "+ + "The cursor column must be included in the SELECT clause of your SQL query. "+ + "The cursor will not advance until the column appears in results.", + m.config.Column, len(rows)) + } else { + m.logger.Warn("All cursor column values were NULL or invalid, cursor unchanged") + } + return nil + } + + previousValue := m.cursorValue + m.cursorValue = bestValue + + // Persist the new state + state := &State{ + Version: StateVersion, + CursorType: m.config.Type, + CursorValue: bestValue.Raw, + UpdatedAt: time.Now().UTC(), + } + + if m.store == nil { + return errors.New("cursor store is closed") + } + if err := m.store.Save(m.stateKey, state); err != nil { + // Revert in-memory state on save failure to keep consistency. + // We restore the exact previous *Value rather than re-parsing from string + // to avoid any edge-case parse issues. + m.cursorValue = previousValue + return fmt.Errorf("failed to save cursor state: %w", err) + } + + prevRaw := nilValuePlaceholder + if previousValue != nil { + prevRaw = previousValue.Raw + } + m.logger.Infof("Cursor updated: %s → %s (%d rows processed)", prevRaw, bestValue.Raw, processed) + return nil +} + +// GetStateKey returns the state key (for testing/debugging). +func (m *Manager) GetStateKey() string { + return m.stateKey +} + +// GetColumn returns the cursor column name. +func (m *Manager) GetColumn() string { + return m.config.Column +} + +func inferTypeFromRows(rows []mapstr.M, columnLower string) (string, error) { + inferredType := "" + for _, row := range rows { + var ( + rawVal interface{} + found bool + ) + + for key, val := range row { + if strings.ToLower(key) == columnLower { + rawVal = val + found = true + break + } + } + if !found || rawVal == nil { + continue + } + + rowType, err := InferTypeFromDatabaseValue(rawVal) + if err != nil { + continue + } + + mergedType, err := mergeInferredTypes(inferredType, rowType) + if err != nil { + return "", err + } + inferredType = mergedType + } + + if inferredType == "" { + return "", errors.New("no inferable cursor values found in rows") + } + return inferredType, nil +} + +func mergeInferredTypes(current, candidate string) (string, error) { + if current == "" { + return candidate, nil + } + if current == candidate { + return current, nil + } + + // date + timestamp should converge to timestamp. + if (current == CursorTypeDate && candidate == CursorTypeTimestamp) || + (current == CursorTypeTimestamp && candidate == CursorTypeDate) { + return CursorTypeTimestamp, nil + } + + if isNumericCursorType(current) && isNumericCursorType(candidate) { + return mergeNumericTypes(current, candidate), nil + } + + return "", fmt.Errorf("conflicting inferred cursor types: %s vs %s", current, candidate) +} + +func mergeNumericTypes(a, b string) string { + if a == CursorTypeDecimal || b == CursorTypeDecimal { + return CursorTypeDecimal + } + if a == CursorTypeFloat || b == CursorTypeFloat { + return CursorTypeFloat + } + return CursorTypeInteger +} + +func isNumericCursorType(t string) bool { + switch t { + case CursorTypeInteger, CursorTypeFloat, CursorTypeDecimal: + return true + default: + return false + } +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/cursor_auto_test.go b/x-pack/metricbeat/module/sql/query/cursor/cursor_auto_test.go new file mode 100644 index 000000000000..024e4517d66a --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/cursor_auto_test.go @@ -0,0 +1,182 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/paths" +) + +func TestManagerAutoType_InferFromDefault(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "id", + Default: "0", + } + + store, _ := newTestStore(t, beatPaths, logger) + mgr, err := NewManager(cfg, store, "localhost", "SELECT id FROM t WHERE id > :cursor ORDER BY id", logger) + require.NoError(t, err) + defer mgr.Close() + + assert.Equal(t, "0", mgr.CursorValueString()) + assert.EqualValues(t, int64(0), mgr.CursorValueForQuery()) + + err = mgr.UpdateFromResults([]mapstr.M{ + {"id": int64(100)}, + {"id": int64(200)}, + }) + require.NoError(t, err) + assert.Equal(t, "200", mgr.CursorValueString()) + assert.EqualValues(t, int64(200), mgr.CursorValueForQuery()) +} + +func TestManagerAutoType_RefineFromRowsAndReloadFromState(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "price", + Default: "0", + } + host := "localhost" + query := "SELECT price FROM t WHERE price > :cursor ORDER BY price" + + store1, _ := newTestStore(t, beatPaths, logger) + mgr1, err := NewManager(cfg, store1, host, query, logger) + require.NoError(t, err) + + err = mgr1.UpdateFromResults([]mapstr.M{ + {"price": []byte("10.25")}, + {"price": []byte("20.50")}, + }) + require.NoError(t, err) + assert.Equal(t, "20.5", mgr1.CursorValueString()) + arg1, ok := mgr1.CursorValueForQuery().(string) + require.True(t, ok) + assert.Equal(t, "20.5", arg1) + require.NoError(t, mgr1.Close()) + + store2, _ := newTestStore(t, beatPaths, logger) + mgr2, err := NewManager(cfg, store2, host, query, logger) + require.NoError(t, err) + defer mgr2.Close() + + assert.Equal(t, "20.5", mgr2.CursorValueString()) + arg2, ok := mgr2.CursorValueForQuery().(string) + require.True(t, ok) + assert.Equal(t, "20.5", arg2) +} + +func TestManagerStateID_PreservesStateAcrossDSNChanges(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + StateID: "payments-prod", + } + query := "SELECT id FROM t WHERE id > :cursor ORDER BY id" + + store1, _ := newTestStore(t, beatPaths, logger) + mgr1, err := NewManager(cfg, store1, "postgres://user:oldpass@localhost:5432/prod", query, logger) + require.NoError(t, err) + + err = mgr1.UpdateFromResults([]mapstr.M{ + {"id": int64(150)}, + }) + require.NoError(t, err) + require.NoError(t, mgr1.Close()) + + store2, _ := newTestStore(t, beatPaths, logger) + mgr2, err := NewManager(cfg, store2, "postgres://user:newpass@localhost:5432/prod", query, logger) + require.NoError(t, err) + defer mgr2.Close() + + assert.Equal(t, "150", mgr2.CursorValueString(), "state_id should keep cursor continuity across DSN changes") +} + +func TestManagerWithoutStateID_ResetsOnDSNChanges(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + query := "SELECT id FROM t WHERE id > :cursor ORDER BY id" + + store1, _ := newTestStore(t, beatPaths, logger) + mgr1, err := NewManager(cfg, store1, "postgres://user:oldpass@localhost:5432/prod", query, logger) + require.NoError(t, err) + + err = mgr1.UpdateFromResults([]mapstr.M{ + {"id": int64(150)}, + }) + require.NoError(t, err) + require.NoError(t, mgr1.Close()) + + store2, _ := newTestStore(t, beatPaths, logger) + mgr2, err := NewManager(cfg, store2, "postgres://user:newpass@localhost:5432/prod", query, logger) + require.NoError(t, err) + defer mgr2.Close() + + assert.Equal(t, "0", mgr2.CursorValueString(), "without state_id, DSN change should create a new state key") +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/cursor_test.go b/x-pack/metricbeat/module/sql/query/cursor/cursor_test.go new file mode 100644 index 000000000000..803745e40e9f --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/cursor_test.go @@ -0,0 +1,1113 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/paths" +) + +func setupTestManager(t *testing.T, cfg Config) (*Manager, func()) { + t.Helper() + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + logger := logp.NewNopLogger() + + store, _ := newTestStore(t, beatPaths, logger) + + mgr, err := NewManager( + cfg, + store, + "localhost:5432", + "SELECT * FROM logs WHERE id > :cursor ORDER BY id", + logger, + ) + require.NoError(t, err) + + cleanup := func() { + mgr.Close() + } + + return mgr, cleanup +} + +func TestNewManager(t *testing.T) { + tests := []struct { + name string + cfg Config + query string + wantErr bool + errMsg string + }{ + { + name: "valid config", + cfg: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + }, + query: "SELECT * FROM logs WHERE id > :cursor", + wantErr: false, + }, + { + name: "invalid config - missing column", + cfg: Config{ + Enabled: true, + Column: "", + Type: CursorTypeInteger, + Default: "0", + }, + query: "SELECT * FROM logs WHERE id > :cursor", + wantErr: true, + errMsg: "cursor.column is required", + }, + { + name: "missing cursor placeholder", + cfg: Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + }, + query: "SELECT * FROM logs WHERE id > 0", + wantErr: true, + errMsg: "query must contain :cursor placeholder", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + logger := logp.NewNopLogger() + store, _ := newTestStore(t, beatPaths, logger) + + mgr, err := NewManager(tt.cfg, store, "host", tt.query, logger) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMsg) + store.Close() + return + } + require.NoError(t, err) + require.NotNil(t, mgr) + mgr.Close() + }) + } +} + +func TestManagerCursorValueForQuery(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "100", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + // Should return the default value initially + val := mgr.CursorValueForQuery() + assert.Equal(t, int64(100), val) + + // String version + assert.Equal(t, "100", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_Integer(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + // Initial value + assert.Equal(t, "0", mgr.CursorValueString()) + + // Update with results + rows := []mapstr.M{ + {"id": int64(100), "data": "row1"}, + {"id": int64(200), "data": "row2"}, + {"id": int64(150), "data": "row3"}, + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should have the max value (200) + assert.Equal(t, "200", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_Timestamp(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + logger := logp.NewNopLogger() + store, _ := newTestStore(t, beatPaths, logger) + + cfg := Config{ + Enabled: true, + Column: "created_at", + Type: CursorTypeTimestamp, + Default: "2024-01-01T00:00:00Z", + } + + mgr, err := NewManager( + cfg, + store, + "localhost", + "SELECT * FROM logs WHERE created_at > :cursor ORDER BY created_at", + logger, + ) + require.NoError(t, err) + defer mgr.Close() + + // Initial value + assert.Equal(t, "2024-01-01T00:00:00Z", mgr.CursorValueString()) + + // Update with results + t1 := time.Date(2024, 6, 15, 10, 0, 0, 0, time.UTC) + t2 := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC) // max + t3 := time.Date(2024, 6, 15, 11, 0, 0, 0, time.UTC) + + rows := []mapstr.M{ + {"created_at": t1, "data": "row1"}, + {"created_at": t2, "data": "row2"}, + {"created_at": t3, "data": "row3"}, + } + + err = mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should have the max timestamp + assert.Equal(t, "2024-06-15T12:00:00Z", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_EmptyResults(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "100", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + // Update with empty results + err := mgr.UpdateFromResults([]mapstr.M{}) + require.NoError(t, err) + + // Cursor should be unchanged + assert.Equal(t, "100", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_NullValues(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + // Update with some NULL values + rows := []mapstr.M{ + {"id": nil, "data": "row1"}, // NULL + {"id": int64(100), "data": "row2"}, // valid + {"id": nil, "data": "row3"}, // NULL + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should use the valid value (100) + assert.Equal(t, "100", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_AllNullValues(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "50", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + // Update with all NULL values + rows := []mapstr.M{ + {"id": nil, "data": "row1"}, + {"id": nil, "data": "row2"}, + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Cursor should be unchanged + assert.Equal(t, "50", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_MissingColumn(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + // Update with rows missing the cursor column + rows := []mapstr.M{ + {"other_column": int64(100), "data": "row1"}, + {"id": int64(200), "data": "row2"}, // this one has it + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should use the value from the row that has the column + assert.Equal(t, "200", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_CaseInsensitiveColumn(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "ID", // uppercase in config + Type: CursorTypeInteger, + Default: "0", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + // Update with lowercase column name in result + rows := []mapstr.M{ + {"id": int64(100), "data": "row1"}, // lowercase in result + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should match case-insensitively + assert.Equal(t, "100", mgr.CursorValueString()) +} + +func TestManagerClose(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + + mgr, _ := setupTestManager(t, cfg) + + // First close + err := mgr.Close() + require.NoError(t, err) + + // Second close should also succeed (idempotent) + err = mgr.Close() + require.NoError(t, err) +} + +func TestManagerGetStateKey(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + key := mgr.GetStateKey() + assert.NotEmpty(t, key) + assert.Contains(t, key, "sql-cursor::") +} + +func TestManagerGetColumn(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "my_column", + Type: CursorTypeInteger, + Default: "0", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + assert.Equal(t, "my_column", mgr.GetColumn()) +} + +func TestManagerStatePersistence(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + host := "localhost:5432" + query := "SELECT * FROM logs WHERE id > :cursor ORDER BY id" + + logger := logp.NewNopLogger() + + // Create first manager and update cursor + store1, _ := newTestStore(t, beatPaths, logger) + + mgr1, err := NewManager(cfg, store1, host, query, logger) + require.NoError(t, err) + + rows := []mapstr.M{ + {"id": int64(500), "data": "row1"}, + } + err = mgr1.UpdateFromResults(rows) + require.NoError(t, err) + assert.Equal(t, "500", mgr1.CursorValueString()) + + mgr1.Close() + + // Create second manager - should load persisted state + store2, _ := newTestStore(t, beatPaths, logger) + + mgr2, err := NewManager(cfg, store2, host, query, logger) + require.NoError(t, err) + defer mgr2.Close() + + // Should have the persisted value + assert.Equal(t, "500", mgr2.CursorValueString()) +} + +// ============================================================================ +// Descending scan tests +// ============================================================================ + +func setupTestManagerWithDirection(t *testing.T, cfg Config) (*Manager, func()) { + t.Helper() + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + logger := logp.NewNopLogger() + + store, _ := newTestStore(t, beatPaths, logger) + + mgr, err := NewManager( + cfg, + store, + "localhost:5432", + "SELECT * FROM logs WHERE id < :cursor ORDER BY id DESC", + logger, + ) + require.NoError(t, err) + + cleanup := func() { + mgr.Close() + } + + return mgr, cleanup +} + +func TestManagerUpdateFromResults_Descending(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "99999", + Direction: CursorDirectionDesc, + } + + mgr, cleanup := setupTestManagerWithDirection(t, cfg) + defer cleanup() + + // Initial value + assert.Equal(t, "99999", mgr.CursorValueString()) + + // Update with results - should track MINIMUM value (200) + rows := []mapstr.M{ + {"id": int64(500), "data": "row1"}, + {"id": int64(200), "data": "row2"}, + {"id": int64(350), "data": "row3"}, + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should have the min value (200) for descending scan + assert.Equal(t, "200", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_DescendingWithNulls(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "99999", + Direction: CursorDirectionDesc, + } + + mgr, cleanup := setupTestManagerWithDirection(t, cfg) + defer cleanup() + + // Update with some NULL values - should find min among valid values + rows := []mapstr.M{ + {"id": nil, "data": "row1"}, + {"id": int64(300), "data": "row2"}, + {"id": int64(100), "data": "row3"}, + {"id": nil, "data": "row4"}, + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should have the min valid value (100) + assert.Equal(t, "100", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_AscendingDefault(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + // Direction defaults to "asc" when empty + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + Direction: "", // should default to asc + } + + // Validate sets default + err := cfg.Validate() + require.NoError(t, err) + assert.Equal(t, CursorDirectionAsc, cfg.Direction) + + mgr, cleanup := setupTestManagerWithDirection(t, cfg) + defer cleanup() + + rows := []mapstr.M{ + {"id": int64(100), "data": "row1"}, + {"id": int64(500), "data": "row2"}, + {"id": int64(300), "data": "row3"}, + } + + err = mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Ascending: should have the max value (500) + assert.Equal(t, "500", mgr.CursorValueString()) +} + +// ============================================================================ +// State resilience tests (loadState coverage) +// ============================================================================ + +func TestManagerLoadState_VersionMismatch(t *testing.T) { + // When stored state has a different version, Manager should fall back to default + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, Config: tmpDir, Data: tmpDir, Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + host := "localhost" + query := "SELECT * FROM t WHERE id > :cursor" + + // First: create a manager, update cursor to 500, close + store1, _ := newTestStore(t, beatPaths, logger) + mgr1, err := NewManager(cfg, store1, host, query, logger) + require.NoError(t, err) + + err = mgr1.UpdateFromResults([]mapstr.M{{"id": int64(500)}}) + require.NoError(t, err) + assert.Equal(t, "500", mgr1.CursorValueString()) + mgr1.Close() + + // Now tamper with the state: save a state with version=99 + store2, _ := newTestStore(t, beatPaths, logger) + key := GenerateStateKey("sql", host, query, "id", CursorDirectionAsc) + err = store2.Save(key, &State{ + Version: 99, // Wrong version + CursorType: CursorTypeInteger, + CursorValue: "500", + UpdatedAt: time.Now().UTC(), + }) + require.NoError(t, err) + store2.Close() + + // Create a new manager — should detect version mismatch and use default + store3, _ := newTestStore(t, beatPaths, logger) + mgr3, err := NewManager(cfg, store3, host, query, logger) + require.NoError(t, err) + defer mgr3.Close() + + assert.Equal(t, "0", mgr3.CursorValueString(), "Should fall back to default on version mismatch") +} + +func TestManagerLoadState_TypeMismatch(t *testing.T) { + // When stored state has a different cursor type, Manager should fall back to default + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, Config: tmpDir, Data: tmpDir, Logs: tmpDir, + } + logger := logp.NewNopLogger() + + host := "localhost" + query := "SELECT * FROM t WHERE ts > :cursor" + + // Save state as integer type — we need to compute the key that the *new* config + // would produce. The new config uses CursorTypeTimestamp + default direction "asc". + // However, the stored state has CursorTypeInteger. The manager detects the type + // mismatch at load time and falls back to default. + cfg := Config{ + Enabled: true, + Column: "ts", + Type: CursorTypeTimestamp, // Different from stored state + Default: "2024-01-01T00:00:00Z", + } + // Validate to set direction default + require.NoError(t, cfg.Validate()) + + store1, _ := newTestStore(t, beatPaths, logger) + key := GenerateStateKey("sql", host, query, "ts", cfg.Direction) + require.NoError(t, store1.Save(key, &State{ + Version: StateVersion, + CursorType: CursorTypeInteger, // Mismatch with config + CursorValue: "500", + UpdatedAt: time.Now().UTC(), + })) + store1.Close() + + // Create manager with timestamp config — should detect type mismatch and use default + store2, _ := newTestStore(t, beatPaths, logger) + mgr, err := NewManager(cfg, store2, host, query, logger) + require.NoError(t, err) + defer mgr.Close() + + assert.Equal(t, "2024-01-01T00:00:00Z", mgr.CursorValueString(), "Should fall back to default on type mismatch") +} + +func TestManagerLoadState_CorruptedValue(t *testing.T) { + // When stored cursor value can't be parsed, Manager should fall back to default + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, Config: tmpDir, Data: tmpDir, Logs: tmpDir, + } + logger := logp.NewNopLogger() + + host := "localhost" + query := "SELECT * FROM t WHERE id > :cursor" + + // Create config first so we can compute the correct state key + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + // Validate to set direction default + require.NoError(t, cfg.Validate()) + + // Save state with an unparseable integer value + store1, _ := newTestStore(t, beatPaths, logger) + key := GenerateStateKey("sql", host, query, "id", cfg.Direction) + require.NoError(t, store1.Save(key, &State{ + Version: StateVersion, + CursorType: CursorTypeInteger, + CursorValue: "not-a-number", // Corrupted + UpdatedAt: time.Now().UTC(), + })) + store1.Close() + + // Create manager — should detect corrupt value and use default + store2, _ := newTestStore(t, beatPaths, logger) + mgr, err := NewManager(cfg, store2, host, query, logger) + require.NoError(t, err) + defer mgr.Close() + + assert.Equal(t, "0", mgr.CursorValueString(), "Should fall back to default on corrupted state value") +} + +// ============================================================================ +// UpdateFromResults edge case tests +// ============================================================================ + +func TestManagerUpdateFromResults_AllColumnsMissing(t *testing.T) { + // When ALL rows are missing the cursor column, cursor should remain unchanged + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "42", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + rows := []mapstr.M{ + {"other_col": int64(100)}, + {"other_col": int64(200)}, + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Cursor unchanged — no valid column found + assert.Equal(t, "42", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_ParseErrors(t *testing.T) { + // When cursor column values can't be parsed, they should be skipped + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + rows := []mapstr.M{ + {"id": "not-a-number", "data": "bad1"}, // parse error + {"id": int64(100), "data": "good"}, // valid + {"id": "also-not-a-number", "data": "bad2"}, // parse error + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should use the one valid value + assert.Equal(t, "100", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_MixedNullAndMissingAndValid(t *testing.T) { + // Mix of NULL values, missing columns, parse errors, and valid values + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + cfg := Config{ + Enabled: true, + Column: "id", + Type: CursorTypeInteger, + Default: "0", + } + + mgr, cleanup := setupTestManager(t, cfg) + defer cleanup() + + rows := []mapstr.M{ + {"id": nil, "data": "null row"}, // NULL + {"other_col": int64(999)}, // missing column + {"id": "not-a-number"}, // parse error + {"id": int64(50), "data": "valid 1"}, // valid + {"id": nil}, // NULL + {"id": int64(75), "data": "valid 2"}, // valid — the max + {"id": "garbage"}, // parse error + } + + err := mgr.UpdateFromResults(rows) + require.NoError(t, err) + + // Should find max among valid values: max(50, 75) = 75 + assert.Equal(t, "75", mgr.CursorValueString()) +} + +func TestManagerUpdateFromResults_FloatCursor(t *testing.T) { + // Verify float cursor works end-to-end through Manager + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, Config: tmpDir, Data: tmpDir, Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "score", + Type: CursorTypeFloat, + Default: "0.0", + } + + store, _ := newTestStore(t, beatPaths, logger) + + mgr, err := NewManager(cfg, store, "host", + "SELECT * FROM t WHERE score > :cursor ORDER BY score", logger) + require.NoError(t, err) + defer mgr.Close() + + assert.Equal(t, "0", mgr.CursorValueString()) + + rows := []mapstr.M{ + {"score": float64(1.5)}, + {"score": float64(3.14)}, + {"score": float64(2.7)}, + } + err = mgr.UpdateFromResults(rows) + require.NoError(t, err) + assert.Equal(t, "3.14", mgr.CursorValueString()) + + // Verify driver arg is float64 + val := mgr.CursorValueForQuery() + _, ok := val.(float64) + assert.True(t, ok, "float cursor should return float64 driver arg") +} + +func TestManagerUpdateFromResults_DecimalCursor(t *testing.T) { + // Verify decimal cursor works end-to-end through Manager + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, Config: tmpDir, Data: tmpDir, Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "price", + Type: CursorTypeDecimal, + Default: "0.00", + } + + store, _ := newTestStore(t, beatPaths, logger) + + mgr, err := NewManager(cfg, store, "host", + "SELECT * FROM t WHERE price > :cursor ORDER BY price", logger) + require.NoError(t, err) + defer mgr.Close() + + assert.Equal(t, "0", mgr.CursorValueString()) + + // Simulate DB returning strings (common for DECIMAL columns via []byte -> string) + rows := []mapstr.M{ + {"price": "10.25"}, + {"price": "99.99"}, + {"price": "50.50"}, + } + err = mgr.UpdateFromResults(rows) + require.NoError(t, err) + assert.Equal(t, "99.99", mgr.CursorValueString()) + + // Verify driver arg is string (for decimal) + val := mgr.CursorValueForQuery() + _, ok := val.(string) + assert.True(t, ok, "decimal cursor should return string driver arg") +} + +func TestManagerUpdateFromResults_DecimalPersistenceRoundTrip(t *testing.T) { + // Verify decimal cursor survives store -> load with exact precision + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, Config: tmpDir, Data: tmpDir, Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "price", + Type: CursorTypeDecimal, + Default: "0.00", + } + host := "localhost" + query := "SELECT * FROM t WHERE price > :cursor ORDER BY price" + + // First manager: update to a precise decimal value + store1, _ := newTestStore(t, beatPaths, logger) + mgr1, err := NewManager(cfg, store1, host, query, logger) + require.NoError(t, err) + + err = mgr1.UpdateFromResults([]mapstr.M{{"price": "123456.789012"}}) + require.NoError(t, err) + assert.Equal(t, "123456.789012", mgr1.CursorValueString()) + mgr1.Close() + + // Second manager: should load exact same value from store + store2, _ := newTestStore(t, beatPaths, logger) + mgr2, err := NewManager(cfg, store2, host, query, logger) + require.NoError(t, err) + defer mgr2.Close() + + assert.Equal(t, "123456.789012", mgr2.CursorValueString(), + "Decimal value must survive store->load round trip with exact precision") +} + +// ============================================================================ +// Timestamp coverage hardening (Manager-level) +// ============================================================================ + +func TestManagerUpdateFromResults_TimestampDescending(t *testing.T) { + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, Config: tmpDir, Data: tmpDir, Logs: tmpDir, + } + logger := logp.NewNopLogger() + + store, _ := newTestStore(t, beatPaths, logger) + + cfg := Config{ + Enabled: true, + Column: "created_at", + Type: CursorTypeTimestamp, + Default: "2099-12-31T23:59:59Z", + Direction: CursorDirectionDesc, + } + + mgr, err := NewManager( + cfg, + store, + "localhost", + "SELECT * FROM logs WHERE created_at < :cursor ORDER BY created_at DESC", + logger, + ) + require.NoError(t, err) + defer mgr.Close() + + // Initial value should be the high default + assert.Equal(t, "2099-12-31T23:59:59Z", mgr.CursorValueString()) + + // Update with results — descending should track MINIMUM timestamp + t1 := time.Date(2024, 6, 15, 10, 0, 0, 0, time.UTC) // min + t2 := time.Date(2024, 6, 15, 14, 0, 0, 0, time.UTC) + t3 := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC) + + rows := []mapstr.M{ + {"created_at": t2, "data": "row1"}, + {"created_at": t1, "data": "row2"}, + {"created_at": t3, "data": "row3"}, + } + + err = mgr.UpdateFromResults(rows) + require.NoError(t, err) + assert.Equal(t, "2024-06-15T10:00:00Z", mgr.CursorValueString(), + "descending cursor should track the minimum timestamp") + + // Second batch — cursor should advance (decrease) further + t4 := time.Date(2024, 6, 15, 8, 0, 0, 0, time.UTC) // new min + t5 := time.Date(2024, 6, 15, 9, 0, 0, 0, time.UTC) + + rows2 := []mapstr.M{ + {"created_at": t4, "data": "row4"}, + {"created_at": t5, "data": "row5"}, + } + + err = mgr.UpdateFromResults(rows2) + require.NoError(t, err) + assert.Equal(t, "2024-06-15T08:00:00Z", mgr.CursorValueString(), + "descending cursor should advance to the new minimum") +} + +func TestManagerTimestampPersistenceRoundTrip(t *testing.T) { + // Critical test: timestamp with nanosecond precision must survive + // Manager1.Update -> Close -> Manager2.Load without losing precision. + if testing.Short() { + t.Skip("skipping manager test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, Config: tmpDir, Data: tmpDir, Logs: tmpDir, + } + logger := logp.NewNopLogger() + + cfg := Config{ + Enabled: true, + Column: "created_at", + Type: CursorTypeTimestamp, + Default: "2024-01-01T00:00:00Z", + } + host := "localhost:5432" + query := "SELECT * FROM logs WHERE created_at > :cursor ORDER BY created_at" + + // --- First manager: update to a timestamp with nanosecond precision --- + store1, _ := newTestStore(t, beatPaths, logger) + mgr1, err := NewManager(cfg, store1, host, query, logger) + require.NoError(t, err) + + tsWithNanos := time.Date(2024, 6, 15, 10, 30, 0, 123456789, time.UTC) + err = mgr1.UpdateFromResults([]mapstr.M{ + {"created_at": tsWithNanos, "data": "row1"}, + }) + require.NoError(t, err) + assert.Equal(t, "2024-06-15T10:30:00.123456789Z", mgr1.CursorValueString()) + + // Verify ToDriverArg preserves nanoseconds + arg1 := mgr1.CursorValueForQuery() + tm1, ok := arg1.(time.Time) + require.True(t, ok) + assert.Equal(t, 123456789, tm1.Nanosecond()) + + mgr1.Close() + + // --- Second manager: should load exact same timestamp from store --- + store2, _ := newTestStore(t, beatPaths, logger) + mgr2, err := NewManager(cfg, store2, host, query, logger) + require.NoError(t, err) + defer mgr2.Close() + + assert.Equal(t, "2024-06-15T10:30:00.123456789Z", mgr2.CursorValueString(), + "Timestamp with nanoseconds must survive store->load round trip") + + // Verify ToDriverArg still produces the correct time.Time after reload + arg2 := mgr2.CursorValueForQuery() + tm2, ok := arg2.(time.Time) + require.True(t, ok) + assert.Equal(t, 123456789, tm2.Nanosecond(), + "Nanosecond component must survive store->close->load->ToDriverArg cycle") + assert.True(t, tm1.Equal(tm2), + "time.Time from first and second manager must be exactly equal") +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/doc.go b/x-pack/metricbeat/module/sql/query/cursor/doc.go new file mode 100644 index 000000000000..ed3bc2e4b552 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/doc.go @@ -0,0 +1,235 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Package cursor implements cursor-based incremental data fetching for the +// SQL metricbeat module. It enables the sql/query MetricSet to track the last +// fetched row and retrieve only new data on subsequent collection cycles. +// +// # Problem +// +// Without cursor support, the SQL Input module executes the same query every +// collection cycle, fetching all rows repeatedly. This causes duplicate data +// ingestion, unnecessary database load, and wasted resources. +// +// # Solution +// +// The cursor package provides state management that persists the value of a +// designated cursor column (for example, an auto-increment ID or timestamp) across +// collection cycles. On each run, the persisted value is substituted into a +// parameterized query via the :cursor placeholder, so only rows beyond the +// last checkpoint are fetched. +// +// # Architecture +// +// The package is organized into the following components: +// +// - [Config]: User-facing configuration (column, optional type, default, optional state_id) with validation +// - [Value]: Type-safe cursor value with serialization and comparison support +// - [Store]: Persistence layer backed by libbeat/statestore with memlog backend +// - [Manager]: Lifecycle coordinator that ties configuration, storage, and query execution together +// - Placeholder translation: Converts the :cursor placeholder to driver-specific syntax ($1, ?, :cursor_val, @p1) +// +// # Cursor Types +// +// Five cursor types are supported: +// +// - "integer": For auto-increment IDs, sequence numbers. Stored as int64. +// - "timestamp": For TIMESTAMP/DATETIME columns. Stored as Unix nanoseconds (int64) in UTC. +// - "date": For DATE columns. Stored as YYYY-MM-DD string. +// - "float": For FLOAT/DOUBLE/REAL columns. Stored as float64. Subject to IEEE 754 +// precision limits -- boundary rows may be duplicated or skipped at the 15th+ +// significant digit. Best for columns where slight imprecision is acceptable. +// - "decimal": For DECIMAL/NUMERIC columns. Uses shopspring/decimal for exact +// arbitrary-precision arithmetic. No data loss at boundaries. Best for financial +// data and exact numeric columns. +// +// cursor.type is optional. When omitted, the manager infers an initial type from +// cursor.default and can refine it from returned row values in auto mode. When +// cursor.type is explicitly configured, that type is authoritative. +// +// # Scan Direction +// +// By default, the cursor tracks the maximum value from each batch (ascending scan). +// For descending scans (ORDER BY col DESC with WHERE col < :cursor), set +// cursor.direction to "desc" to track the minimum value instead. +// +// # Database Support +// +// The placeholder translation layer supports: +// +// - PostgreSQL / CockroachDB: $1 +// - MySQL: ? +// - Oracle (godror): :cursor_val +// - MSSQL (sqlserver): @p1 +// +// The cursor value is always passed as a parameterized query argument, +// making it safe against SQL injection. +// +// # State Persistence +// +// Cursor state is persisted using libbeat/statestore with a memlog backend +// (the same proven backend used by Filebeat). State is stored at +// {data.path}/sql-cursor/ and survives process restarts and crashes. +// +// Each cursor gets a unique state key generated by [GenerateStateKey], which +// length-prefixes and hashes the following components via xxhash (no secrets +// appear in the key): +// +// - Input type ("sql") — fixed namespace, never changes +// - State identity — one of: (a) full database URI/DSN when cursor.state_id +// is unset (not normalized; includes host, port, database name, credentials, +// and connection parameters), or (b) cursor.state_id when set (a stable +// operator-provided identity) +// - Full query string — not normalized; exact byte match, so whitespace +// or capitalization changes produce a different key +// - Cursor column name +// - Cursor direction ("asc" or "desc") +// +// Any change to these components produces a different key, which resets the +// cursor to its configured default value. Examples of changes that reset: +// +// - Changing the database password or username in the DSN (when state_id is unset) +// - Adding connection parameters (e.g., ?sslmode=require) (when state_id is unset) +// - Reformatting the SQL query (whitespace, capitalization) +// - Renaming the cursor column +// - Switching direction from "asc" to "desc" +// - Changing cursor.state_id +// +// Examples of changes that do NOT reset the cursor: +// +// - Changing cursor.default (only used on first run) +// - Changing period or timeout (runtime settings, not part of the key) +// - Changing DSN credentials/parameters when cursor.state_id is set and unchanged +// +// The full URI is used (rather than just host:port) so that two databases +// on the same server produce distinct state keys. This is essential for +// multi-host configurations (hosts: [host1, host2]) where each host gets +// its own MetricSet, and for cross-database isolation (prod_db vs test_db). +// +// Known tradeoff: including the full DSN means that password rotation, +// username changes, or adding connection parameters will reset the cursor. +// This is a safe failure mode — it causes re-ingestion (duplicates), never +// data loss. To avoid cursor resets on credential changes, configure +// cursor.state_id as a stable logical source identifier. When state_id is +// unset and DSN changes, backing up and restoring the state directory does not +// help — the new DSN produces a different hash, so the restored state is never +// looked up. +// +// # Error Philosophy +// +// The cursor follows an "at-least-once" delivery model: +// +// - Events are emitted BEFORE the cursor is updated +// - If cursor persistence fails, the next cycle re-fetches the same rows +// - Duplicates are preferred over data loss +// - NULL cursor values, missing columns, and parse errors are logged and skipped per-row +// - "Column not found" and "all values NULL" are reported with distinct error messages +// - The cursor value is only updated if at least one valid value was found +// - Float cursors reject NaN and Inf values during default parsing/inference and DB result parsing +// +// NULL cursor values are never used to advance state. If your WHERE clause +// explicitly includes NULL rows (for example, "updated_at > :cursor OR updated_at IS NULL"), +// those NULL rows can be emitted repeatedly across collection cycles and restarts. +// This is expected at-least-once behavior; use downstream deduplication if needed. +// +// # Concurrency and Query Timeout +// +// The [Manager] is safe for concurrent use. All cursor reads and writes are +// protected by a mutex. The MetricSet uses a separate fetchMutex with TryLock +// to prevent overlapping collection cycles when a query takes longer than the +// configured period. +// +// Each cursor-based fetch is wrapped with context.WithTimeout using the +// module's configured timeout (which defaults to period). This prevents hung +// queries (e.g., table locks, network partitions) from blocking the goroutine +// indefinitely and causing all subsequent collection cycles to be skipped +// via fetchMutex.TryLock(). When a query exceeds the timeout, the context is +// cancelled, the database driver aborts the query, and the cursor remains +// unchanged (no data loss). +// +// The timeout is applied only to the cursor path. The non-cursor path does +// not enforce a timeout for backward compatibility with existing users whose +// queries may legitimately take longer than their configured period. +// +// # Resource Cleanup +// +// The MetricSet implements mb.Closer. When the module stops, [Manager.Close] +// is called, which closes the [Store] handle (decrementing the ref count). +// The shared memlog registry is managed by the Module and closed separately. +// Close is idempotent. +// +// # Usage +// +// Users configure the cursor in metricbeat.yml: +// +// metricbeat.modules: +// - module: sql +// metricsets: ["query"] +// period: 60s +// hosts: ["postgres://user:pass@localhost:5432/mydb"] +// driver: postgres +// sql_query: > +// SELECT id, data FROM events +// WHERE id > :cursor ORDER BY id ASC LIMIT 1000 +// sql_response_format: table +// cursor: +// enabled: true +// column: id +// type: integer +// default: "0" +// # state_id: payments-prod +// +// The :cursor placeholder is replaced at startup with the driver-specific +// syntax. At each collection cycle: +// +// 1. The current cursor value is loaded (from store or default). +// 2. The query executes with the cursor value as a parameterized argument. +// 3. All returned rows are emitted as events. +// 4. The cursor value from the batch is persisted (max for asc, min for desc). +// +// # Boundary Behavior: > vs >= +// +// When the cursor column has unique values (auto-increment IDs), use the > +// operator: WHERE id > :cursor. When the cursor column may have duplicates +// (timestamps, scores), use >= to avoid missing late-arriving rows that share +// the same value as the cursor. The >= operator causes the boundary row to be +// re-fetched (a duplicate), but prevents data loss. Configure Elasticsearch +// document IDs or deduplication pipelines when using >=. +// +// # Driver-specific Notes +// +// - MySQL: include parseTime=true in the DSN for timestamp cursors so the +// driver correctly handles time.Time query parameters. +// - Oracle: set session timezone to UTC for timestamp cursors via the godror +// alterSession DSN parameter, or timestamp comparisons may silently fail. +// Preflight: the host running Metricbeat must have Oracle Instant Client +// installed and libclntsh discoverable (macOS: DYLD_LIBRARY_PATH, +// Linux: LD_LIBRARY_PATH), otherwise startup/fetch will fail with DPI-1047. +// - Decimal: ToDriverArg returns string. Most drivers implicitly cast to +// DECIMAL, but if not, the user can add CAST(:cursor AS DECIMAL) in SQL. +// +// # Limitations +// +// - Single query only (not sql_queries with multiple queries) +// - Single cursor column (no composite cursors) +// - String, UUID, and ULID columns are not supported as cursor types (Phase 3) +// - MySQL backslash-escaped strings (for example, 'it\'s :cursor') may cause +// :cursor inside the literal to be misdetected; standard SQL ” escaping is handled +// - ORDER BY must match the cursor column and the configured direction +// - The cursor column must be included in the SELECT clause +// - All matching rows are loaded into memory; use LIMIT (500-5000 recommended) +// - Float cursors have IEEE 754 precision limits; use decimal for exact comparisons +// - Float cursors reject NaN and Inf (both as default values and from DB results) +// +// # Testing +// +// Unit tests cover configuration validation, type parsing (including edge +// cases like overflow, NULL, timezone conversion), placeholder translation, +// state persistence, and the full manager lifecycle. +// +// Integration tests verify end-to-end cursor operation against PostgreSQL, +// MySQL, Oracle, and MSSQL with all five cursor types (integer, timestamp, +// date, float, decimal), both scan directions, compound WHERE clauses, +// NULL handling, state persistence, and state isolation. +package cursor diff --git a/x-pack/metricbeat/module/sql/query/cursor/placeholder.go b/x-pack/metricbeat/module/sql/query/cursor/placeholder.go new file mode 100644 index 000000000000..ec3b9d735766 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/placeholder.go @@ -0,0 +1,204 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "fmt" + "strings" +) + +// CursorPlaceholder is the user-facing placeholder in SQL queries. +// Users write :cursor in their queries, and it gets translated to +// the appropriate database-specific placeholder. +const CursorPlaceholder = ":cursor" + +// ValidateQueryHasCursor checks if the query contains exactly one cursor +// placeholder in executable SQL (outside strings and comments). +// Returns an error if no placeholder is found or more than one is found. +func ValidateQueryHasCursor(query string) error { + count := CountPlaceholders(query) + if count == 0 { + return fmt.Errorf("query must contain %s placeholder when cursor is enabled", CursorPlaceholder) + } + if count > 1 { + return fmt.Errorf("query must contain exactly one %s placeholder, found %d", CursorPlaceholder, count) + } + return nil +} + +// TranslateQuery replaces the :cursor placeholder in executable SQL with the +// driver-specific parameterized placeholder. Occurrences inside quoted strings, +// identifiers, and SQL comments are left untouched. +// +// Driver placeholder mapping: +// - PostgreSQL, CockroachDB: $1 +// - MySQL: ? +// - Oracle: :cursor_val +// - MSSQL: @p1 +func TranslateQuery(query, driver string) string { + placeholder := getDriverPlaceholder(driver) + positions := findPlaceholderPositions(query) + if len(positions) == 0 { + return query + } + + // Pre-size: original length adjusted for each replacement. + sizeDelta := len(placeholder) - len(CursorPlaceholder) + var b strings.Builder + b.Grow(len(query) + sizeDelta*len(positions)) + + last := 0 + for _, pos := range positions { + b.WriteString(query[last:pos]) + b.WriteString(placeholder) + last = pos + len(CursorPlaceholder) + } + b.WriteString(query[last:]) + return b.String() +} + +// getDriverPlaceholder returns the parameterized placeholder syntax for the +// given driver name. +func getDriverPlaceholder(driver string) string { + switch strings.ToLower(driver) { + case "postgres", "postgresql", "cockroachdb", "cockroach": + return "$1" + case "mysql": + return "?" + case "oracle", "godror": + return ":cursor_val" + case "mssql", "sqlserver": + return "@p1" + default: + return "?" + } +} + +// CountPlaceholders returns the number of :cursor placeholders in executable +// SQL (outside quoted strings and comments). +func CountPlaceholders(query string) int { + return len(findPlaceholderPositions(query)) +} + +// Scanner states for findPlaceholderPositions. +const ( + stateNormal = iota + stateSingleQuote + stateDoubleQuote + stateBacktick + stateLineComment + stateBlockComment +) + +// findPlaceholderPositions returns the byte offsets of every :cursor +// placeholder that appears in executable SQL. The scanner skips content +// inside: +// - Single-quoted strings ('...'), with ” escape handling +// - Double-quoted identifiers ("..."), with "" escape handling +// - Backtick-quoted identifiers (`...`), with “ escape handling +// - Line comments (-- ...) +// - Block comments (/* ... */) +// +// Limitation: MySQL's default sql_mode allows backslash escapes inside +// strings (e.g., 'it\'s :cursor'). This scanner does not handle backslash +// escapes and would incorrectly treat the :cursor inside such a string as +// a real placeholder. This is acceptable because operator-written queries +// rarely use backslash escapes, and MySQL's NO_BACKSLASH_ESCAPES mode +// disables them entirely. The standard SQL escape (doubled quotes: +// 'it”s :cursor') is handled correctly. +func findPlaceholderPositions(query string) []int { + positions := make([]int, 0, 1) + n := len(query) + state := stateNormal + + for i := 0; i < n; i++ { + ch := query[i] + + switch state { + case stateNormal: + if ch == '-' && i+1 < n && query[i+1] == '-' { + state = stateLineComment + i++ + } else if ch == '/' && i+1 < n && query[i+1] == '*' { + state = stateBlockComment + i++ + } else if ch == '\'' { + state = stateSingleQuote + } else if ch == '"' { + state = stateDoubleQuote + } else if ch == '`' { + state = stateBacktick + } else if ch == ':' && matchesPlaceholder(query, i, n) { + positions = append(positions, i) + i += len(CursorPlaceholder) - 1 + } + + case stateSingleQuote: + if ch == '\'' { + if i+1 < n && query[i+1] == '\'' { + i++ // skip escaped quote + } else { + state = stateNormal + } + } + + case stateDoubleQuote: + if ch == '"' { + if i+1 < n && query[i+1] == '"' { + i++ // skip escaped quote + } else { + state = stateNormal + } + } + + case stateBacktick: + if ch == '`' { + if i+1 < n && query[i+1] == '`' { + i++ // skip escaped backtick + } else { + state = stateNormal + } + } + + case stateLineComment: + if ch == '\n' || ch == '\r' { + state = stateNormal + } + + case stateBlockComment: + if ch == '*' && i+1 < n && query[i+1] == '/' { + state = stateNormal + i++ + } + } + } + + return positions +} + +// matchesPlaceholder reports whether query[i:] starts with ":cursor" +// followed by a non-word character (or end of string). The caller must +// ensure query[i] == ':' before calling. +func matchesPlaceholder(query string, i, n int) bool { + end := i + len(CursorPlaceholder) + if end > n { + return false + } + if query[i:end] != CursorPlaceholder { + return false + } + if end == n { + return true + } + return !isWordChar(query[end]) +} + +// isWordChar reports whether ch is an ASCII letter, digit, or underscore. +func isWordChar(ch byte) bool { + return (ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || + ch == '_' +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/placeholder_test.go b/x-pack/metricbeat/module/sql/query/cursor/placeholder_test.go new file mode 100644 index 000000000000..6a3e08a7b0e3 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/placeholder_test.go @@ -0,0 +1,249 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateQueryHasCursor(t *testing.T) { + tests := []struct { + name string + query string + wantErr bool + errMsg string + }{ + { + name: "valid - single cursor", + query: "SELECT * FROM logs WHERE id > :cursor ORDER BY id", + wantErr: false, + }, + { + name: "valid - cursor in middle", + query: "SELECT * FROM logs WHERE timestamp > :cursor AND status = 'active'", + wantErr: false, + }, + { + name: "valid - cursor with LIMIT", + query: "SELECT * FROM logs WHERE id > :cursor ORDER BY id ASC LIMIT 1000", + wantErr: false, + }, + { + name: "no cursor placeholder", + query: "SELECT * FROM logs WHERE id > 0", + wantErr: true, + errMsg: "query must contain :cursor placeholder", + }, + { + name: "multiple cursor placeholders", + query: "SELECT * FROM logs WHERE id > :cursor AND updated_at > :cursor", + wantErr: true, + errMsg: "query must contain exactly one :cursor placeholder, found 2", + }, + { + name: "similar but not cursor", + query: "SELECT * FROM logs WHERE id > :cursor_value", + wantErr: true, + errMsg: "query must contain :cursor placeholder", + }, + { + name: "empty query", + query: "", + wantErr: true, + errMsg: "query must contain :cursor placeholder", + }, + { + name: "ignore placeholder inside single-quoted string", + query: "SELECT * FROM logs WHERE note = ':cursor' AND id > :cursor", + wantErr: false, + }, + { + name: "ignore placeholder inside line comment", + query: "SELECT * FROM logs -- :cursor\nWHERE id > :cursor", + wantErr: false, + }, + { + name: "ignore placeholder inside block comment", + query: "SELECT * FROM logs /* :cursor */ WHERE id > :cursor", + wantErr: false, + }, + { + name: "only placeholder in comment is invalid", + query: "SELECT * FROM logs /* :cursor */ WHERE id > 0", + wantErr: true, + errMsg: "query must contain :cursor placeholder", + }, + { + name: "ignore placeholder inside double-quoted identifier", + query: `SELECT * FROM "my:cursor" WHERE id > :cursor`, + wantErr: false, + }, + { + name: "ignore placeholder inside backtick-quoted identifier", + query: "SELECT * FROM `my:cursor` WHERE id > :cursor", + wantErr: false, + }, + { + name: "escaped single quote does not break out of string", + query: "SELECT * FROM t WHERE note = 'it''s :cursor' AND id > :cursor", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateQueryHasCursor(tt.query) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMsg) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestTranslateQuery(t *testing.T) { + baseQuery := "SELECT * FROM logs WHERE id > :cursor ORDER BY id ASC LIMIT 1000" + + tests := []struct { + name string + driver string + want string + }{ + { + name: "postgres", + driver: "postgres", + want: "SELECT * FROM logs WHERE id > $1 ORDER BY id ASC LIMIT 1000", + }, + { + name: "postgresql", + driver: "postgresql", + want: "SELECT * FROM logs WHERE id > $1 ORDER BY id ASC LIMIT 1000", + }, + { + name: "cockroachdb", + driver: "cockroachdb", + want: "SELECT * FROM logs WHERE id > $1 ORDER BY id ASC LIMIT 1000", + }, + { + name: "cockroach", + driver: "cockroach", + want: "SELECT * FROM logs WHERE id > $1 ORDER BY id ASC LIMIT 1000", + }, + { + name: "mysql", + driver: "mysql", + want: "SELECT * FROM logs WHERE id > ? ORDER BY id ASC LIMIT 1000", + }, + { + name: "oracle", + driver: "oracle", + want: "SELECT * FROM logs WHERE id > :cursor_val ORDER BY id ASC LIMIT 1000", + }, + { + name: "godror", + driver: "godror", + want: "SELECT * FROM logs WHERE id > :cursor_val ORDER BY id ASC LIMIT 1000", + }, + { + name: "mssql", + driver: "mssql", + want: "SELECT * FROM logs WHERE id > @p1 ORDER BY id ASC LIMIT 1000", + }, + { + name: "sqlserver", + driver: "sqlserver", + want: "SELECT * FROM logs WHERE id > @p1 ORDER BY id ASC LIMIT 1000", + }, + { + name: "unknown driver - defaults to ?", + driver: "unknown", + want: "SELECT * FROM logs WHERE id > ? ORDER BY id ASC LIMIT 1000", + }, + { + name: "case insensitive - POSTGRES", + driver: "POSTGRES", + want: "SELECT * FROM logs WHERE id > $1 ORDER BY id ASC LIMIT 1000", + }, + { + name: "case insensitive - MySQL", + driver: "MySQL", + want: "SELECT * FROM logs WHERE id > ? ORDER BY id ASC LIMIT 1000", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := TranslateQuery(baseQuery, tt.driver) + assert.Equal(t, tt.want, result) + }) + } + + t.Run("does not replace placeholder inside string or comments", func(t *testing.T) { + query := "SELECT ':cursor' AS s /* :cursor */ FROM logs -- :cursor\nWHERE id > :cursor" + got := TranslateQuery(query, "postgres") + want := "SELECT ':cursor' AS s /* :cursor */ FROM logs -- :cursor\nWHERE id > $1" + assert.Equal(t, want, got) + }) +} + +func TestCountPlaceholders(t *testing.T) { + tests := []struct { + name string + query string + want int + }{ + { + name: "no cursor", + query: "SELECT * FROM logs", + want: 0, + }, + { + name: "one cursor", + query: "SELECT * FROM logs WHERE id > :cursor", + want: 1, + }, + { + name: "two cursors", + query: "SELECT * FROM logs WHERE id > :cursor AND ts > :cursor", + want: 2, + }, + { + name: "cursor_value not matched", + query: "SELECT * FROM logs WHERE id > :cursor_value", + want: 0, + }, + { + name: "ignore placeholders in string and comments", + query: "SELECT ':cursor' /* :cursor */ FROM logs -- :cursor\nWHERE id > :cursor", + want: 1, + }, + { + name: "ignore placeholder in double-quoted identifier", + query: `SELECT * FROM "my:cursor" WHERE id > :cursor`, + want: 1, + }, + { + name: "ignore placeholder in backtick-quoted identifier", + query: "SELECT * FROM `my:cursor` WHERE id > :cursor", + want: 1, + }, + { + name: "escaped single quote does not break string", + query: "SELECT * FROM t WHERE note = 'it''s :cursor' AND id > :cursor", + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, CountPlaceholders(tt.query)) + }) + } +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/store.go b/x-pack/metricbeat/module/sql/query/cursor/store.go new file mode 100644 index 000000000000..f7ddb2841104 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/store.go @@ -0,0 +1,152 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/cespare/xxhash/v2" + + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/elastic-agent-libs/logp" +) + +// StateVersion is the current version of the state format. +// Increment this when making breaking changes to the State struct. +const StateVersion = 1 + +// stateStoreName is the statestore bucket name used for cursor entries. +const stateStoreName = "cursor-state" + +// State represents the persisted cursor state (versioned for future migrations) +type State struct { + Version int `json:"version"` + CursorType string `json:"cursor_type"` + CursorValue string `json:"cursor_value"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Store persists cursor state using libbeat/statestore with memlog backend. +// The Store does not own the registry — the caller (Module) manages its lifecycle. +type Store struct { + store *statestore.Store + logger *logp.Logger +} + +// NewStoreFromRegistry creates a Store using a shared statestore.Registry. +// The registry is NOT owned by this Store — the caller (Module) manages its lifecycle. +// Each call obtains a ref-counted Store handle from the shared registry. +func NewStoreFromRegistry(registry *statestore.Registry, logger *logp.Logger) (*Store, error) { + store, err := registry.Get(stateStoreName) + if err != nil { + return nil, fmt.Errorf("failed to open cursor store: %w", err) + } + + return &Store{ + store: store, + logger: logger, + }, nil +} + +// Load retrieves cursor state for the given key. +// Returns nil if the key doesn't exist (not an error). +func (s *Store) Load(key string) (*State, error) { + if s.store == nil { + return nil, errors.New("store is closed") + } + + exists, err := s.store.Has(key) + if err != nil { + return nil, fmt.Errorf("failed to check cursor state existence: %w", err) + } + if !exists { + return nil, nil + } + + var state State + if err := s.store.Get(key, &state); err != nil { + return nil, fmt.Errorf("failed to load cursor state: %w", err) + } + + return &state, nil +} + +// Save persists cursor state for the given key. +func (s *Store) Save(key string, state *State) error { + if s.store == nil { + return errors.New("store is closed") + } + if err := s.store.Set(key, state); err != nil { + return fmt.Errorf("failed to save cursor state: %w", err) + } + return nil +} + +// Close releases the store handle. Must be called when done. +// Close is idempotent — calling it multiple times is safe. +// Only the store handle is closed (decrementing the ref count). +// The shared registry is managed by the caller (Module). +func (s *Store) Close() error { + if s.store != nil { + err := s.store.Close() + s.store = nil + if err != nil { + return fmt.Errorf("failed to close store: %w", err) + } + } + return nil +} + +// GenerateStateKey creates a unique key for cursor state persistence. +// The key is based on module configuration to ensure separate state per unique config. +// +// IMPORTANT: Any change to the components below will cause the cursor to reset to +// its default value and re-ingest all data from scratch. This includes: +// +// Components that trigger cursor reset: +// - inputType: "sql" (for namespacing) - hardcoded, never changes +// - stateIdentity — one of: (a) full database URI/DSN (NOT normalized) when +// cursor.state_id is unset, or (b) cursor.state_id when set (stable across +// DSN changes). For DSN-based identity: changing host, password, or connection +// params resets cursor; includes database name for isolation (prod_db vs +// test_db on same server). Changing cursor.state_id also resets cursor. +// - query: Full query string (NOT normalized - exact byte match) +// - Adding/removing whitespace resets cursor +// - Changing SQL capitalization (SELECT → select) resets cursor +// - Changing LIMIT value resets cursor +// - Modifying WHERE clause resets cursor +// - cursorColumn: The column name being tracked +// - Renaming cursor column resets cursor +// - direction: The cursor scan direction ("asc" or "desc") +// - Changing direction resets cursor (prevents using a max-tracked value +// as a min-tracking starting point, or vice versa) +// +// Design rationale: +// - Safety: Query changes could affect result set semantics. Better to start +// fresh than risk missing data or duplicates from incompatible queries. +// - Simplicity: SQL normalization is complex and database-specific. Avoiding +// SQL parsing keeps implementation simple and reliable. +// - Isolation: Different databases on same server (e.g., prod_db vs test_db) +// must have separate cursor states. Including full DSN ensures this. +// - Operability: Optional cursor.state_id allows stable cursor continuity +// across credential/DSN changes when operators explicitly opt in. +// - Direction safety: A cursor value tracked as a maximum (asc) is semantically +// incompatible with minimum tracking (desc). Changing direction must reset. +// +// The combined string is hashed via xxhash, so no secrets appear in the stored key. +// Each part is length-prefixed to avoid ambiguity when parts contain the delimiter. +func GenerateStateKey(inputType, stateIdentity, query, cursorColumn, direction string) string { + keyParts := []string{inputType, stateIdentity, query, cursorColumn, direction} + + var b strings.Builder + for _, p := range keyParts { + fmt.Fprintf(&b, "%d:%s|", len(p), p) + } + hash := xxhash.Sum64String(b.String()) + return fmt.Sprintf("sql-cursor::%x", hash) +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/store_test.go b/x-pack/metricbeat/module/sql/query/cursor/store_test.go new file mode 100644 index 000000000000..bec7c1a72c32 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/store_test.go @@ -0,0 +1,363 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/beats/v7/libbeat/statestore/backend/memlog" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/paths" +) + +func TestGenerateStateKey(t *testing.T) { + tests := []struct { + name string + inputType string + host string + query string + cursorColumn string + direction string + }{ + { + name: "basic asc", + inputType: "sql", + host: "localhost:5432", + query: "SELECT * FROM logs WHERE id > :cursor", + cursorColumn: "id", + direction: "asc", + }, + { + name: "basic desc", + inputType: "sql", + host: "localhost:5432", + query: "SELECT * FROM logs WHERE id > :cursor", + cursorColumn: "id", + direction: "desc", + }, + { + name: "different host", + inputType: "sql", + host: "remotehost:5432", + query: "SELECT * FROM logs WHERE id > :cursor", + cursorColumn: "id", + direction: "asc", + }, + { + name: "different query", + inputType: "sql", + host: "localhost:5432", + query: "SELECT * FROM events WHERE id > :cursor", + cursorColumn: "id", + direction: "asc", + }, + { + name: "different column", + inputType: "sql", + host: "localhost:5432", + query: "SELECT * FROM logs WHERE id > :cursor", + cursorColumn: "event_id", + direction: "asc", + }, + } + + // Generate keys and ensure they're unique + keys := make(map[string]string) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + key := GenerateStateKey(tt.inputType, tt.host, tt.query, tt.cursorColumn, tt.direction) + + // Key should have expected prefix + assert.Contains(t, key, "sql-cursor::") + + // Key should be hex formatted + assert.Regexp(t, `^sql-cursor::[0-9a-f]+$`, key) + + // Keys should be unique for different inputs + identifier := tt.inputType + tt.host + tt.query + tt.cursorColumn + tt.direction + if existingKey, exists := keys[identifier]; exists { + assert.Equal(t, existingKey, key, "same inputs should produce same key") + } else { + keys[identifier] = key + } + }) + } + + // Verify different inputs produce different keys + key1 := GenerateStateKey("sql", "host1", "query", "col", "asc") + key2 := GenerateStateKey("sql", "host2", "query", "col", "asc") + assert.NotEqual(t, key1, key2, "different hosts should produce different keys") + + // Verify direction changes the key + key3 := GenerateStateKey("sql", "host", "query", "col", "asc") + key4 := GenerateStateKey("sql", "host", "query", "col", "desc") + assert.NotEqual(t, key3, key4, "changing direction should change the key") + + // Verify whitespace in query changes the key (no normalization) + key5 := GenerateStateKey("sql", "host", "SELECT * FROM logs", "col", "asc") + key6 := GenerateStateKey("sql", "host", "SELECT * FROM logs", "col", "asc") + assert.NotEqual(t, key5, key6, "whitespace differences should produce different keys") +} + +func TestStoreOperations(t *testing.T) { + // Skip if running in short mode + if testing.Short() { + t.Skip("skipping store test in short mode") + } + + // Create a temporary directory for the test + tmpDir := t.TempDir() + + // Create paths configuration pointing to temp dir + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + logger := logp.NewNopLogger() + + // Test store creation + store, _ := newTestStore(t, beatPaths, logger) + defer store.Close() + + // Test saving state + testKey := "test-key" + testState := &State{ + Version: StateVersion, + CursorType: CursorTypeInteger, + CursorValue: "12345", + UpdatedAt: time.Now().UTC(), + } + + require.NoError(t, store.Save(testKey, testState)) + + // Test loading state + loadedState, err := store.Load(testKey) + require.NoError(t, err) + require.NotNil(t, loadedState) + assert.Equal(t, testState.Version, loadedState.Version) + assert.Equal(t, testState.CursorType, loadedState.CursorType) + assert.Equal(t, testState.CursorValue, loadedState.CursorValue) + + // Test loading non-existent key + missingState, err := store.Load("non-existent-key") + require.NoError(t, err) + assert.Nil(t, missingState) + + // Test updating state + testState.CursorValue = "67890" + err = store.Save(testKey, testState) + require.NoError(t, err) + + loadedState, err = store.Load(testKey) + require.NoError(t, err) + require.NotNil(t, loadedState) + assert.Equal(t, "67890", loadedState.CursorValue) +} + +func TestNewStoreFromRegistry(t *testing.T) { + // Skip if running in short mode + if testing.Short() { + t.Skip("skipping store test in short mode") + } + + tmpDir := t.TempDir() + + // Create paths configuration pointing to temp dir + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + logger := logp.NewNopLogger() + dataPath := beatPaths.Resolve(paths.Data, "sql-cursor") + + // Create a shared memlog registry (simulating what ModuleBuilder does) + reg, err := memlog.New(logger.Named("memlog"), memlog.Settings{ + Root: dataPath, + FileMode: 0o600, + }) + require.NoError(t, err) + + registry := statestore.NewRegistry(reg) + defer registry.Close() + + // Create two stores from the same registry (simulating 2 MetricSets) + store1, err := NewStoreFromRegistry(registry, logger.Named("store1")) + require.NoError(t, err) + require.NotNil(t, store1) + + store2, err := NewStoreFromRegistry(registry, logger.Named("store2")) + require.NoError(t, err) + require.NotNil(t, store2) + + // Store1 writes a key + testState := &State{ + Version: StateVersion, + CursorType: CursorTypeInteger, + CursorValue: "100", + UpdatedAt: time.Now().UTC(), + } + err = store1.Save("key-from-store1", testState) + require.NoError(t, err) + + // Store2 can read the same key (shared backend) + loaded, err := store2.Load("key-from-store1") + require.NoError(t, err) + require.NotNil(t, loaded) + assert.Equal(t, "100", loaded.CursorValue) + + // Store2 writes a different key + testState2 := &State{ + Version: StateVersion, + CursorType: CursorTypeTimestamp, + CursorValue: "2026-01-01T00:00:00Z", + UpdatedAt: time.Now().UTC(), + } + err = store2.Save("key-from-store2", testState2) + require.NoError(t, err) + + // Store1 can read it + loaded2, err := store1.Load("key-from-store2") + require.NoError(t, err) + require.NotNil(t, loaded2) + assert.Equal(t, "2026-01-01T00:00:00Z", loaded2.CursorValue) + + // Close stores — should NOT close the shared registry + require.NoError(t, store1.Close()) + require.NoError(t, store2.Close()) + + // Registry is still usable (not closed by stores) + store3, err := NewStoreFromRegistry(registry, logger.Named("store3")) + require.NoError(t, err) + require.NotNil(t, store3) + + // Can still read previously written data + loaded3, err := store3.Load("key-from-store1") + require.NoError(t, err) + require.NotNil(t, loaded3) + assert.Equal(t, "100", loaded3.CursorValue) + + require.NoError(t, store3.Close()) +} + +func TestStoreClose(t *testing.T) { + // Skip if running in short mode + if testing.Short() { + t.Skip("skipping store test in short mode") + } + + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + logger := logp.NewNopLogger() + + store, _ := newTestStore(t, beatPaths, logger) + + // First close should succeed + require.NoError(t, store.Close()) + + // Second close should also succeed (idempotent) + require.NoError(t, store.Close()) +} + +func TestStoreOwnershipClosingBehavior(t *testing.T) { + // Skip if running in short mode + if testing.Short() { + t.Skip("skipping store test in short mode") + } + + t.Run("store close does not close shared registry", func(t *testing.T) { + tmpDir := t.TempDir() + beatPaths := &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } + + logger := logp.NewNopLogger() + dataPath := beatPaths.Resolve(paths.Data, "sql-cursor") + + reg, err := memlog.New(logger.Named("memlog"), memlog.Settings{ + Root: dataPath, + FileMode: 0o600, + }) + require.NoError(t, err) + + registry := statestore.NewRegistry(reg) + defer registry.Close() + + store, err := NewStoreFromRegistry(registry, logger) + require.NoError(t, err) + + testState := &State{ + Version: StateVersion, + CursorType: CursorTypeInteger, + CursorValue: "200", + UpdatedAt: time.Now().UTC(), + } + require.NoError(t, store.Save("test-key", testState)) + require.NoError(t, store.Close()) + + // Verify store operations fail after close + require.Error(t, store.Save("another-key", testState)) + + // Verify registry is still open — another store can be opened + store2, err := NewStoreFromRegistry(registry, logger) + require.NoError(t, err, "Registry should still be open after closing store") + + loaded, err := store2.Load("test-key") + require.NoError(t, err) + require.NotNil(t, loaded) + assert.Equal(t, "200", loaded.CursorValue) + + require.NoError(t, store2.Close()) + }) +} + +// newTestStore creates a memlog-backed store for cursor persistence in tests. +// It returns the Store and the Registry separately so the test can manage +// the registry lifecycle (the Store itself does not own the registry). +// +// Production code should use NewStoreFromRegistry with a shared Module-level +// registry to avoid multiple registries operating on the same files. +func newTestStore(t *testing.T, beatPaths *paths.Path, logger *logp.Logger) (*Store, *statestore.Registry) { + t.Helper() + + if beatPaths == nil { + beatPaths = paths.Paths + } + + dataPath := beatPaths.Resolve(paths.Data, "sql-cursor") + + reg, err := memlog.New(logger.Named("memlog"), memlog.Settings{ + Root: dataPath, + FileMode: 0o600, + }) + require.NoError(t, err, "failed to create memlog registry") + + registry := statestore.NewRegistry(reg) + t.Cleanup(func() { registry.Close() }) + + store, err := NewStoreFromRegistry(registry, logger) + require.NoError(t, err, "failed to open cursor store") + + return store, registry +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/types.go b/x-pack/metricbeat/module/sql/query/cursor/types.go new file mode 100644 index 000000000000..b010dc6f82a1 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/types.go @@ -0,0 +1,543 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "errors" + "fmt" + "math" + "strconv" + "strings" + "time" + + "github.com/shopspring/decimal" +) + +// Value represents a cursor value that can be serialized and passed to SQL drivers +type Value struct { + Type string `json:"type"` // "integer", "timestamp", "date", "float", or "decimal" + Raw string `json:"raw"` // String representation for persistence + Timestamp int64 `json:"timestamp,omitempty"` // Unix nanoseconds (timestamp only) +} + +// ParseValue creates a Value from a string representation. +// This is used to parse the default value from config and stored state. +func ParseValue(raw, valueType string) (*Value, error) { + v := &Value{Type: valueType, Raw: raw} + + switch valueType { + case CursorTypeInteger: + if _, err := strconv.ParseInt(raw, 10, 64); err != nil { + return nil, fmt.Errorf("invalid integer: %w", err) + } + + case CursorTypeTimestamp: + t, err := parseTimestampString(raw) + if err != nil { + return nil, fmt.Errorf("invalid timestamp: %w", err) + } + v.Timestamp = t.UnixNano() + v.Raw = t.Format(time.RFC3339Nano) // Normalize format + + case CursorTypeDate: + d, err := parseDateString(raw) + if err != nil { + return nil, fmt.Errorf("invalid date: %w", err) + } + v.Raw = d.Format("2006-01-02") // Normalize format + + case CursorTypeFloat: + f, err := strconv.ParseFloat(raw, 64) + if err != nil { + return nil, fmt.Errorf("invalid float: %w", err) + } + // Reject special IEEE 754 values that would break comparisons and query semantics. + if math.IsNaN(f) { + return nil, errors.New("value is NaN") + } + if math.IsInf(f, 0) { + return nil, fmt.Errorf("value is infinite: %f", f) + } + // Normalize: re-format to ensure consistent representation + v.Raw = strconv.FormatFloat(f, 'g', -1, 64) + + case CursorTypeDecimal: + d, err := decimal.NewFromString(raw) + if err != nil { + return nil, fmt.Errorf("invalid decimal: %w", err) + } + // Normalize: use decimal library's canonical string representation + v.Raw = d.String() + + default: + return nil, fmt.Errorf("unsupported cursor type: %s", valueType) + } + + return v, nil +} + +// InferTypeFromDefaultValue infers a cursor type from cursor.default when +// cursor.type is omitted from configuration. +func InferTypeFromDefaultValue(raw string) (string, error) { + return inferTypeFromString(raw, false) +} + +// InferTypeFromDatabaseValue infers a cursor type from a database row value. +func InferTypeFromDatabaseValue(dbVal interface{}) (string, error) { + if dbVal == nil { + return "", errors.New("column value is NULL") + } + + switch v := dbVal.(type) { + case int, int32, int64, uint, uint32, uint64: + return CursorTypeInteger, nil + case float32: + if math.IsNaN(float64(v)) { + return "", errors.New("value is NaN") + } + if math.IsInf(float64(v), 0) { + return "", fmt.Errorf("value is infinite: %f", v) + } + return CursorTypeFloat, nil + case float64: + if math.IsNaN(v) { + return "", errors.New("value is NaN") + } + if math.IsInf(v, 0) { + return "", fmt.Errorf("value is infinite: %f", v) + } + return CursorTypeFloat, nil + case time.Time: + return CursorTypeTimestamp, nil + case *time.Time: + if v == nil { + return "", errors.New("column value is NULL") + } + return CursorTypeTimestamp, nil + case []byte: + return inferTypeFromString(string(v), true) + case string: + return inferTypeFromString(v, true) + default: + return "", fmt.Errorf("cannot infer cursor type from value type %T", dbVal) + } +} + +func inferTypeFromString(raw string, preferDecimal bool) (string, error) { + s := strings.TrimSpace(raw) + if s == "" { + return "", errors.New("value is empty") + } + + if isDateOnlyString(s) { + return CursorTypeDate, nil + } + + if _, err := parseTimestampString(s); err == nil { + return CursorTypeTimestamp, nil + } + + if _, err := strconv.ParseInt(s, 10, 64); err == nil { + return CursorTypeInteger, nil + } + + if preferDecimal && strings.ContainsAny(s, ".eE") { + if _, err := decimal.NewFromString(s); err == nil { + return CursorTypeDecimal, nil + } + } + + if f, err := strconv.ParseFloat(s, 64); err == nil { + if math.IsNaN(f) { + return "", errors.New("value is NaN") + } + if math.IsInf(f, 0) { + return "", fmt.Errorf("value is infinite: %f", f) + } + return CursorTypeFloat, nil + } + + if preferDecimal { + if _, err := decimal.NewFromString(s); err == nil { + return CursorTypeDecimal, nil + } + } + + return "", fmt.Errorf("cannot infer cursor type from value %q", raw) +} + +func isDateOnlyString(s string) bool { + if len(s) != len("2006-01-02") { + return false + } + _, err := time.Parse("2006-01-02", s) + return err == nil +} + +// FromDatabaseValue creates a Value from a database result. +// This handles the various types that SQL drivers may return. +func FromDatabaseValue(dbVal interface{}, valueType string) (*Value, error) { + if dbVal == nil { + return nil, errors.New("column value is NULL") + } + + switch valueType { + case CursorTypeInteger: + return parseIntegerFromDB(dbVal) + case CursorTypeTimestamp: + return parseTimestampFromDB(dbVal) + case CursorTypeDate: + return parseDateFromDB(dbVal) + case CursorTypeFloat: + return parseFloatFromDB(dbVal) + case CursorTypeDecimal: + return parseDecimalFromDB(dbVal) + default: + return nil, fmt.Errorf("unsupported cursor type: %s", valueType) + } +} + +// ToDriverArg converts the Value to a type suitable for database/sql Query. +// The returned value can be passed directly to db.QueryContext(). +func (v *Value) ToDriverArg() interface{} { + switch v.Type { + case CursorTypeInteger: + i, _ := strconv.ParseInt(v.Raw, 10, 64) + return i + case CursorTypeTimestamp: + return time.Unix(0, v.Timestamp).UTC() + case CursorTypeDate: + return v.Raw + case CursorTypeFloat: + f, _ := strconv.ParseFloat(v.Raw, 64) + return f + case CursorTypeDecimal: + // Most SQL drivers accept string for DECIMAL/NUMERIC bind parameters. + // If a driver doesn't, the user can cast in SQL: CAST(:cursor AS DECIMAL(10,2)) + return v.Raw + default: + return v.Raw + } +} + +// Compare returns -1 if v < other, 0 if equal, 1 if v > other. +// Returns an error if the types don't match. +func (v *Value) Compare(other *Value) (int, error) { + if v.Type != other.Type { + return 0, fmt.Errorf("cannot compare %s with %s", v.Type, other.Type) + } + + switch v.Type { + case CursorTypeInteger: + a, _ := strconv.ParseInt(v.Raw, 10, 64) + b, _ := strconv.ParseInt(other.Raw, 10, 64) + return compareInt64(a, b), nil + + case CursorTypeTimestamp: + return compareInt64(v.Timestamp, other.Timestamp), nil + + case CursorTypeDate: + if v.Raw < other.Raw { + return -1, nil + } else if v.Raw > other.Raw { + return 1, nil + } + return 0, nil + + case CursorTypeFloat: + a, _ := strconv.ParseFloat(v.Raw, 64) + b, _ := strconv.ParseFloat(other.Raw, 64) + return compareFloat64(a, b), nil + + case CursorTypeDecimal: + a, errA := decimal.NewFromString(v.Raw) + b, errB := decimal.NewFromString(other.Raw) + if errA != nil || errB != nil { + return 0, fmt.Errorf("failed to parse decimal for comparison: a=%q b=%q", v.Raw, other.Raw) + } + return a.Cmp(b), nil + } + + return 0, fmt.Errorf("unsupported type: %s", v.Type) +} + +// String returns the string representation of the cursor value. +func (v *Value) String() string { + return v.Raw +} + +func compareInt64(a, b int64) int { + if a < b { + return -1 + } else if a > b { + return 1 + } + return 0 +} + +func compareFloat64(a, b float64) int { + if a < b { + return -1 + } else if a > b { + return 1 + } + return 0 +} + +// --- Integer parsing --- + +func parseIntegerFromDB(dbVal interface{}) (*Value, error) { + var intVal int64 + + switch v := dbVal.(type) { + case int: + intVal = int64(v) + case int32: + intVal = int64(v) + case int64: + intVal = v + case uint: + if uint64(v) > math.MaxInt64 { + return nil, fmt.Errorf("uint overflow: %d exceeds max int64", v) + } + intVal = int64(uint64(v)) //nolint:gosec // overflow checked above + case uint32: + intVal = int64(v) + case uint64: + if v > math.MaxInt64 { + return nil, fmt.Errorf("uint64 overflow: %d exceeds max int64", v) + } + intVal = int64(v) + case float32: + // Some drivers may return float32 for certain numeric types + if v > float32(math.MaxInt64) || v < float32(math.MinInt64) { + return nil, fmt.Errorf("float32 overflow: %f exceeds int64 range", v) + } + if float32(int64(v)) != v { + return nil, fmt.Errorf("float32 value has fractional part: %f", v) + } + intVal = int64(v) + case float64: + if v > float64(math.MaxInt64) || v < float64(math.MinInt64) { + return nil, fmt.Errorf("float64 overflow: %f exceeds int64 range", v) + } + if float64(int64(v)) != v { + return nil, fmt.Errorf("float64 value has fractional part: %f", v) + } + intVal = int64(v) + case []byte: + parsed, err := strconv.ParseInt(string(v), 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse []byte as integer: %w", err) + } + intVal = parsed + case string: + parsed, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse string as integer: %w", err) + } + intVal = parsed + default: + return nil, fmt.Errorf("unsupported integer type: %T", dbVal) + } + + return &Value{ + Type: CursorTypeInteger, + Raw: strconv.FormatInt(intVal, 10), + }, nil +} + +// --- Float parsing --- + +func parseFloatFromDB(dbVal interface{}) (*Value, error) { + var floatVal float64 + + switch v := dbVal.(type) { + case float32: + floatVal = float64(v) + case float64: + floatVal = v + case int: + floatVal = float64(v) + case int32: + floatVal = float64(v) + case int64: + floatVal = float64(v) + case []byte: + parsed, err := strconv.ParseFloat(string(v), 64) + if err != nil { + return nil, fmt.Errorf("cannot parse []byte as float: %w", err) + } + floatVal = parsed + case string: + parsed, err := strconv.ParseFloat(v, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse string as float: %w", err) + } + floatVal = parsed + default: + return nil, fmt.Errorf("unsupported float type: %T", dbVal) + } + + if math.IsNaN(floatVal) { + return nil, errors.New("value is NaN") + } + if math.IsInf(floatVal, 0) { + return nil, fmt.Errorf("value is infinite: %f", floatVal) + } + + return &Value{ + Type: CursorTypeFloat, + Raw: strconv.FormatFloat(floatVal, 'g', -1, 64), + }, nil +} + +// --- Decimal parsing --- + +func parseDecimalFromDB(dbVal interface{}) (*Value, error) { + var d decimal.Decimal + + switch v := dbVal.(type) { + case float32: + // Convert via string to avoid float64 precision loss: + // decimal.NewFromFloat32 preserves the float32 representation. + d = decimal.NewFromFloat32(v) + case float64: + // Use string conversion for better precision preservation: + // float64 -> string -> decimal avoids double rounding. + d = decimal.NewFromFloat(v) + case []byte: + parsed, err := decimal.NewFromString(string(v)) + if err != nil { + return nil, fmt.Errorf("cannot parse []byte as decimal: %w", err) + } + d = parsed + case string: + parsed, err := decimal.NewFromString(v) + if err != nil { + return nil, fmt.Errorf("cannot parse string as decimal: %w", err) + } + d = parsed + case int: + d = decimal.NewFromInt(int64(v)) + case int32: + d = decimal.NewFromInt32(v) + case int64: + d = decimal.NewFromInt(v) + default: + return nil, fmt.Errorf("unsupported decimal type: %T", dbVal) + } + + return &Value{ + Type: CursorTypeDecimal, + Raw: d.String(), + }, nil +} + +// --- Timestamp parsing --- + +// timestampFormats lists the supported timestamp formats in order of preference. +// RFC3339Nano is first as it's the canonical format we use for storage. +var timestampFormats = []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999", + "2006-01-02 15:04:05.999999", + "2006-01-02 15:04:05.999", + "2006-01-02 15:04:05", + "2006-01-02T15:04:05", + "2006-01-02", +} + +func parseTimestampString(s string) (time.Time, error) { + for _, format := range timestampFormats { + if t, err := time.Parse(format, s); err == nil { + return t.UTC(), nil + } + } + return time.Time{}, fmt.Errorf("cannot parse timestamp: %s", s) +} + +func parseTimestampFromDB(dbVal interface{}) (*Value, error) { + var t time.Time + + switch v := dbVal.(type) { + case time.Time: + t = v.UTC() + case *time.Time: + if v == nil { + return nil, errors.New("column value is NULL") + } + t = v.UTC() + case []byte: + // MySQL returns timestamps as []byte + parsed, err := parseTimestampString(string(v)) + if err != nil { + return nil, fmt.Errorf("cannot parse []byte as timestamp: %w", err) + } + t = parsed + case string: + parsed, err := parseTimestampString(v) + if err != nil { + return nil, fmt.Errorf("cannot parse string as timestamp: %w", err) + } + t = parsed + default: + return nil, fmt.Errorf("unsupported timestamp type: %T", dbVal) + } + + return &Value{ + Type: CursorTypeTimestamp, + Raw: t.Format(time.RFC3339Nano), + Timestamp: t.UnixNano(), + }, nil +} + +// --- Date parsing --- + +func parseDateString(s string) (time.Time, error) { + // Try the canonical date format first + if t, err := time.Parse("2006-01-02", s); err == nil { + return t.UTC(), nil + } + // Fall back to timestamp parsing (extracts date portion) + if t, err := parseTimestampString(s); err == nil { + return t.UTC(), nil + } + return time.Time{}, fmt.Errorf("cannot parse date: %s", s) +} + +func parseDateFromDB(dbVal interface{}) (*Value, error) { + var d time.Time + + switch v := dbVal.(type) { + case time.Time: + d = v.UTC() + case *time.Time: + if v == nil { + return nil, errors.New("column value is NULL") + } + d = v.UTC() + case []byte: + parsed, err := parseDateString(string(v)) + if err != nil { + return nil, fmt.Errorf("cannot parse []byte as date: %w", err) + } + d = parsed + case string: + parsed, err := parseDateString(v) + if err != nil { + return nil, fmt.Errorf("cannot parse string as date: %w", err) + } + d = parsed + default: + return nil, fmt.Errorf("unsupported date type: %T", dbVal) + } + + return &Value{ + Type: CursorTypeDate, + Raw: d.Format("2006-01-02"), + }, nil +} diff --git a/x-pack/metricbeat/module/sql/query/cursor/types_test.go b/x-pack/metricbeat/module/sql/query/cursor/types_test.go new file mode 100644 index 000000000000..441c7f9b0fb7 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor/types_test.go @@ -0,0 +1,1152 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cursor + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseValue(t *testing.T) { + tests := []struct { + name string + raw string + valueType string + wantErr bool + wantRaw string + wantTsNano int64 + }{ + // Integer tests + { + name: "integer - zero", + raw: "0", + valueType: CursorTypeInteger, + wantErr: false, + wantRaw: "0", + }, + { + name: "integer - positive", + raw: "12345", + valueType: CursorTypeInteger, + wantErr: false, + wantRaw: "12345", + }, + { + name: "integer - negative", + raw: "-12345", + valueType: CursorTypeInteger, + wantErr: false, + wantRaw: "-12345", + }, + { + name: "integer - max int64", + raw: "9223372036854775807", + valueType: CursorTypeInteger, + wantErr: false, + wantRaw: "9223372036854775807", + }, + { + name: "integer - invalid", + raw: "not-a-number", + valueType: CursorTypeInteger, + wantErr: true, + }, + { + name: "integer - float string", + raw: "123.45", + valueType: CursorTypeInteger, + wantErr: true, + }, + + // Timestamp tests + { + name: "timestamp - RFC3339", + raw: "2024-01-15T10:30:00Z", + valueType: CursorTypeTimestamp, + wantErr: false, + wantRaw: "2024-01-15T10:30:00Z", + wantTsNano: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + { + name: "timestamp - RFC3339Nano", + raw: "2024-01-15T10:30:00.123456789Z", + valueType: CursorTypeTimestamp, + wantErr: false, + wantRaw: "2024-01-15T10:30:00.123456789Z", + wantTsNano: time.Date(2024, 1, 15, 10, 30, 0, 123456789, time.UTC).UnixNano(), + }, + { + name: "timestamp - space format", + raw: "2024-01-15 10:30:00", + valueType: CursorTypeTimestamp, + wantErr: false, + wantRaw: "2024-01-15T10:30:00Z", + wantTsNano: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + { + name: "timestamp - invalid", + raw: "not-a-timestamp", + valueType: CursorTypeTimestamp, + wantErr: true, + }, + + // Date tests + { + name: "date - valid", + raw: "2024-01-15", + valueType: CursorTypeDate, + wantErr: false, + wantRaw: "2024-01-15", + }, + { + name: "date - from timestamp", + raw: "2024-01-15T10:30:00Z", + valueType: CursorTypeDate, + wantErr: false, + wantRaw: "2024-01-15", + }, + { + name: "date - invalid", + raw: "not-a-date", + valueType: CursorTypeDate, + wantErr: true, + }, + + // Unsupported type + { + name: "unsupported type", + raw: "value", + valueType: "unsupported", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := ParseValue(tt.raw, tt.valueType) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, tt.valueType, val.Type) + if tt.valueType == CursorTypeTimestamp { + assert.Equal(t, tt.wantTsNano, val.Timestamp) + } + }) + } +} + +func TestFromDatabaseValue_Integer(t *testing.T) { + tests := []struct { + name string + dbVal interface{} + wantRaw string + wantErr bool + }{ + { + name: "int", + dbVal: int(123), + wantRaw: "123", + }, + { + name: "int32", + dbVal: int32(123), + wantRaw: "123", + }, + { + name: "int64", + dbVal: int64(123), + wantRaw: "123", + }, + { + name: "uint", + dbVal: uint(123), + wantRaw: "123", + }, + { + name: "uint32", + dbVal: uint32(123), + wantRaw: "123", + }, + { + name: "uint64", + dbVal: uint64(123), + wantRaw: "123", + }, + { + name: "uint64 overflow", + dbVal: uint64(math.MaxUint64), + wantErr: true, + }, + { + name: "uint overflow (on 64-bit systems)", + dbVal: uint(math.MaxUint64), + wantErr: true, + }, + { + name: "float32", + dbVal: float32(123), + wantRaw: "123", + }, + { + name: "float32 overflow", + dbVal: float32(math.MaxFloat32), + wantErr: true, + }, + { + name: "float64", + dbVal: float64(123), + wantRaw: "123", + }, + { + name: "float64 overflow", + dbVal: float64(math.MaxFloat64), + wantErr: true, + }, + { + name: "[]byte", + dbVal: []byte("123"), + wantRaw: "123", + }, + { + name: "[]byte invalid", + dbVal: []byte("not-a-number"), + wantErr: true, + }, + { + name: "string", + dbVal: "123", + wantRaw: "123", + }, + { + name: "string invalid", + dbVal: "not-a-number", + wantErr: true, + }, + { + name: "nil", + dbVal: nil, + wantErr: true, + }, + { + name: "unsupported type", + dbVal: struct{}{}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := FromDatabaseValue(tt.dbVal, CursorTypeInteger) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, CursorTypeInteger, val.Type) + }) + } +} + +func TestFromDatabaseValue_Timestamp(t *testing.T) { + baseTime := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) + + tests := []struct { + name string + dbVal interface{} + wantRaw string + wantTsNano int64 + wantErr bool + }{ + { + name: "time.Time", + dbVal: baseTime, + wantRaw: "2024-01-15T10:30:00Z", + wantTsNano: baseTime.UnixNano(), + }, + { + name: "*time.Time", + dbVal: &baseTime, + wantRaw: "2024-01-15T10:30:00Z", + wantTsNano: baseTime.UnixNano(), + }, + { + name: "*time.Time nil", + dbVal: (*time.Time)(nil), + wantErr: true, + }, + { + name: "[]byte", + dbVal: []byte("2024-01-15 10:30:00"), + wantRaw: "2024-01-15T10:30:00Z", + wantTsNano: baseTime.UnixNano(), + }, + { + name: "[]byte invalid", + dbVal: []byte("invalid"), + wantErr: true, + }, + { + name: "string", + dbVal: "2024-01-15 10:30:00", + wantRaw: "2024-01-15T10:30:00Z", + wantTsNano: baseTime.UnixNano(), + }, + { + name: "string invalid", + dbVal: "invalid", + wantErr: true, + }, + { + name: "nil", + dbVal: nil, + wantErr: true, + }, + { + name: "unsupported type", + dbVal: 123, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := FromDatabaseValue(tt.dbVal, CursorTypeTimestamp) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, tt.wantTsNano, val.Timestamp) + assert.Equal(t, CursorTypeTimestamp, val.Type) + }) + } +} + +func TestFromDatabaseValue_Date(t *testing.T) { + baseTime := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) + + tests := []struct { + name string + dbVal interface{} + wantRaw string + wantErr bool + }{ + { + name: "time.Time", + dbVal: baseTime, + wantRaw: "2024-01-15", + }, + { + name: "*time.Time", + dbVal: &baseTime, + wantRaw: "2024-01-15", + }, + { + name: "*time.Time nil", + dbVal: (*time.Time)(nil), + wantErr: true, + }, + { + name: "[]byte", + dbVal: []byte("2024-01-15"), + wantRaw: "2024-01-15", + }, + { + name: "[]byte invalid", + dbVal: []byte("invalid"), + wantErr: true, + }, + { + name: "string", + dbVal: "2024-01-15", + wantRaw: "2024-01-15", + }, + { + name: "string invalid", + dbVal: "invalid", + wantErr: true, + }, + { + name: "nil", + dbVal: nil, + wantErr: true, + }, + { + name: "unsupported type", + dbVal: 123, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := FromDatabaseValue(tt.dbVal, CursorTypeDate) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, CursorTypeDate, val.Type) + }) + } +} + +func TestValueToDriverArg(t *testing.T) { + tests := []struct { + name string + value *Value + wantType string + wantValue interface{} + }{ + { + name: "integer", + value: &Value{ + Type: CursorTypeInteger, + Raw: "12345", + }, + wantType: "int64", + wantValue: int64(12345), + }, + { + name: "timestamp", + value: &Value{ + Type: CursorTypeTimestamp, + Raw: "2024-01-15T10:30:00Z", + Timestamp: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + wantType: "time.Time", + wantValue: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC), + }, + { + name: "date", + value: &Value{ + Type: CursorTypeDate, + Raw: "2024-01-15", + }, + wantType: "string", + wantValue: "2024-01-15", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.value.ToDriverArg() + assert.IsType(t, tt.wantValue, result) + assert.Equal(t, tt.wantValue, result) + }) + } +} + +func TestValueCompare(t *testing.T) { + tests := []struct { + name string + v1 *Value + v2 *Value + want int + wantErr bool + }{ + // Integer comparisons + { + name: "integer equal", + v1: &Value{Type: CursorTypeInteger, Raw: "100"}, + v2: &Value{Type: CursorTypeInteger, Raw: "100"}, + want: 0, + wantErr: false, + }, + { + name: "integer less than", + v1: &Value{Type: CursorTypeInteger, Raw: "100"}, + v2: &Value{Type: CursorTypeInteger, Raw: "200"}, + want: -1, + wantErr: false, + }, + { + name: "integer greater than", + v1: &Value{Type: CursorTypeInteger, Raw: "200"}, + v2: &Value{Type: CursorTypeInteger, Raw: "100"}, + want: 1, + wantErr: false, + }, + + // Timestamp comparisons + { + name: "timestamp equal", + v1: &Value{ + Type: CursorTypeTimestamp, + Raw: "2024-01-15T10:30:00Z", + Timestamp: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + v2: &Value{ + Type: CursorTypeTimestamp, + Raw: "2024-01-15T10:30:00Z", + Timestamp: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + want: 0, + wantErr: false, + }, + { + name: "timestamp less than", + v1: &Value{ + Type: CursorTypeTimestamp, + Raw: "2024-01-15T10:30:00Z", + Timestamp: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + v2: &Value{ + Type: CursorTypeTimestamp, + Raw: "2024-01-15T11:30:00Z", + Timestamp: time.Date(2024, 1, 15, 11, 30, 0, 0, time.UTC).UnixNano(), + }, + want: -1, + wantErr: false, + }, + { + name: "timestamp greater than", + v1: &Value{ + Type: CursorTypeTimestamp, + Raw: "2024-01-15T11:30:00Z", + Timestamp: time.Date(2024, 1, 15, 11, 30, 0, 0, time.UTC).UnixNano(), + }, + v2: &Value{ + Type: CursorTypeTimestamp, + Raw: "2024-01-15T10:30:00Z", + Timestamp: time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + want: 1, + wantErr: false, + }, + + // Date comparisons + { + name: "date equal", + v1: &Value{Type: CursorTypeDate, Raw: "2024-01-15"}, + v2: &Value{Type: CursorTypeDate, Raw: "2024-01-15"}, + want: 0, + wantErr: false, + }, + { + name: "date less than", + v1: &Value{Type: CursorTypeDate, Raw: "2024-01-15"}, + v2: &Value{Type: CursorTypeDate, Raw: "2024-01-16"}, + want: -1, + wantErr: false, + }, + { + name: "date greater than", + v1: &Value{Type: CursorTypeDate, Raw: "2024-01-16"}, + v2: &Value{Type: CursorTypeDate, Raw: "2024-01-15"}, + want: 1, + wantErr: false, + }, + + // Type mismatch + { + name: "type mismatch", + v1: &Value{Type: CursorTypeInteger, Raw: "100"}, + v2: &Value{Type: CursorTypeDate, Raw: "2024-01-15"}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := tt.v1.Compare(tt.v2) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, result) + }) + } +} + +func TestValueString(t *testing.T) { + val := &Value{Type: CursorTypeInteger, Raw: "12345"} + assert.Equal(t, "12345", val.String()) +} + +func TestFromDatabaseValue_UnsupportedType(t *testing.T) { + _, err := FromDatabaseValue("some_value", "unsupported") + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported cursor type") +} + +func TestFromDatabaseValue_NilValue(t *testing.T) { + _, err := FromDatabaseValue(nil, CursorTypeInteger) + require.Error(t, err) + assert.Contains(t, err.Error(), "NULL") +} + +func TestValueToDriverArg_UnsupportedType(t *testing.T) { + val := &Value{Type: "unknown", Raw: "hello"} + result := val.ToDriverArg() + assert.Equal(t, "hello", result) +} + +func TestValueCompare_UnsupportedType(t *testing.T) { + v1 := &Value{Type: "unknown", Raw: "a"} + v2 := &Value{Type: "unknown", Raw: "b"} + _, err := v1.Compare(v2) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported type") +} + +func TestFromDatabaseValue_Integer_NegativeValues(t *testing.T) { + // Test negative int values + val, err := FromDatabaseValue(int(-42), CursorTypeInteger) + require.NoError(t, err) + assert.Equal(t, "-42", val.Raw) + + // Test negative int64 + val, err = FromDatabaseValue(int64(-9999), CursorTypeInteger) + require.NoError(t, err) + assert.Equal(t, "-9999", val.Raw) + + // Test negative float64 + val, err = FromDatabaseValue(float64(-100), CursorTypeInteger) + require.NoError(t, err) + assert.Equal(t, "-100", val.Raw) + + // Test negative string + val, err = FromDatabaseValue("-777", CursorTypeInteger) + require.NoError(t, err) + assert.Equal(t, "-777", val.Raw) +} + +func TestFromDatabaseValue_Timestamp_WithTimezone(t *testing.T) { + // Timestamp with timezone offset should be converted to UTC + loc := time.FixedZone("EST", -5*60*60) + eastern := time.Date(2024, 6, 15, 10, 0, 0, 0, loc) + + val, err := FromDatabaseValue(eastern, CursorTypeTimestamp) + require.NoError(t, err) + // Should be converted to UTC (15:00) + assert.Equal(t, "2024-06-15T15:00:00Z", val.Raw) +} + +func TestFromDatabaseValue_Date_FromTimestamp(t *testing.T) { + // Date from a time.Time with time portion should extract just the date + ts := time.Date(2024, 6, 15, 23, 59, 59, 0, time.UTC) + val, err := FromDatabaseValue(ts, CursorTypeDate) + require.NoError(t, err) + assert.Equal(t, "2024-06-15", val.Raw) +} + +func TestFromDatabaseValue_Date_StringTimestamp(t *testing.T) { + // Date from a string that contains a full timestamp + val, err := FromDatabaseValue("2024-06-15T10:30:00Z", CursorTypeDate) + require.NoError(t, err) + assert.Equal(t, "2024-06-15", val.Raw) +} + +// ============================================================================ +// Float type tests +// ============================================================================ + +func TestParseValue_Float(t *testing.T) { + tests := []struct { + name string + raw string + wantRaw string + wantErr bool + }{ + {"zero", "0.0", "0", false}, + {"positive", "3.14159", "3.14159", false}, + {"negative", "-2.718", "-2.718", false}, + {"integer-like", "42", "42", false}, + {"scientific", "1.5e10", "1.5e+10", false}, + {"very small", "0.000001", "1e-06", false}, + {"nan", "NaN", "", true}, + {"pos inf", "+Inf", "", true}, + {"neg inf", "-Inf", "", true}, + {"invalid", "not-a-float", "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := ParseValue(tt.raw, CursorTypeFloat) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, CursorTypeFloat, val.Type) + }) + } +} + +func TestFromDatabaseValue_Float(t *testing.T) { + tests := []struct { + name string + dbVal interface{} + wantRaw string + wantErr bool + }{ + {"float32", float32(3.14), "3.140000104904175", false}, + {"float64", float64(3.14159), "3.14159", false}, + {"int", int(42), "42", false}, + {"int32", int32(42), "42", false}, + {"int64", int64(42), "42", false}, + {"[]byte", []byte("99.95"), "99.95", false}, + {"string", "99.95", "99.95", false}, + {"[]byte invalid", []byte("abc"), "", true}, + {"string invalid", "abc", "", true}, + {"nil", nil, "", true}, + {"unsupported", struct{}{}, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := FromDatabaseValue(tt.dbVal, CursorTypeFloat) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, CursorTypeFloat, val.Type) + }) + } +} + +func TestFromDatabaseValue_Float_NaN(t *testing.T) { + _, err := FromDatabaseValue(math.NaN(), CursorTypeFloat) + require.Error(t, err) + assert.Contains(t, err.Error(), "NaN") +} + +func TestFromDatabaseValue_Float_Inf(t *testing.T) { + _, err := FromDatabaseValue(math.Inf(1), CursorTypeFloat) + require.Error(t, err) + assert.Contains(t, err.Error(), "infinite") +} + +func TestValueCompare_Float(t *testing.T) { + tests := []struct { + name string + a, b string + want int + }{ + {"equal", "3.14", "3.14", 0}, + {"less", "2.0", "3.0", -1}, + {"greater", "3.0", "2.0", 1}, + {"negative", "-1.5", "1.5", -1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v1 := &Value{Type: CursorTypeFloat, Raw: tt.a} + v2 := &Value{Type: CursorTypeFloat, Raw: tt.b} + cmp, err := v1.Compare(v2) + require.NoError(t, err) + assert.Equal(t, tt.want, cmp) + }) + } +} + +func TestValueToDriverArg_Float(t *testing.T) { + val := &Value{Type: CursorTypeFloat, Raw: "3.14"} + arg := val.ToDriverArg() + f, ok := arg.(float64) + require.True(t, ok) + assert.InDelta(t, 3.14, f, 0.001) +} + +// ============================================================================ +// Decimal type tests +// ============================================================================ + +func TestParseValue_Decimal(t *testing.T) { + tests := []struct { + name string + raw string + wantRaw string + wantErr bool + }{ + {"zero", "0", "0", false}, + {"positive", "99.95", "99.95", false}, + {"negative", "-42.50", "-42.5", false}, + {"high precision", "123456789.123456789", "123456789.123456789", false}, + {"integer-like", "100", "100", false}, + {"invalid", "not-a-decimal", "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := ParseValue(tt.raw, CursorTypeDecimal) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, CursorTypeDecimal, val.Type) + }) + } +} + +func TestFromDatabaseValue_Decimal(t *testing.T) { + tests := []struct { + name string + dbVal interface{} + wantRaw string + wantErr bool + }{ + {"float64", float64(99.95), "99.95", false}, + {"float32", float32(3.14), "3.14", false}, + {"int", int(42), "42", false}, + {"int32", int32(42), "42", false}, + {"int64", int64(42), "42", false}, + {"[]byte", []byte("123.456"), "123.456", false}, + {"string", "123.456", "123.456", false}, + {"high precision string", "999999999999.999999999", "999999999999.999999999", false}, + {"[]byte invalid", []byte("abc"), "", true}, + {"string invalid", "abc", "", true}, + {"nil", nil, "", true}, + {"unsupported", struct{}{}, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := FromDatabaseValue(tt.dbVal, CursorTypeDecimal) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, CursorTypeDecimal, val.Type) + }) + } +} + +func TestValueCompare_Decimal(t *testing.T) { + tests := []struct { + name string + a, b string + want int + }{ + {"equal", "99.95", "99.95", 0}, + {"less", "99.94", "99.95", -1}, + {"greater", "99.96", "99.95", 1}, + {"negative", "-1.5", "1.5", -1}, + {"high precision", "0.123456789", "0.123456788", 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v1 := &Value{Type: CursorTypeDecimal, Raw: tt.a} + v2 := &Value{Type: CursorTypeDecimal, Raw: tt.b} + cmp, err := v1.Compare(v2) + require.NoError(t, err) + assert.Equal(t, tt.want, cmp) + }) + } +} + +func TestValueToDriverArg_Decimal(t *testing.T) { + val := &Value{Type: CursorTypeDecimal, Raw: "99.95"} + arg := val.ToDriverArg() + s, ok := arg.(string) + require.True(t, ok) + assert.Equal(t, "99.95", s) +} + +func TestDecimalPrecisionRoundTrip(t *testing.T) { + // Critical test: verify that decimal preserves exact values through + // parse -> store -> load -> compare cycle + original := "123456789.123456789012345" + val, err := ParseValue(original, CursorTypeDecimal) + require.NoError(t, err) + + // Re-parse from stored Raw + val2, err := ParseValue(val.Raw, CursorTypeDecimal) + require.NoError(t, err) + + cmp, err := val.Compare(val2) + require.NoError(t, err) + assert.Equal(t, 0, cmp, "decimal round-trip must preserve exact value") +} + +// ============================================================================ +// Timestamp coverage hardening +// ============================================================================ + +func TestTimestampNanosecondPrecisionRoundTrip(t *testing.T) { + // Critical test: verify that nanosecond precision is preserved through + // parse -> Raw -> re-parse -> compare -> ToDriverArg cycle. + original := "2024-06-15T10:30:00.123456789Z" + val, err := ParseValue(original, CursorTypeTimestamp) + require.NoError(t, err) + assert.Equal(t, original, val.Raw, "Raw must preserve original nanosecond string") + + // Re-parse from stored Raw (simulates store->load) + val2, err := ParseValue(val.Raw, CursorTypeTimestamp) + require.NoError(t, err) + assert.Equal(t, val.Timestamp, val2.Timestamp, "Timestamp (UnixNano) must survive round-trip") + + // Compare must report equal + cmp, err := val.Compare(val2) + require.NoError(t, err) + assert.Equal(t, 0, cmp, "round-tripped timestamp must compare equal") + + // ToDriverArg must produce a time.Time with the exact nanosecond + arg := val2.ToDriverArg() + tm, ok := arg.(time.Time) + require.True(t, ok) + assert.Equal(t, 123456789, tm.Nanosecond(), "ToDriverArg must preserve nanosecond component") +} + +func TestTimestampCompare_NanosecondBoundaries(t *testing.T) { + // Two timestamps differing only in the nanosecond component must compare correctly. + ts1 := time.Date(2024, 6, 15, 10, 30, 0, 100, time.UTC) // 100 ns + ts2 := time.Date(2024, 6, 15, 10, 30, 0, 200, time.UTC) // 200 ns + + v1, err := FromDatabaseValue(ts1, CursorTypeTimestamp) + require.NoError(t, err) + v2, err := FromDatabaseValue(ts2, CursorTypeTimestamp) + require.NoError(t, err) + + cmp, err := v1.Compare(v2) + require.NoError(t, err) + assert.Equal(t, -1, cmp, "100ns should be less than 200ns") + + cmp, err = v2.Compare(v1) + require.NoError(t, err) + assert.Equal(t, 1, cmp, "200ns should be greater than 100ns") + + // Same nanosecond + v3, err := FromDatabaseValue(ts1, CursorTypeTimestamp) + require.NoError(t, err) + cmp, err = v1.Compare(v3) + require.NoError(t, err) + assert.Equal(t, 0, cmp, "same nanosecond should compare equal") +} + +func TestParseTimestamp_AllSupportedFormats(t *testing.T) { + // Verify every format in timestampFormats is correctly parsed. + // The canonical output is always RFC3339Nano in UTC. + tests := []struct { + name string + input string + wantRaw string + wantTsNano int64 + }{ + { + name: "RFC3339Nano", + input: "2024-06-15T10:30:00.123456789Z", + wantRaw: "2024-06-15T10:30:00.123456789Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 123456789, time.UTC).UnixNano(), + }, + { + name: "RFC3339", + input: "2024-06-15T10:30:00Z", + wantRaw: "2024-06-15T10:30:00Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + { + name: "space 9-digit nanoseconds", + input: "2024-06-15 10:30:00.123456789", + wantRaw: "2024-06-15T10:30:00.123456789Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 123456789, time.UTC).UnixNano(), + }, + { + name: "space 6-digit microseconds", + input: "2024-06-15 10:30:00.123456", + wantRaw: "2024-06-15T10:30:00.123456Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 123456000, time.UTC).UnixNano(), + }, + { + name: "space 3-digit milliseconds", + input: "2024-06-15 10:30:00.123", + wantRaw: "2024-06-15T10:30:00.123Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 123000000, time.UTC).UnixNano(), + }, + { + name: "space seconds only", + input: "2024-06-15 10:30:00", + wantRaw: "2024-06-15T10:30:00Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + { + name: "T separator no timezone", + input: "2024-06-15T10:30:00", + wantRaw: "2024-06-15T10:30:00Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + { + name: "date only", + input: "2024-06-15", + wantRaw: "2024-06-15T00:00:00Z", + wantTsNano: time.Date(2024, 6, 15, 0, 0, 0, 0, time.UTC).UnixNano(), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := ParseValue(tt.input, CursorTypeTimestamp) + require.NoError(t, err, "format %q should be parseable", tt.input) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, tt.wantTsNano, val.Timestamp, "UnixNano mismatch for format %q", tt.input) + }) + } +} + +func TestFromDatabaseValue_Timestamp_SubsecondPrecision(t *testing.T) { + // Database drivers may return time.Time with various precision levels. + // Verify all are correctly captured. + tests := []struct { + name string + dbVal time.Time + wantNs int + wantRaw string + }{ + { + name: "milliseconds", + dbVal: time.Date(2024, 6, 15, 10, 30, 0, 123000000, time.UTC), + wantNs: 123000000, + wantRaw: "2024-06-15T10:30:00.123Z", + }, + { + name: "microseconds", + dbVal: time.Date(2024, 6, 15, 10, 30, 0, 123456000, time.UTC), + wantNs: 123456000, + wantRaw: "2024-06-15T10:30:00.123456Z", + }, + { + name: "nanoseconds", + dbVal: time.Date(2024, 6, 15, 10, 30, 0, 123456789, time.UTC), + wantNs: 123456789, + wantRaw: "2024-06-15T10:30:00.123456789Z", + }, + { + name: "no sub-second", + dbVal: time.Date(2024, 6, 15, 10, 30, 0, 0, time.UTC), + wantNs: 0, + wantRaw: "2024-06-15T10:30:00Z", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := FromDatabaseValue(tt.dbVal, CursorTypeTimestamp) + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + + // Verify nanosecond precision is preserved through ToDriverArg + arg := val.ToDriverArg() + tm, ok := arg.(time.Time) + require.True(t, ok) + assert.Equal(t, tt.wantNs, tm.Nanosecond(), + "nanosecond component must be preserved through FromDatabaseValue -> ToDriverArg") + }) + } +} + +func TestFromDatabaseValue_Timestamp_ByteFormats(t *testing.T) { + // MySQL returns timestamps as []byte in various formats. + // Verify all supported formats are handled. + tests := []struct { + name string + dbVal []byte + wantRaw string + wantTsNano int64 + }{ + { + name: "MySQL DATETIME (no fractional)", + dbVal: []byte("2024-06-15 10:30:00"), + wantRaw: "2024-06-15T10:30:00Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 0, time.UTC).UnixNano(), + }, + { + name: "MySQL DATETIME(3) milliseconds", + dbVal: []byte("2024-06-15 10:30:00.123"), + wantRaw: "2024-06-15T10:30:00.123Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 123000000, time.UTC).UnixNano(), + }, + { + name: "MySQL DATETIME(6) microseconds", + dbVal: []byte("2024-06-15 10:30:00.123456"), + wantRaw: "2024-06-15T10:30:00.123456Z", + wantTsNano: time.Date(2024, 6, 15, 10, 30, 0, 123456000, time.UTC).UnixNano(), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val, err := FromDatabaseValue(tt.dbVal, CursorTypeTimestamp) + require.NoError(t, err) + assert.Equal(t, tt.wantRaw, val.Raw) + assert.Equal(t, tt.wantTsNano, val.Timestamp) + }) + } +} + +func TestInferTypeFromDefaultValue(t *testing.T) { + tests := []struct { + name string + raw string + wantType string + wantErr bool + }{ + {name: "integer", raw: "0", wantType: CursorTypeInteger}, + {name: "timestamp", raw: "2024-01-01T00:00:00Z", wantType: CursorTypeTimestamp}, + {name: "date", raw: "2024-01-01", wantType: CursorTypeDate}, + {name: "float", raw: "12.34", wantType: CursorTypeFloat}, + {name: "invalid", raw: "not-a-value", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := InferTypeFromDefaultValue(tt.raw) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantType, got) + }) + } +} + +func TestInferTypeFromDatabaseValue(t *testing.T) { + tests := []struct { + name string + dbVal interface{} + wantType string + wantErr bool + }{ + {name: "int64", dbVal: int64(42), wantType: CursorTypeInteger}, + {name: "float64", dbVal: 3.14, wantType: CursorTypeFloat}, + {name: "timestamp", dbVal: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), wantType: CursorTypeTimestamp}, + {name: "date-bytes", dbVal: []byte("2024-06-15"), wantType: CursorTypeDate}, + {name: "decimal-bytes", dbVal: []byte("123.45"), wantType: CursorTypeDecimal}, + {name: "integer-string", dbVal: "123", wantType: CursorTypeInteger}, + {name: "invalid", dbVal: struct{}{}, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := InferTypeFromDatabaseValue(tt.dbVal) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantType, got) + }) + } +} + +func TestFromDatabaseValue_Integer_RejectsFractionalFloat(t *testing.T) { + _, err := FromDatabaseValue(float64(12.5), CursorTypeInteger) + require.Error(t, err) + assert.Contains(t, err.Error(), "fractional part") +} diff --git a/x-pack/metricbeat/module/sql/query/cursor_integration_test.go b/x-pack/metricbeat/module/sql/query/cursor_integration_test.go new file mode 100644 index 000000000000..8a2f8222f790 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/cursor_integration_test.go @@ -0,0 +1,2435 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && !requirefips + +package query + +import ( + "database/sql" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/godror/godror" + _ "github.com/lib/pq" + _ "github.com/microsoft/go-mssqldb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp/logptest" + "github.com/elastic/elastic-agent-libs/paths" + + "github.com/elastic/beats/v7/libbeat/tests/compose" + "github.com/elastic/beats/v7/metricbeat/mb" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/mysql" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + sqlmod "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql/query/cursor" +) + +const testTableName = "cursor_test_events" + +// newMetricSetWithPaths creates a MetricSet with custom paths for cursor storage. +// It sets the global paths.Paths.Data to the test's data directory so that +// GetCursorRegistry resolves to the correct temp path, and uses t.Cleanup +// to restore the original value when the test completes. +func newMetricSetWithPaths(t *testing.T, config map[string]interface{}, p *paths.Path) mb.MetricSet { + t.Helper() + + // Override the global data path so GetCursorRegistry creates its + // registry under the per-test temp directory instead of the shared + // process-level path. + origData := paths.Paths.Data + paths.Paths.Data = p.Data + t.Cleanup(func() { paths.Paths.Data = origData }) + + c, err := conf.NewConfigFrom(config) + require.NoError(t, err) + + logger := logptest.NewTestingLogger(t, "") + _, metricsets, err := mb.NewModule(c, mb.Registry, p, logger) + require.NoError(t, err) + require.Len(t, metricsets, 1) + + return metricsets[0] +} + +// createTestPaths creates paths for testing with cursor +func createTestPaths(t *testing.T) *paths.Path { + t.Helper() + tmpDir := t.TempDir() + return &paths.Path{ + Home: tmpDir, + Config: tmpDir, + Data: tmpDir, + Logs: tmpDir, + } +} + +// TestPostgreSQLCursor tests cursor functionality with PostgreSQL +func TestPostgreSQLCursor(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + dsn := fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port) + + // Set up test table + db, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer db.Close() + + setupPostgresTestTable(t, db) + + // Test integer cursor + t.Run("integer cursor", func(t *testing.T) { + testIntegerCursor(t, "postgres", dsn) + }) + + // Test timestamp cursor + t.Run("timestamp cursor", func(t *testing.T) { + testTimestampCursor(t, "postgres", dsn) + }) + + // Test float cursor + t.Run("float cursor", func(t *testing.T) { + testFloatCursor(t, "postgres", dsn) + }) + + // Test decimal cursor + t.Run("decimal cursor", func(t *testing.T) { + testDecimalCursor(t, "postgres", dsn) + }) + + // Test descending integer cursor + t.Run("descending integer cursor", func(t *testing.T) { + testDescendingIntegerCursor(t, "postgres", dsn) + }) + + // Test compound WHERE clause (cursor + additional filter) + t.Run("compound where clause", func(t *testing.T) { + testCompoundWhereCursor(t, "postgres", dsn) + }) +} + +// TestMySQLCursor tests cursor functionality with MySQL +func TestMySQLCursor(t *testing.T) { + service := compose.EnsureUp(t, "mysql") + baseDSN := mysql.GetMySQLEnvDSN(service.Host()) + + // First connect without database to create test database + db0, err := sql.Open("mysql", baseDSN) + require.NoError(t, err) + _, err = db0.Exec("CREATE DATABASE IF NOT EXISTS cursor_test") + require.NoError(t, err) + db0.Close() + + // Now connect to the test database + dsn := baseDSN + "cursor_test" + + // Set up test table + db, err := sql.Open("mysql", dsn) + require.NoError(t, err) + defer db.Close() + defer func() { _, _ = db.Exec("DROP DATABASE IF EXISTS cursor_test") }() + + setupMySQLTestTable(t, db) + + // Test integer cursor + t.Run("integer cursor", func(t *testing.T) { + testIntegerCursor(t, "mysql", dsn) + }) + + // Test timestamp cursor + t.Run("timestamp cursor", func(t *testing.T) { + testTimestampCursor(t, "mysql", dsn) + }) + + // Test decimal cursor (MySQL DECIMAL is very common) + t.Run("decimal cursor", func(t *testing.T) { + testDecimalCursor(t, "mysql", dsn) + }) + + // Test descending scan on MySQL + t.Run("descending integer cursor", func(t *testing.T) { + testDescendingIntegerCursor(t, "mysql", dsn) + }) +} + +// insertTestData inserts n rows into the test table using the appropriate +// placeholder syntax for the given driver. +func insertTestData(t *testing.T, db *sql.DB, driver string, n int) { + t.Helper() + + var insertSQL string + switch driver { + case "postgres": + insertSQL = fmt.Sprintf(`INSERT INTO %s (event_data) VALUES ($1)`, testTableName) + case "mysql": + insertSQL = fmt.Sprintf(`INSERT INTO %s (event_data) VALUES (?)`, testTableName) + default: + t.Fatalf("unsupported driver for insertTestData: %s", driver) + } + + for i := 0; i < n; i++ { + _, err := db.Exec(insertSQL, fmt.Sprintf("event-%d", i)) + require.NoError(t, err) + } +} + +func setupPostgresTestTable(t *testing.T, db *sql.DB) { + t.Helper() + + t.Cleanup(func() { + cleanupTestTable(t, db, "postgres") + }) + + // Drop table if exists + _, err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", testTableName)) + require.NoError(t, err) + + // Create table with columns for all cursor types + createSQL := fmt.Sprintf(` + CREATE TABLE %s ( + id SERIAL PRIMARY KEY, + event_data TEXT, + created_at TIMESTAMP DEFAULT NOW(), + score DOUBLE PRECISION, + price NUMERIC(10,2) + ) + `, testTableName) + _, err = db.Exec(createSQL) + require.NoError(t, err) + + // Insert test data with values for all cursor types + insertSQL := fmt.Sprintf(`INSERT INTO %s (event_data, created_at, score, price) VALUES ($1, $2, $3, $4)`, testTableName) + now := time.Now().UTC() + for i := 0; i < 5; i++ { + score := float64(i+1) * 1.5 // 1.5, 3.0, 4.5, 6.0, 7.5 + price := float64(i+1) * 10.25 // 10.25, 20.50, 30.75, 41.00, 51.25 + _, err := db.Exec(insertSQL, fmt.Sprintf("event-%d", i), now.Add(time.Duration(i)*time.Second), score, price) + require.NoError(t, err) + } +} + +func setupMySQLTestTable(t *testing.T, db *sql.DB) { + t.Helper() + + t.Cleanup(func() { + cleanupTestTable(t, db, "mysql") + }) + + // Drop table if exists + _, err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", testTableName)) + require.NoError(t, err) + + // Create table with columns for all cursor types + createSQL := fmt.Sprintf(` + CREATE TABLE %s ( + id INT AUTO_INCREMENT PRIMARY KEY, + event_data TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + price DECIMAL(10,2) + ) + `, testTableName) + _, err = db.Exec(createSQL) + require.NoError(t, err) + + // Insert test data with values for all cursor types + insertSQL := fmt.Sprintf(`INSERT INTO %s (event_data, created_at, price) VALUES (?, ?, ?)`, testTableName) + now := time.Now().UTC() + for i := 0; i < 5; i++ { + price := float64(i+1) * 10.25 // 10.25, 20.50, 30.75, 41.00, 51.25 + _, err := db.Exec(insertSQL, fmt.Sprintf("event-%d", i), now.Add(time.Duration(i)*time.Second), price) + require.NoError(t, err) + } +} + +func cleanupTestTable(t *testing.T, db *sql.DB, driver string) { + t.Helper() + _, err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", testTableName)) + if err != nil { + t.Logf("Warning: failed to cleanup test table: %v", err) + } +} + +func testIntegerCursor(t *testing.T, driver, dsn string) { + t.Helper() + + // Set up temp paths for cursor store + testPaths := createTestPaths(t) + + query := fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC LIMIT 3", testTableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": driver, + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // First fetch - should get first 3 rows (id 1-3) + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 3, "First fetch should return 3 events") + + // Close the first metricset to persist state + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - should get remaining rows (id 4-5) + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 2, "Second fetch should return 2 events") + + // Close second metricset + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch - should get no rows + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3) + require.Empty(t, events3, "Third fetch should return 0 events") + + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +func testTimestampCursor(t *testing.T, driver, dsn string) { + t.Helper() + + // Set up temp paths for cursor store + testPaths := createTestPaths(t) + + // Set default to a time before our test data + defaultTimestamp := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) + + query := fmt.Sprintf("SELECT id, event_data, created_at FROM %s WHERE created_at > :cursor ORDER BY created_at ASC LIMIT 3", testTableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": driver, + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "created_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": defaultTimestamp, + } + + // First fetch - should get first 3 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 3, "First fetch should return 3 events") + + // Close first metricset to persist state + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - should get remaining rows + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 2, "Second fetch should return 2 events") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +func testFloatCursor(t *testing.T, driver, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + // Scores are: 1.5, 3.0, 4.5, 6.0, 7.5 + // With cursor default 0.0 and LIMIT 3, first fetch gets 1.5, 3.0, 4.5 + query := fmt.Sprintf("SELECT id, event_data, score FROM %s WHERE score > :cursor ORDER BY score ASC LIMIT 3", testTableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": driver, + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "score", + "cursor.type": cursor.CursorTypeFloat, + "cursor.default": "0.0", + } + + // First fetch - should get 3 rows (scores 1.5, 3.0, 4.5) + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 3, "First float fetch should return 3 events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - should get remaining 2 rows (scores 6.0, 7.5) + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 2, "Second float fetch should return 2 events") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch - should get no rows + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3) + require.Empty(t, events3, "Third float fetch should return 0 events") + + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +func testDecimalCursor(t *testing.T, driver, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + // Prices are: 10.25, 20.50, 30.75, 41.00, 51.25 + // With cursor default 0.00 and LIMIT 3, first fetch gets 10.25, 20.50, 30.75 + query := fmt.Sprintf("SELECT id, event_data, price FROM %s WHERE price > :cursor ORDER BY price ASC LIMIT 3", testTableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": driver, + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "price", + "cursor.type": cursor.CursorTypeDecimal, + "cursor.default": "0.00", + } + + // First fetch - should get 3 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 3, "First decimal fetch should return 3 events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - should get remaining 2 rows + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 2, "Second decimal fetch should return 2 events") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch - should get no rows + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3) + require.Empty(t, events3, "Third decimal fetch should return 0 events") + + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +func testDescendingIntegerCursor(t *testing.T, driver, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + // IDs are 1-5. With descending scan, cursor starts high and works down. + // Default 999999, first fetch gets ids 5, 4, 3 (ORDER BY id DESC LIMIT 3) + query := fmt.Sprintf("SELECT id, event_data FROM %s WHERE id < :cursor ORDER BY id DESC LIMIT 3", testTableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": driver, + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "999999", + "cursor.direction": "desc", + } + + // First fetch - should get 3 rows (ids 5, 4, 3) + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 3, "First descending fetch should return 3 events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - cursor should be at min(5,4,3)=3, so fetch ids < 3 → ids 2, 1 + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 2, "Second descending fetch should return 2 events") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch - cursor should be at min(2,1)=1, so fetch ids < 1 → empty + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3) + require.Empty(t, events3, "Third descending fetch should return 0 events") + + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +func testCompoundWhereCursor(t *testing.T, driver, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + // Real-world pattern: cursor combined with an additional filter condition. + // The table has 5 rows with event_data: event-0, event-1, event-2, event-3, event-4. + // We filter for event_data LIKE 'event-%' (matches all) AND id > :cursor. + // This verifies :cursor works correctly when it's not the only WHERE condition. + query := fmt.Sprintf( + "SELECT id, event_data FROM %s WHERE id > :cursor AND event_data LIKE 'event-%%' ORDER BY id ASC LIMIT 3", + testTableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": driver, + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // First fetch - 3 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 3, "First fetch with compound WHERE should return 3 events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - remaining 2 rows + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 2, "Second fetch with compound WHERE should return 2 events") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch - 0 rows + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3) + require.Empty(t, events3, "Third fetch with compound WHERE should return 0 events") + + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// TestCursorStatePersistence verifies cursor state survives restarts +func TestCursorStatePersistence(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + dsn := fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port) + + // Set up test table + db, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer db.Close() + + setupPostgresTestTable(t, db) + + // Set up temp paths - we need to track tmpDir to check file existence + testPaths := createTestPaths(t) + + query := fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC", testTableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // First fetch - get all 5 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 5, "Should get all 5 rows") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Verify cursor state file exists + cursorDir := filepath.Join(testPaths.Data, "sql-cursor") + _, statErr := os.Stat(cursorDir) + assert.NoError(t, statErr, "Cursor state directory should exist") + + // Create new metricset - cursor should be loaded from state + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Empty(t, events2, "Should get 0 rows after cursor loaded from state") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// TestCursorNullValues verifies handling of NULL values in cursor column +func TestCursorNullValues(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + dsn := fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port) + + db, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer db.Close() + + // Create table with nullable timestamp + tableName := "cursor_null_test" + _, err = db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) + require.NoError(t, err) + + _, err = db.Exec(fmt.Sprintf(` + CREATE TABLE %s ( + id SERIAL PRIMARY KEY, + event_data TEXT, + updated_at TIMESTAMP + ) + `, tableName)) + require.NoError(t, err) + defer func() { _, _ = db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }() + + // Insert data with NULL timestamps + now := time.Now().UTC() + _, err = db.Exec(fmt.Sprintf(`INSERT INTO %s (event_data, updated_at) VALUES ($1, $2)`, tableName), "event-1", now) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf(`INSERT INTO %s (event_data, updated_at) VALUES ($1, NULL)`, tableName), "event-2-null") + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf(`INSERT INTO %s (event_data, updated_at) VALUES ($1, $2)`, tableName), "event-3", now.Add(time.Second)) + require.NoError(t, err) + + // Set up temp paths for cursor store + testPaths := createTestPaths(t) + + defaultTimestamp := now.Add(-time.Hour).Format(time.RFC3339) + query := fmt.Sprintf("SELECT id, event_data, updated_at FROM %s WHERE updated_at > :cursor OR updated_at IS NULL ORDER BY id ASC", tableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "updated_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": defaultTimestamp, + } + + ms := newMetricSetWithPaths(t, cfg, testPaths) + events, errs := fetchEvents(t, ms) + require.Empty(t, errs) + + // Should get all 3 rows even though one has NULL + require.Len(t, events, 3, "Should get all 3 events including NULL one") + + if closer, ok := ms.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// fetchEvents is a helper to fetch events from a MetricSet +func fetchEvents(t *testing.T, ms mb.MetricSet) ([]mb.Event, []error) { + t.Helper() + + switch v := ms.(type) { + case mb.ReportingMetricSetV2WithContext: + return mbtest.ReportingFetchV2WithContext(v) + case mb.ReportingMetricSetV2Error: + return mbtest.ReportingFetchV2Error(v) + case mb.ReportingMetricSetV2: + return mbtest.ReportingFetchV2(v) + default: + t.Fatalf("unknown metricset type: %T", ms) + return nil, nil + } +} + +// ============================================================================ +// ORACLE CURSOR TESTS +// ============================================================================ + +// TestOracleCursor tests cursor functionality with Oracle database. +// +// This is a comprehensive test covering: +// - Integer cursor: first run, subsequent runs, restart (state persistence) +// - Timestamp cursor: multi-batch incremental fetch, timezone handling, precision +// - Date cursor: Oracle DATE column handling +// - NULL handling: NULL values in cursor column +// - Empty result set: no matching rows +// - Query change resets cursor: different query → new cursor state +// - Oracle driver type conversions: NUMBER, TIMESTAMP, DATE → Go types +// +// The timestamp multi-batch test verifies cursor-based pagination for Oracle +// TIMESTAMP columns. Note: the timezone mismatch subtest is currently skipped +// due to a known limitation where godror sends time.Time as TIMESTAMP WITH +// TIME ZONE, causing implicit TZ conversion in Oracle comparisons. +func TestOracleCursor(t *testing.T) { + // Skip if Oracle Instant Client is not installed. + // The godror driver requires the Oracle Instant Client library (libclntsh.dylib/so). + // See: https://oracle.github.io/odpi/doc/installation.html + testDB, err := sql.Open("godror", "user/pass@localhost:1521/test") + if err == nil { + err = testDB.Ping() + testDB.Close() + } + if err != nil && containsOracleClientError(err.Error()) { + t.Skip("Skipping Oracle cursor tests: Oracle Instant Client not installed. " + + "See https://oracle.github.io/odpi/doc/installation.html") + } + + service := compose.EnsureUp(t, "oracle") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + waitForOracleConnection(t, host, port) + + dsn := GetOracleConnectionDetails(t, host, port) + + db, err := sql.Open("godror", dsn) + require.NoError(t, err, "Failed to connect to Oracle") + defer db.Close() + + setupOracleTestTable(t, db) + defer cleanupOracleTestTable(t, db) + + // --- Integer cursor tests --- + t.Run("integer_cursor_first_and_subsequent_runs", func(t *testing.T) { + testOracleIntegerCursor(t, dsn) + }) + + t.Run("integer_cursor_restart_preserves_state", func(t *testing.T) { + testOracleIntegerCursorRestart(t, dsn) + }) + + // --- Timestamp cursor tests --- + t.Run("timestamp_cursor_multi_batch", func(t *testing.T) { + testOracleTimestampCursorMultiBatch(t, dsn) + }) + + t.Run("timestamp_cursor_timezone_handling", func(t *testing.T) { + testOracleTimestampTimezoneHandling(t, dsn) + }) + + t.Run("timestamp_cursor_precision", func(t *testing.T) { + testOracleTimestampPrecision(t, dsn) + }) + + // --- Date cursor test --- + t.Run("date_cursor", func(t *testing.T) { + testOracleDateCursor(t, dsn) + }) + + // --- Edge case tests --- + t.Run("null_handling", func(t *testing.T) { + testOracleNullHandling(t, db, dsn) + }) + + t.Run("empty_result_set", func(t *testing.T) { + testOracleEmptyResultSet(t, dsn) + }) + + t.Run("query_change_resets_cursor", func(t *testing.T) { + testOracleQueryChangeResetsCursor(t, dsn) + }) + + // --- Timezone mismatch test (reproduces reported bug) --- + t.Run("timestamp_cursor_timezone_mismatch", func(t *testing.T) { + testOracleTimestampCursorTimezoneMismatch(t, host, port) + }) + + // --- Oracle-specific tests --- + t.Run("driver_type_conversions", func(t *testing.T) { + testOracleDriverTypeConversions(t, db) + }) +} + +func setupOracleTestTable(t *testing.T, db *sql.DB) { + t.Helper() + + t.Cleanup(func() { + cleanupTestTable(t, db, "oracle") + }) + + // Drop table if exists (Oracle doesn't have IF EXISTS, so we ignore errors) + _, _ = db.Exec("DROP TABLE cursor_test_events") + + // Create table with Oracle-specific syntax. + // Uses TIMESTAMP (not TIMESTAMP WITH TIME ZONE) and DATE to test + // Oracle-specific type handling through the godror driver. + createSQL := ` + CREATE TABLE cursor_test_events ( + id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + event_data VARCHAR2(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + event_date DATE DEFAULT CURRENT_DATE + ) + ` + _, err := db.Exec(createSQL) + require.NoError(t, err, "Failed to create Oracle test table") + + // Insert 10 rows with timestamps 1 second apart. + // This gives enough data to test multi-batch pagination (e.g., 3+3+3+1+0). + insertSQL := `INSERT INTO cursor_test_events (event_data, created_at, event_date) VALUES (:1, :2, :3)` + now := time.Now().UTC() + for i := 0; i < 10; i++ { + ts := now.Add(time.Duration(i) * time.Second) + _, err := db.Exec(insertSQL, fmt.Sprintf("event-%d", i), ts, ts) + require.NoError(t, err, "Failed to insert Oracle test data row %d", i) + } + t.Log("Oracle test table created with 10 rows") +} + +func cleanupOracleTestTable(t *testing.T, db *sql.DB) { + t.Helper() + _, err := db.Exec("DROP TABLE cursor_test_events") + if err != nil { + t.Logf("Warning: failed to cleanup Oracle test table: %v", err) + } +} + +// testOracleIntegerCursor verifies integer cursor pagination with Oracle. +// With 10 rows and FETCH FIRST 3, the pattern is: 3 + 3 + 3 + 1 + 0. +// Uses :cursor named parameter binding via :cursor_val. +func testOracleIntegerCursor(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + // :cursor is translated to :cursor_val for Oracle by cursor.TranslateQuery + query := "SELECT id, event_data FROM cursor_test_events WHERE id > :cursor ORDER BY id ASC FETCH FIRST 3 ROWS ONLY" + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // First fetch - ids 1-3 + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1, "First fetch should not have errors") + require.Len(t, events1, 3, "First fetch should return 3 events") + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - ids 4-6 + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2, "Second fetch should not have errors") + require.Len(t, events2, 3, "Second fetch should return 3 events") + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch - ids 7-9 + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3, "Third fetch should not have errors") + require.Len(t, events3, 3, "Third fetch should return 3 events") + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Fourth fetch - id 10 (last row) + ms4 := newMetricSetWithPaths(t, cfg, testPaths) + events4, errs4 := fetchEvents(t, ms4) + require.Empty(t, errs4, "Fourth fetch should not have errors") + require.Len(t, events4, 1, "Fourth fetch should return 1 event (last row)") + if closer, ok := ms4.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Fifth fetch - all consumed, should return 0 + ms5 := newMetricSetWithPaths(t, cfg, testPaths) + events5, errs5 := fetchEvents(t, ms5) + require.Empty(t, errs5, "Fifth fetch should not have errors") + require.Len(t, events5, 0, "Fifth fetch should return 0 events (all consumed)") + if closer, ok := ms5.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// testOracleIntegerCursorRestart verifies that cursor state persists across restarts. +// Fetches all rows, closes MetricSet, re-creates it with same paths, and verifies +// the cursor was loaded from persisted state (0 new rows returned). +func testOracleIntegerCursorRestart(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + // No FETCH FIRST — get all rows in one go + query := "SELECT id, event_data FROM cursor_test_events WHERE id > :cursor ORDER BY id ASC" + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // First fetch - get all 10 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 10, "Should get all 10 rows") + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Verify cursor state directory exists (state was persisted) + cursorDir := filepath.Join(testPaths.Data, "sql-cursor") + _, statErr := os.Stat(cursorDir) + assert.NoError(t, statErr, "Cursor state directory should exist after close") + + // "Restart" — create new MetricSet with same paths (simulates process restart) + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 0, "After restart, should get 0 rows (cursor loaded from persisted state)") + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// testOracleTimestampCursorMultiBatch tests multi-batch timestamp cursor pagination +// with Oracle. With 10 rows and FETCH FIRST 3, the expected pattern is: 3 + 3 + 3 + 1 + 0. +// +// NOTE: This test uses a DSN with session TIME_ZONE='UTC' (via GetOracleConnectionDetails), +// which avoids the timezone mismatch bug where godror sends time.Time as TIMESTAMP WITH +// TIME ZONE and Oracle implicitly converts stored TIMESTAMP values using the session TZ. +// See testOracleTimestampCursorTimezoneMismatch for the known-broken TZ mismatch scenario. +func testOracleTimestampCursorMultiBatch(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + defaultTimestamp := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) + + query := "SELECT id, event_data, created_at FROM cursor_test_events WHERE created_at > :cursor ORDER BY created_at ASC FETCH FIRST 3 ROWS ONLY" + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "created_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": defaultTimestamp, + } + + // First fetch — should get 3 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1, "First timestamp fetch should not have errors") + require.Len(t, events1, 3, "First timestamp fetch should return 3 events") + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch — should get next 3 rows. + // This works because the session TIME_ZONE is UTC, matching godror's + // Timezone parameter. If returned 0 rows, the cursor bind parameter + // may have a type mismatch (TIMESTAMP WITH TIME ZONE vs TIMESTAMP). + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2, "Second timestamp fetch should not have errors") + require.Len(t, events2, 3, + "Second timestamp fetch should return 3 events; "+ + "got 0 may indicate a timezone mismatch between session TZ and godror TZ") + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch — should get 3 more rows + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3, "Third timestamp fetch should not have errors") + require.Len(t, events3, 3, "Third timestamp fetch should return 3 events") + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Fourth fetch — last row + ms4 := newMetricSetWithPaths(t, cfg, testPaths) + events4, errs4 := fetchEvents(t, ms4) + require.Empty(t, errs4, "Fourth timestamp fetch should not have errors") + require.Len(t, events4, 1, "Fourth timestamp fetch should return 1 event (last row)") + if closer, ok := ms4.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Fifth fetch — all consumed + ms5 := newMetricSetWithPaths(t, cfg, testPaths) + events5, errs5 := fetchEvents(t, ms5) + require.Empty(t, errs5, "Fifth timestamp fetch should not have errors") + require.Len(t, events5, 0, "Fifth timestamp fetch should return 0 events (all consumed)") + if closer, ok := ms5.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// testOracleTimestampTimezoneHandling verifies that Oracle timestamps are +// correctly handled in UTC regardless of the database timezone setting. +// The Oracle session is configured with TIME_ZONE='UTC' via GetOracleConnectionDetails. +func testOracleTimestampTimezoneHandling(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + // Use a UTC timestamp as default — the session timezone is set to UTC + defaultTimestamp := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) + + query := "SELECT id, event_data, created_at FROM cursor_test_events WHERE created_at > :cursor ORDER BY created_at ASC FETCH FIRST 5 ROWS ONLY" + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "created_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": defaultTimestamp, + } + + // First fetch — 5 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 5, "First fetch with UTC timezone should return 5 events") + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch — remaining 5 rows (verifies cursor updated correctly across timezone boundary) + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 5, "Second fetch with UTC timezone should return 5 events") + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch — all consumed + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3) + require.Len(t, events3, 0, "Third fetch should return 0 events (all consumed)") + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// testOracleTimestampPrecision verifies that Oracle TIMESTAMP sub-second +// precision is correctly preserved through the cursor round-trip. +// Fetches rows one-at-a-time to verify each cursor update is precise enough +// to skip exactly the current row and return the next. +func testOracleTimestampPrecision(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + defaultTimestamp := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339Nano) + + // FETCH FIRST 1 ROW ONLY to test single-row pagination + query := "SELECT id, event_data, created_at FROM cursor_test_events WHERE created_at > :cursor ORDER BY created_at ASC FETCH FIRST 1 ROWS ONLY" + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "created_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": defaultTimestamp, + } + + // Fetch rows one at a time — precision loss would cause rows to be + // skipped (cursor jumps too far) or re-fetched (cursor doesn't advance). + var totalFetched int + for i := 0; i < 12; i++ { // 10 rows + 2 safety iterations + ms := newMetricSetWithPaths(t, cfg, testPaths) + events, errs := fetchEvents(t, ms) + require.Empty(t, errs, "Fetch %d should not have errors", i+1) + if closer, ok := ms.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + if len(events) == 0 { + break + } + require.Len(t, events, 1, "Fetch %d should return exactly 1 event", i+1) + totalFetched += len(events) + } + + require.Equal(t, 10, totalFetched, + "Should fetch all 10 rows one-at-a-time; precision loss would cause rows to be skipped or re-fetched") +} + +// testOracleDateCursor verifies cursor operation on Oracle DATE columns. +// Oracle DATE includes time component (unlike SQL standard DATE), so the +// cursor uses TO_DATE(:cursor_val, 'YYYY-MM-DD') for proper comparison. +func testOracleDateCursor(t *testing.T, dsn string) { + t.Helper() + + // Oracle DATE includes a time component, so TO_DATE('2026-03-03','YYYY-MM-DD') + // produces midnight. Rows on the same calendar date but with a non-zero time + // are still "greater than" midnight. To test date cursor pagination correctly + // we need rows spread across distinct calendar dates with their time component + // truncated to midnight. + tableName := "cursor_date_test_oracle" + db, err := sql.Open("godror", dsn) + require.NoError(t, err) + defer db.Close() + + _, _ = db.Exec(fmt.Sprintf("DROP TABLE %s", tableName)) + _, err = db.Exec(fmt.Sprintf(` + CREATE TABLE %s ( + id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + event_data VARCHAR2(255), + event_date DATE + ) + `, tableName)) + require.NoError(t, err, "Failed to create date cursor test table") + defer func() { _, _ = db.Exec(fmt.Sprintf("DROP TABLE %s", tableName)) }() + + // Insert 6 rows across 6 distinct dates (today-5 .. today), all at midnight + // so TO_DATE comparison works cleanly. + today := time.Now().UTC().Truncate(24 * time.Hour) + for i := 0; i < 6; i++ { + d := today.AddDate(0, 0, i-5) // today-5, today-4, ..., today + _, err := db.Exec( + fmt.Sprintf("INSERT INTO %s (event_data, event_date) VALUES (:1, :2)", tableName), + fmt.Sprintf("event-%d", i), d, + ) + require.NoError(t, err, "Failed to insert date cursor test row %d", i) + } + + testPaths := createTestPaths(t) + defaultDate := today.AddDate(0, 0, -6).Format("2006-01-02") // day before oldest row + + query := fmt.Sprintf( + "SELECT id, event_data, event_date FROM %s WHERE event_date > TO_DATE(:cursor, 'YYYY-MM-DD') ORDER BY event_date ASC, id ASC FETCH FIRST 3 ROWS ONLY", + tableName, + ) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "event_date", + "cursor.type": cursor.CursorTypeDate, + "cursor.default": defaultDate, + } + + // First fetch — returns 3 oldest rows (dates today-5, today-4, today-3) + // Cursor advances to today-3. + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1, "First date fetch should not have errors") + require.Len(t, events1, 3, "First date fetch should return 3 events") + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch — returns next 3 rows (dates today-2, today-1, today) + // Cursor advances to today. + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2, "Second date fetch should not have errors") + require.Len(t, events2, 3, "Second date fetch should return 3 events") + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch — no rows remain after today + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3, "Third date fetch should not have errors") + require.Empty(t, events3, "Third date fetch should return 0 events") + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// testOracleNullHandling verifies that NULL values in the cursor column +// don't cause errors and are properly skipped during cursor updates. +func testOracleNullHandling(t *testing.T, db *sql.DB, dsn string) { + t.Helper() + + // Create a separate table with NULL values + tableName := "cursor_null_test_oracle" + _, _ = db.Exec(fmt.Sprintf("DROP TABLE %s", tableName)) + + _, err := db.Exec(fmt.Sprintf(` + CREATE TABLE %s ( + id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + event_data VARCHAR2(255), + updated_at TIMESTAMP + ) + `, tableName)) + require.NoError(t, err, "Failed to create NULL test table") + defer func() { _, _ = db.Exec(fmt.Sprintf("DROP TABLE %s", tableName)) }() + + // Insert rows: some with timestamps, some with NULL + now := time.Now().UTC() + _, err = db.Exec(fmt.Sprintf("INSERT INTO %s (event_data, updated_at) VALUES (:1, :2)", tableName), "event-1", now) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf("INSERT INTO %s (event_data, updated_at) VALUES (:1, NULL)", tableName), "event-2-null") + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf("INSERT INTO %s (event_data, updated_at) VALUES (:1, :2)", tableName), "event-3", now.Add(time.Second)) + require.NoError(t, err) + + testPaths := createTestPaths(t) + defaultTimestamp := now.Add(-time.Hour).Format(time.RFC3339) + + // Query includes OR updated_at IS NULL to also fetch NULL rows + query := fmt.Sprintf( + "SELECT id, event_data, updated_at FROM %s WHERE updated_at > :cursor OR updated_at IS NULL ORDER BY id ASC", + tableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "updated_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": defaultTimestamp, + } + + ms := newMetricSetWithPaths(t, cfg, testPaths) + events, errs := fetchEvents(t, ms) + require.Empty(t, errs) + // Should get all 3 rows: 2 with timestamps + 1 with NULL + require.Len(t, events, 3, "Should get all 3 events including NULL timestamp row") + if closer, ok := ms.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// testOracleEmptyResultSet verifies that an empty result set (no matching rows) +// is handled correctly: no errors, 0 events, cursor unchanged. +func testOracleEmptyResultSet(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + // Use a far-future default so no rows match + farFuture := time.Now().Add(100 * 365 * 24 * time.Hour).UTC().Format(time.RFC3339) + + query := "SELECT id, event_data, created_at FROM cursor_test_events WHERE created_at > :cursor ORDER BY created_at ASC FETCH FIRST 3 ROWS ONLY" + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "created_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": farFuture, + } + + ms := newMetricSetWithPaths(t, cfg, testPaths) + events, errs := fetchEvents(t, ms) + require.Empty(t, errs, "Empty result set should not cause errors") + require.Len(t, events, 0, "Should return 0 events when cursor is in the far future") + + // Verify cursor remains unchanged after empty result + queryMs, ok := ms.(*MetricSet) + require.True(t, ok) + + // ParseValue normalizes the format, so compare normalized values + normalizedFarFuture, err := cursor.ParseValue(farFuture, cursor.CursorTypeTimestamp) + require.NoError(t, err) + assert.Equal(t, normalizedFarFuture.String(), queryMs.cursorManager.CursorValueString(), + "Cursor should remain at far-future default after empty result set") + + if closer, ok := ms.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// testOracleQueryChangeResetsCursor verifies that changing the SQL query causes +// the cursor state key to change (because the query is part of the key hash), +// effectively resetting the cursor to the default value. +func testOracleQueryChangeResetsCursor(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + // First query — fetch 5 rows + query1 := "SELECT id, event_data FROM cursor_test_events WHERE id > :cursor ORDER BY id ASC FETCH FIRST 5 ROWS ONLY" + + cfg1 := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query1, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + ms1 := newMetricSetWithPaths(t, cfg1, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 5, "First query should return 5 events") + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Change the query (different FETCH FIRST) — this generates a different + // state key hash, so the cursor resets to default. + query2 := "SELECT id, event_data FROM cursor_test_events WHERE id > :cursor ORDER BY id ASC FETCH FIRST 3 ROWS ONLY" + + cfg2 := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "oracle", + "sql_query": query2, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // Should start from default (0) because the query changed + ms2 := newMetricSetWithPaths(t, cfg2, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 3, + "Changed query should start from default cursor (0), returning first 3 events") + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// testOracleDriverTypeConversions verifies that Oracle column types are correctly +// handled through the cursor pipeline. This tests the full round-trip: +// Oracle type → godror Go type → getValue() → FromDatabaseValue/ParseValue. +func testOracleDriverTypeConversions(t *testing.T, db *sql.DB) { + t.Helper() + + var debug []string + t.Cleanup(func() { + if !t.Failed() { + return + } + for _, msg := range debug { + t.Log(msg) + } + }) + + // Query a row to inspect godror's type mapping for Oracle columns + rows, err := db.Query("SELECT id, created_at, event_date FROM cursor_test_events WHERE ROWNUM <= 1") + require.NoError(t, err) + defer rows.Close() + + require.True(t, rows.Next(), "Should have at least one row") + + // Log Oracle column types for debugging + cols, err := rows.ColumnTypes() + require.NoError(t, err) + for _, col := range cols { + debug = append(debug, fmt.Sprintf( + "Oracle column mapping: %s → DatabaseTypeName=%s, ScanType=%v", + col.Name(), col.DatabaseTypeName(), col.ScanType(), + )) + } + + var id, createdAt, eventDate interface{} + err = rows.Scan(&id, &createdAt, &eventDate) + require.NoError(t, err) + + debug = append(debug, fmt.Sprintf("Oracle NUMBER (id) → Go type: %T, value: %v", id, id)) + debug = append(debug, fmt.Sprintf("Oracle TIMESTAMP (created_at) → Go type: %T, value: %v", createdAt, createdAt)) + debug = append(debug, fmt.Sprintf("Oracle DATE (event_date) → Go type: %T, value: %v", eventDate, eventDate)) + + // Verify Oracle TIMESTAMP → time.Time (which godror returns natively). + // This is important because the cursor pipeline relies on getValue() + // converting time.Time to RFC3339Nano string for mapstr.M storage. + if ts, ok := createdAt.(time.Time); ok { + formatted := ts.Format(time.RFC3339Nano) + val, err := cursor.ParseValue(formatted, cursor.CursorTypeTimestamp) + assert.NoError(t, err, + "Oracle TIMESTAMP → time.Time → RFC3339Nano should round-trip through cursor.ParseValue") + if err == nil { + // Verify ToDriverArg returns time.Time (which godror needs for TIMESTAMP binding) + driverArg := val.ToDriverArg() + _, isTime := driverArg.(time.Time) + assert.True(t, isTime, + "cursor.ToDriverArg() for timestamp should return time.Time, got %T", driverArg) + } + } else { + debug = append(debug, fmt.Sprintf( + "Note: Oracle TIMESTAMP scanned as %T (not time.Time); cursor pipeline handles this via getValue()", + createdAt, + )) + } + + // Verify Oracle DATE → time.Time round-trip + if dt, ok := eventDate.(time.Time); ok { + formatted := dt.Format("2006-01-02") + _, err := cursor.ParseValue(formatted, cursor.CursorTypeDate) + assert.NoError(t, err, + "Oracle DATE → time.Time → date string should round-trip through cursor.ParseValue") + } else { + debug = append(debug, fmt.Sprintf( + "Note: Oracle DATE scanned as %T (not time.Time); cursor pipeline handles this via getValue()", + eventDate, + )) + } + + // Verify Oracle NUMBER round-trip through fmt.Sprint (which getValue uses for unknown types) + idStr := fmt.Sprint(id) + _, err = cursor.ParseValue(idStr, cursor.CursorTypeInteger) + assert.NoError(t, err, + "Oracle NUMBER → fmt.Sprint(%T) → %q should be parseable as integer cursor value", id, idStr) +} + +// getOracleDSNWithTimezone builds an Oracle DSN with explicit timezone settings. +// sessionTZ is the Oracle session TIME_ZONE (e.g., "+05:00", "US/Eastern"). +// goTZ is the timezone godror uses to interpret TIMESTAMP values from Oracle. +func getOracleDSNWithTimezone(t *testing.T, host, port, sessionTZ string, goTZ *time.Location) string { + t.Helper() + connectString := GetOracleConnectString(host, port) + params, err := godror.ParseDSN(connectString) + require.NoError(t, err, "Failed to parse Oracle DSN: %s", connectString) + params.AlterSession = append(params.AlterSession, [2]string{"TIME_ZONE", sessionTZ}) + params.Timezone = goTZ + return params.StringWithPassword() +} + +// testOracleTimestampCursorTimezoneMismatch demonstrates why Oracle timestamp +// cursors require ALTER SESSION SET TIME_ZONE=UTC (a documented constraint). +// +// ## Why TIME_ZONE=UTC is required +// +// Oracle's TIMESTAMP column stores values without timezone info. The godror driver +// sends Go time.Time bind parameters as TIMESTAMP WITH TIME ZONE (OCI type +// SQLT_TIMESTAMP_TZ), carrying the UTC zone marker. When Oracle compares a stored +// TIMESTAMP against a TIMESTAMP WITH TIME ZONE bind parameter, it implicitly +// converts the stored TIMESTAMP to TIMESTAMP WITH TIME ZONE using the session +// timezone (per Oracle comparison rules). If the session timezone is not UTC, +// this conversion shifts the effective time, causing the comparison to fail. +// +// ## How the mismatch manifests +// +// 1. Set Oracle session TIME_ZONE = '+05:00', godror params.Timezone = UTC +// 2. Insert: Go sends time.Time{14:00 UTC}. godror writes to TIMESTAMP column; +// Oracle stores the raw value 14:00 (no TZ conversion for plain TIMESTAMP). +// 3. Read: Oracle returns TIMESTAMP '14:00'. godror (Timezone=UTC) creates +// time.Time{14:00 UTC}. Round-trip is consistent — difference is 0. +// 4. Cursor stores 14:00 UTC. ToDriverArg() returns time.Time{14:00 UTC}. +// 5. Second fetch bind: godror sends TIMESTAMP WITH TIME ZONE '14:00 UTC'. +// Oracle compares stored TIMESTAMP '14:00' by converting it using session +// TZ: 14:00 +05:00 = 09:00 UTC. Comparison: 09:00 UTC > 14:00 UTC → FALSE. +// Returns 0 rows. +// +// The mismatch does NOT manifest when the Oracle session timezone is UTC (as in +// the other tests that use GetOracleConnectionDetails which sets TIME_ZONE='UTC'). +// This is why TIME_ZONE=UTC is a documented requirement for Oracle timestamp cursors. +func testOracleTimestampCursorTimezoneMismatch(t *testing.T, host, port string) { + t.Helper() + + // This test is skipped because it intentionally violates the documented + // requirement that Oracle timestamp cursors need TIME_ZONE=UTC. + // It is kept as a regression test to demonstrate WHY that requirement exists. + t.Skip("Demonstrates documented Oracle TIME_ZONE=UTC requirement — " + + "skipped because non-UTC session timezone is not a supported configuration for timestamp cursors") + + // Build a DSN with timezone MISMATCH: + // - Oracle session timezone = '+05:00' (simulates a non-UTC database) + // - godror interprets timestamps as UTC (simulates Metricbeat running on UTC host) + // + // In production, users don't call ALTER SESSION SET TIME_ZONE and godror + // uses the local timezone. If the Oracle DB defaults to a non-UTC timezone, + // this exact mismatch occurs. + mismatchDSN := getOracleDSNWithTimezone(t, host, port, "+05:00", time.UTC) + + // Create a separate table for this test to avoid interfering with other tests + tableName := "cursor_tz_mismatch_test" + db, err := sql.Open("godror", mismatchDSN) + require.NoError(t, err, "Failed to connect with timezone-mismatch DSN") + defer db.Close() + + // Setup table + _, _ = db.Exec(fmt.Sprintf("DROP TABLE %s", tableName)) + _, err = db.Exec(fmt.Sprintf(` + CREATE TABLE %s ( + id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + event_data VARCHAR2(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + `, tableName)) + require.NoError(t, err, "Failed to create timezone test table") + defer func() { _, _ = db.Exec(fmt.Sprintf("DROP TABLE %s", tableName)) }() + + // Insert 10 rows with timestamps 1 second apart. + // With params.Timezone=UTC, godror sends the raw UTC value and Oracle stores + // it as-is in the plain TIMESTAMP column (no timezone conversion on storage). + now := time.Now().UTC() + for i := 0; i < 10; i++ { + ts := now.Add(time.Duration(i) * time.Second) + _, err := db.Exec( + fmt.Sprintf("INSERT INTO %s (event_data, created_at) VALUES (:1, :2)", tableName), + fmt.Sprintf("event-%d", i), ts) + require.NoError(t, err, "Failed to insert row %d", i) + } + + // Diagnostic: verify what Oracle stored and how godror reads it back. + // The round-trip should be consistent (difference ≈ 0) because both + // insert and read use the same params.Timezone=UTC. The mismatch only + // manifests during comparison (WHERE clause with > operator). + var storedTS time.Time + err = db.QueryRow(fmt.Sprintf( + "SELECT created_at FROM %s WHERE ROWNUM = 1 ORDER BY id", tableName)).Scan(&storedTS) + require.NoError(t, err) + t.Logf("Inserted Go time (UTC): %s", now.Format(time.RFC3339Nano)) + t.Logf("Read back via godror: %s", storedTS.Format(time.RFC3339Nano)) + t.Logf("Round-trip difference: %v (expected ≈ 0)", storedTS.Sub(now).Truncate(time.Millisecond)) + + // Diagnostic: verify the timezone mismatch affects comparisons. + // Use a raw SQL query with a TIMESTAMP WITH TIME ZONE literal to show + // that Oracle converts stored TIMESTAMP using session TZ for comparison. + var countDirect int + err = db.QueryRow(fmt.Sprintf( + "SELECT COUNT(*) FROM %s WHERE created_at > TO_TIMESTAMP(:1, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF\"Z\"')", + tableName), + now.Add(-1*time.Hour).Format("2006-01-02T15:04:05.000000Z")).Scan(&countDirect) + require.NoError(t, err) + t.Logf("Direct query (TO_TIMESTAMP, no TZ info): %d rows (should be 10)", countDirect) + + // Now run the timestamp cursor test with this mismatched DSN. + // Use a far-past default so the first fetch succeeds even with the TZ shift. + // Oracle converts stored TIMESTAMP '14:48:04' using session TZ +05:00 to + // '14:48:04 +05:00' = '09:48:04 UTC'. With default '2000-01-01T00:00:00Z', + // '09:48:04 UTC > 2000-01-01 00:00:00 UTC' → TRUE, so first fetch works. + testPaths := createTestPaths(t) + defaultTimestamp := "2000-01-01T00:00:00Z" + + query := fmt.Sprintf( + "SELECT id, event_data, created_at FROM %s WHERE created_at > :cursor ORDER BY created_at ASC FETCH FIRST 3 ROWS ONLY", + tableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{mismatchDSN}, + "driver": "oracle", + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "created_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": defaultTimestamp, + } + + // First fetch — should get 3 rows. + // Default cursor is year 2000, so even with the +05:00 timezone shift making + // stored values appear ~5h earlier in UTC, all rows still satisfy the condition. + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1, "First fetch should not error") + require.Len(t, events1, 3, "First fetch should return 3 events") + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch — THIS IS WHERE THE BUG MANIFESTS. + // + // After the first fetch, the cursor stores a timestamp read back by godror + // (e.g., 14:48:06 UTC). ToDriverArg() returns time.Time{14:48:06 UTC}. + // godror sends this as TIMESTAMP WITH TIME ZONE '14:48:06 UTC'. + // + // Oracle converts stored TIMESTAMP '14:48:07' (next row) using session + // TZ (+05:00): 14:48:07 +05:00 = 09:48:07 UTC. + // + // Comparison: 09:48:07 UTC > 14:48:06 UTC → FALSE → 0 rows returned. + // + // The stored value (09:48:07 UTC after TZ conversion) appears ~5 hours + // BEHIND the cursor value (14:48:06 UTC), so no rows match. + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2, "Second fetch should not error") + require.Len(t, events2, 3, + "Second fetch with timezone mismatch should return 3 events; "+ + "got 0 indicates the Oracle timestamp cursor is affected by "+ + "timezone mismatch between session TZ (+05:00) and godror TZ (UTC) — "+ + "the bind parameter arrives as TIMESTAMP WITH TIME ZONE (UTC) but "+ + "Oracle converts stored TIMESTAMP using session TZ for comparison") + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch — verify full pagination continues + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3, "Third fetch should not error") + require.Len(t, events3, 3, "Third fetch should return 3 events") + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Fourth fetch — last row + ms4 := newMetricSetWithPaths(t, cfg, testPaths) + events4, errs4 := fetchEvents(t, ms4) + require.Empty(t, errs4, "Fourth fetch should not error") + require.Len(t, events4, 1, "Fourth fetch should return 1 event") + if closer, ok := ms4.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// Note: Oracle helper functions (GetOracleConnectionDetails, GetOracleEnvServiceName, +// GetOracleEnvUsername, GetOracleEnvPassword, GetOracleConnectString, waitForOracleConnection) +// are defined in query_integration_test.go and shared with these tests. + +// ============================================================================ +// MSSQL CURSOR TESTS +// ============================================================================ + +// TestMSSQLCursor tests cursor functionality with Microsoft SQL Server +func TestMSSQLCursor(t *testing.T) { + service := compose.EnsureUp(t, "mssql") + host := service.Host() + + // Wait for MSSQL to be ready + waitForMSSQLConnection(t, host) + + dsn := GetMSSQLConnectionDSN(host) + + // Set up test table + db, err := sql.Open("sqlserver", dsn) + require.NoError(t, err, "Failed to connect to MSSQL") + defer db.Close() + + setupMSSQLTestTable(t, db) + defer cleanupMSSQLTestTable(t, db) + + // Test integer cursor + t.Run("integer_cursor", func(t *testing.T) { + testMSSQLIntegerCursor(t, dsn) + }) + + // Test timestamp cursor + t.Run("timestamp_cursor", func(t *testing.T) { + testMSSQLTimestampCursor(t, dsn) + }) + + // Test date cursor + t.Run("date_cursor", func(t *testing.T) { + testMSSQLDateCursor(t, dsn) + }) +} + +func setupMSSQLTestTable(t *testing.T, db *sql.DB) { + t.Helper() + + t.Cleanup(func() { + cleanupTestTable(t, db, "mssql") + }) + + // Drop table if exists + _, _ = db.Exec("DROP TABLE IF EXISTS cursor_test_events") + + // Create table with MSSQL-specific syntax + createSQL := ` + CREATE TABLE cursor_test_events ( + id INT IDENTITY(1,1) PRIMARY KEY, + event_data NVARCHAR(255), + created_at DATETIME2 DEFAULT GETUTCDATE(), + event_date DATE DEFAULT CAST(GETUTCDATE() AS DATE) + ) + ` + _, err := db.Exec(createSQL) + require.NoError(t, err, "Failed to create MSSQL test table") + + // Insert test data + insertSQL := `INSERT INTO cursor_test_events (event_data, created_at, event_date) VALUES (@p1, @p2, @p3)` + now := time.Now().UTC() + for i := 0; i < 5; i++ { + ts := now.Add(time.Duration(i) * time.Second) + _, err := db.Exec(insertSQL, fmt.Sprintf("event-%d", i), ts, ts) + require.NoError(t, err, "Failed to insert MSSQL test data") + } + t.Log("MSSQL test table created with 5 rows") +} + +func cleanupMSSQLTestTable(t *testing.T, db *sql.DB) { + t.Helper() + _, err := db.Exec("DROP TABLE IF EXISTS cursor_test_events") + if err != nil { + t.Logf("Warning: failed to cleanup MSSQL test table: %v", err) + } +} + +func testMSSQLIntegerCursor(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + + query := "SELECT TOP 3 id, event_data FROM cursor_test_events WHERE id > @p1 ORDER BY id ASC" + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "mssql", + "sql_query": "SELECT TOP 3 id, event_data FROM cursor_test_events WHERE id > :cursor ORDER BY id ASC", + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + _ = query // query is translated internally + + // First fetch - should get first 3 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1, "First fetch should not have errors") + require.Len(t, events1, 3, "First fetch should return 3 events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - should get remaining 2 rows + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2, "Second fetch should not have errors") + require.Len(t, events2, 2, "Second fetch should return 2 events") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Third fetch - should get no rows + ms3 := newMetricSetWithPaths(t, cfg, testPaths) + events3, errs3 := fetchEvents(t, ms3) + require.Empty(t, errs3, "Third fetch should not have errors") + require.Len(t, events3, 0, "Third fetch should return 0 events") + + if closer, ok := ms3.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +func testMSSQLTimestampCursor(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + defaultTimestamp := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "mssql", + "sql_query": "SELECT TOP 3 id, event_data, created_at FROM cursor_test_events WHERE created_at > :cursor ORDER BY created_at ASC", + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "created_at", + "cursor.type": cursor.CursorTypeTimestamp, + "cursor.default": defaultTimestamp, + } + + // First fetch + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1, "First timestamp fetch should not have errors") + require.Len(t, events1, 3, "First timestamp fetch should return 3 events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2, "Second timestamp fetch should not have errors") + require.Len(t, events2, 2, "Second timestamp fetch should return 2 events") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +func testMSSQLDateCursor(t *testing.T, dsn string) { + t.Helper() + + testPaths := createTestPaths(t) + defaultDate := time.Now().Add(-24 * time.Hour).UTC().Format("2006-01-02") + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "mssql", + "sql_query": "SELECT TOP 3 id, event_data, event_date FROM cursor_test_events WHERE event_date > :cursor ORDER BY event_date ASC, id ASC", + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "event_date", + "cursor.type": cursor.CursorTypeDate, + "cursor.default": defaultDate, + } + + // First fetch + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1, "First date fetch should not have errors") + require.GreaterOrEqual(t, len(events1), 1, "First date fetch should return events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// MSSQL helper functions +func GetMSSQLConnectionDSN(host string) string { + user := GetMSSQLEnvUser() + password := GetMSSQLEnvPassword() + // Disable TLS encryption to avoid certificate validation issues in testing + return fmt.Sprintf("sqlserver://%s:%s@%s?encrypt=disable", user, password, host) +} + +func GetMSSQLEnvUser() string { + user := os.Getenv("MSSQL_USER") + if user == "" { + user = "SA" + } + return user +} + +func GetMSSQLEnvPassword() string { + password := os.Getenv("MSSQL_PASSWORD") + if password == "" { + password = "1234_asdf" + } + return password +} + +func waitForMSSQLConnection(t *testing.T, host string) { + maxRetries := 30 + baseDelay := 2 * time.Second + + dsn := GetMSSQLConnectionDSN(host) + + for i := 0; i < maxRetries; i++ { + db, err := sql.Open("sqlserver", dsn) + if err == nil { + err = db.Ping() + db.Close() + if err == nil { + t.Log("MSSQL is ready") + // Give it a bit more time for stability + time.Sleep(5 * time.Second) + return + } + } + + delay := time.Duration(1< 30*time.Second { + delay = 30 * time.Second + } + + t.Logf("MSSQL not ready yet (attempt %d/%d), waiting %v: %v", i+1, maxRetries, delay, err) + time.Sleep(delay) + } + + t.Fatalf("MSSQL service did not become ready after %d attempts", maxRetries) +} + +// ============================================================================ +// MULTI-DATABASE CURSOR STATE ISOLATION TEST +// ============================================================================ + +// TestCursorQueryChangeResetsState verifies that changing the SQL query text +// causes a new cursor state key to be used, so fetching restarts from the +// configured default cursor. This mirrors binary restart behavior where the +// same path.data is reused across process restarts. +func TestCursorQueryChangeResetsState(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + dsn := fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port) + + db, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer db.Close() + + setupPostgresTestTable(t, db) + + testPaths := createTestPaths(t) + + query1 := fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC LIMIT 2", testTableName) + query2 := fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC LIMIT 3", testTableName) + + cfgBase := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + cfg1 := map[string]interface{}{} + for k, v := range cfgBase { + cfg1[k] = v + } + cfg1["sql_query"] = query1 + + cfg2 := map[string]interface{}{} + for k, v := range cfgBase { + cfg2[k] = v + } + cfg2["sql_query"] = query2 + + // First run with query1 advances cursor to id=2. + ms1 := newMetricSetWithPaths(t, cfg1, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 2, "First query should return 2 events") + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second run with same query1 resumes from persisted state (ids 3,4). + ms1Again := newMetricSetWithPaths(t, cfg1, testPaths) + events1Again, errs1Again := fetchEvents(t, ms1Again) + require.Empty(t, errs1Again) + require.Len(t, events1Again, 2, "Second run with same query should continue from previous cursor") + if closer, ok := ms1Again.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Changing query text should use a different state key and reset to default. + ms2 := newMetricSetWithPaths(t, cfg2, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 3, "Changed query should reset cursor and return first 3 rows") + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// TestCursorStateIsolation verifies that cursor states are isolated per database/query +func TestCursorStateIsolation(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + dsn := fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port) + + db, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer db.Close() + + setupPostgresTestTable(t, db) + + // Use shared paths so both MetricSets would share state if not properly isolated + testPaths := createTestPaths(t) + + // Two different queries on same table should have separate cursor states + query1 := fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC LIMIT 2", testTableName) + query2 := fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC LIMIT 3", testTableName) + + cfg1 := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "sql_query": query1, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + cfg2 := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "sql_query": query2, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // First query fetch - gets 2 rows (ids 1, 2) + ms1 := newMetricSetWithPaths(t, cfg1, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 2, "First query should return 2 events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second query fetch - should get 3 rows (ids 1, 2, 3) because it has separate state + ms2 := newMetricSetWithPaths(t, cfg2, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 3, "Second query should return 3 events (separate cursor state)") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // First query again - should continue from where it left off (gets ids 3, 4) + ms1again := newMetricSetWithPaths(t, cfg1, testPaths) + events1again, errs1again := fetchEvents(t, ms1again) + require.Empty(t, errs1again) + require.Len(t, events1again, 2, "First query (second fetch) should return 2 more events") + + if closer, ok := ms1again.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + t.Log("Cursor state isolation verified - different queries maintain separate cursor states") +} + +// TestCursorRegistrySharing verifies that multiple SQL module instances share +// the same statestore.Registry pointer, preventing file lock conflicts. +// +// This test ensures the ModuleBuilder closure pattern correctly shares the +// registry across all module instances, which is critical for avoiding the +// original bug where multiple independent stores operated on the same files. +func TestCursorRegistrySharing(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + dsn := fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port) + + // Setup test database + db, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer db.Close() + + setupPostgresTestTable(t, db) + + // Insert additional test data: 10 rows (setupPostgresTestTable already inserts 5) + insertTestData(t, db, "postgres", 10) + + // Create shared test paths - both MetricSets will use same data directory + testPaths := createTestPaths(t) + + // Configuration for first MetricSet - query with LIMIT 2 + cfg1 := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "period": "10s", + "sql_query": fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC LIMIT 2", testTableName), + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // Configuration for second MetricSet - different query with LIMIT 3 + cfg2 := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "period": "10s", + "sql_query": fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC LIMIT 3", testTableName), + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // Create two MetricSet instances using the same paths + ms1 := newMetricSetWithPaths(t, cfg1, testPaths) + ms2 := newMetricSetWithPaths(t, cfg2, testPaths) + + // Extract the underlying modules + metricSet1, ok := ms1.(*MetricSet) + require.True(t, ok, "MetricSet should be *query.MetricSet") + + metricSet2, ok := ms2.(*MetricSet) + require.True(t, ok, "MetricSet should be *query.MetricSet") + + // Type-assert to sql.Module interface to access GetCursorRegistry + mod1, ok := metricSet1.Module().(sqlmod.Module) + require.True(t, ok, "Module should implement sqlmod.Module interface") + + mod2, ok := metricSet2.Module().(sqlmod.Module) + require.True(t, ok, "Module should implement sqlmod.Module interface") + + // Get registry from both modules + registry1, err1 := mod1.GetCursorRegistry() + require.NoError(t, err1, "GetCursorRegistry should not error") + require.NotNil(t, registry1, "Registry should not be nil") + + registry2, err2 := mod2.GetCursorRegistry() + require.NoError(t, err2, "GetCursorRegistry should not error") + require.NotNil(t, registry2, "Registry should not be nil") + + // CRITICAL ASSERTION: Verify they're the SAME pointer (shared instance) + // This is the core of the fix - if pointers differ, multiple stores will + // try to access the same files, causing lock conflicts + require.Same(t, registry1, registry2, + "Both module instances MUST share the exact same registry pointer to avoid file conflicts") + + // Also verify state isolation works correctly with the shared registry + // Each query should maintain its own cursor state via unique state keys + + // Fetch from ms1 (gets 2 rows: id=1, id=2) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1) + require.Len(t, events1, 2, "First fetch from ms1 should return 2 rows") + + // Fetch from ms2 (gets 3 rows: id=1, id=2, id=3) + // This should have separate state from ms1 — cursor starts at 0 independently + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2) + require.Len(t, events2, 3, "First fetch from ms2 should return 3 rows") + + // Fetch from ms1 again (continues from id=2, gets id=3, id=4) + events3, errs3 := fetchEvents(t, ms1) + require.Empty(t, errs3) + require.Len(t, events3, 2, "Second fetch from ms1 should return next 2 rows") + + // Fetch from ms2 again (continues from id=3, gets id=4, id=5, id=6) + events4, errs4 := fetchEvents(t, ms2) + require.Empty(t, errs4) + require.Len(t, events4, 3, "Second fetch from ms2 should return next 3 rows") + + // Cleanup + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// ============================================================================ +// QUERY TIMEOUT TEST +// ============================================================================ + +// TestCursorQueryTimeout verifies that a hung query is cancelled after the +// module's configured timeout. Uses PostgreSQL's pg_sleep() to simulate a +// query that takes longer than the timeout. +func TestCursorQueryTimeout(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + dsn := fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port) + + // Ensure test table exists (we need a valid cursor query structure) + db, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer db.Close() + + setupPostgresTestTable(t, db) + + testPaths := createTestPaths(t) + + // Query that sleeps for 30 seconds — well beyond the 2s timeout we'll set. + // pg_sleep returns void, so we wrap it in a subquery that also selects from + // our real table so the cursor column ("id") is present. + // Using a CTE: the sleep executes, then we return rows from the real table. + slowQuery := fmt.Sprintf( + "SELECT id, event_data FROM %s WHERE id > :cursor AND pg_sleep(30) IS NOT NULL ORDER BY id ASC LIMIT 3", + testTableName, + ) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "period": "60s", + "timeout": "2s", // Very short timeout — query should be cancelled + "sql_query": slowQuery, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + ms := newMetricSetWithPaths(t, cfg, testPaths) + + // Measure the time taken — should be roughly the timeout, not 30s + start := time.Now() + _, errs := fetchEvents(t, ms) + elapsed := time.Since(start) + + // Should have an error (context deadline exceeded) + require.NotEmpty(t, errs, "Expected an error from the timed-out query") + t.Logf("Query timeout error: %v", errs[0]) + + // Verify it contains a context-related error + errMsg := errs[0].Error() + assert.True(t, + strings.Contains(errMsg, "context deadline exceeded") || + strings.Contains(errMsg, "context canceled") || + strings.Contains(errMsg, "canceling statement due to user request"), + "Error should indicate context cancellation, got: %s", errMsg) + + // Verify it completed quickly (within ~5s) rather than waiting 30s + assert.Less(t, elapsed, 10*time.Second, + "Query should have been cancelled by timeout, not waited for pg_sleep(30)") + t.Logf("Query was cancelled after %v (timeout was 2s)", elapsed) + + // Verify cursor was NOT advanced (it should remain at default "0") + queryMs, ok := ms.(*MetricSet) + require.True(t, ok, "MetricSet should be of type *MetricSet") + assert.Equal(t, "0", queryMs.cursorManager.CursorValueString(), + "Cursor should remain at default after timeout") + + if closer, ok := ms.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } +} + +// TestCursorNormalQueryCompletesWithinTimeout verifies that a normal (fast) +// query completes successfully even with a timeout configured. +func TestCursorNormalQueryCompletesWithinTimeout(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + dsn := fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port) + + db, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer db.Close() + + setupPostgresTestTable(t, db) + + testPaths := createTestPaths(t) + + query := fmt.Sprintf("SELECT id, event_data FROM %s WHERE id > :cursor ORDER BY id ASC LIMIT 3", testTableName) + + cfg := map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{dsn}, + "driver": "postgres", + "period": "60s", + "timeout": "30s", // Generous timeout — query should complete well within + "sql_query": query, + "sql_response_format": tableResponseFormat, + "raw_data.enabled": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": cursor.CursorTypeInteger, + "cursor.default": "0", + } + + // First fetch - should work fine and return 3 rows + ms1 := newMetricSetWithPaths(t, cfg, testPaths) + events1, errs1 := fetchEvents(t, ms1) + require.Empty(t, errs1, "Normal query should succeed with timeout configured") + require.Len(t, events1, 3, "First fetch should return 3 events") + + if closer, ok := ms1.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + // Second fetch - should get remaining 2 rows (cursor persisted correctly) + ms2 := newMetricSetWithPaths(t, cfg, testPaths) + events2, errs2 := fetchEvents(t, ms2) + require.Empty(t, errs2, "Second fetch should succeed") + require.Len(t, events2, 2, "Second fetch should return 2 events") + + if closer, ok := ms2.(mb.Closer); ok { + require.NoError(t, closer.Close()) + } + + t.Log("Normal query completes successfully with timeout configured") +} + +// containsOracleClientError checks if the error message indicates Oracle Instant Client is missing +func containsOracleClientError(errMsg string) bool { + oracleClientErrors := []string{ + "Cannot locate a 64-bit Oracle Client library", + "libclntsh", + "DPI-1047", + "oracle client", + } + errLower := strings.ToLower(errMsg) + for _, pattern := range oracleClientErrors { + if strings.Contains(errLower, strings.ToLower(pattern)) { + return true + } + } + return false +} diff --git a/x-pack/metricbeat/module/sql/query/dsn.go b/x-pack/metricbeat/module/sql/query/dsn.go index d33a3165c0db..cd2b110fae6a 100644 --- a/x-pack/metricbeat/module/sql/query/dsn.go +++ b/x-pack/metricbeat/module/sql/query/dsn.go @@ -37,11 +37,11 @@ func ParseDSN(mod mb.Module, host string) (_ mb.HostData, fetchErr error) { fetchErr = sql.SanitizeError(fetchErr, host) }() - logger := logp.NewLogger("") - // At the time of writing, mod always is of type *mb.BaseModule. - // If this assumption is ever broken, we use global logger then - sqlModule, ok := mod.(*mb.BaseModule) - if ok { + logger := logp.NewLogger("sql") + // The module may be *mb.BaseModule (DefaultModuleFactory) or a custom type + // embedding BaseModule (e.g., the sql.module from ModuleBuilder). In either + // case, try to get the module-scoped logger for better log context. + if sqlModule, ok := mod.(*mb.BaseModule); ok { logger = sqlModule.Logger } @@ -112,7 +112,6 @@ func oracleParseDSN(config ConnectionDetails, host string) (mb.HostData, error) func mysqlParseDSN(config ConnectionDetails, host string, logger *logp.Logger) (mb.HostData, error) { c, err := mysql.ParseDSN(host) - if err != nil { return mb.HostData{}, fmt.Errorf("error trying to parse connection string in field 'hosts': %w", err) } diff --git a/x-pack/metricbeat/module/sql/query/dsn_test.go b/x-pack/metricbeat/module/sql/query/dsn_test.go index b2e3d73e17ae..6dbcc59927d4 100644 --- a/x-pack/metricbeat/module/sql/query/dsn_test.go +++ b/x-pack/metricbeat/module/sql/query/dsn_test.go @@ -28,9 +28,9 @@ const ( ) func prepare(t *testing.T) { - require.NoError(t, os.WriteFile(caPath, []byte(mockCA), 0644)) - require.NoError(t, os.WriteFile(keyPath, []byte(mockKey), 0600)) - require.NoError(t, os.WriteFile(certPath, []byte(mockCert), 0644)) + require.NoError(t, os.WriteFile(caPath, []byte(mockCA), 0o644)) + require.NoError(t, os.WriteFile(keyPath, []byte(mockKey), 0o600)) + require.NoError(t, os.WriteFile(certPath, []byte(mockCert), 0o644)) } func cleanup(t *testing.T) { diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index bf5590683d68..bdff45debe9d 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -10,9 +10,14 @@ import ( "context" "errors" "fmt" + "strings" + "sync" "github.com/elastic/beats/v7/metricbeat/helper/sql" "github.com/elastic/beats/v7/metricbeat/mb" + sqlmod "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql/query/cursor" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -54,6 +59,9 @@ type config struct { // Support fetch response for given queries from all databases. // NOTE: Currently, mssql driver only respects FetchFromAllDatabases. FetchFromAllDatabases bool `config:"fetch_from_all_databases"` + + // Cursor configuration for incremental data fetching + Cursor cursor.Config `config:"cursor"` } // MetricSet holds any configuration or state information. It must implement @@ -63,6 +71,11 @@ type config struct { type MetricSet struct { mb.BaseMetricSet Config config + + // Cursor-related fields (only used when cursor is enabled) + cursorManager *cursor.Manager + translatedQuery string // Query with driver-specific placeholder + fetchMutex sync.Mutex // Prevents concurrent fetch operations } // rawData is the minimum required set of fields to generate fully customized events with their own module key space @@ -71,6 +84,39 @@ type rawData struct { Enabled bool `config:"enabled"` } +// dbClient captures the subset of DB operations used by the query metricset. +// It allows unit tests to exercise fetch paths without real DB connections. +type dbClient interface { + FetchTableMode(ctx context.Context, query string) ([]mapstr.M, error) + FetchTableModeWithParams(ctx context.Context, query string, args ...interface{}) ([]mapstr.M, error) + FetchVariableMode(ctx context.Context, query string) (mapstr.M, error) + Close() error +} + +var newDBClient = func(driver, dsn string, logger *logp.Logger) (dbClient, error) { + return sql.NewDBClient(driver, dsn, logger) +} + +// maybeWrapOracleClientError enriches Oracle client-library load failures with +// an actionable preflight hint for operators. +func maybeWrapOracleClientError(driver string, err error) error { + if err == nil { + return nil + } + + if sql.SwitchDriverName(driver) != "godror" { + return err + } + + errMsg := err.Error() + if !strings.Contains(errMsg, "DPI-1047") && !strings.Contains(errMsg, "Cannot locate a 64-bit Oracle Client library") { + return err + } + + return fmt.Errorf("%w. Oracle preflight check failed: install Oracle Instant Client and make libclntsh discoverable "+ + "(macOS: DYLD_LIBRARY_PATH, Linux: LD_LIBRARY_PATH). See https://oracle.github.io/odpi/doc/installation.html", err) +} + // New creates a new instance of the MetricSet. New is responsible for unpacking // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { @@ -104,9 +150,105 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, fmt.Errorf("both query inputs provided, must provide either sql_query or sql_queries") } + // Initialize cursor if enabled + if b.Config.Cursor.Enabled { + if err := b.initCursor(base); err != nil { + return nil, err + } + } + return b, nil } +// initCursor initializes the cursor manager if cursor is enabled. +// This validates cursor configuration and sets up the state store. +func (m *MetricSet) initCursor(base mb.BaseMetricSet) error { + // Cursor only works with single query mode + if len(m.Config.Queries) > 0 { + return errors.New("cursor is not supported with sql_queries (multiple queries)") + } + + // Cursor is not compatible with fetch_from_all_databases + if m.Config.FetchFromAllDatabases { + return errors.New("cursor is not supported with fetch_from_all_databases") + } + + // Cursor requires table response format + if m.Config.ResponseFormat != tableResponseFormat { + return errors.New("cursor requires sql_response_format: table") + } + + // Translate query placeholder for the driver + m.translatedQuery = cursor.TranslateQuery(m.Config.Query, m.Config.Driver) + + // Get a cursor Store handle from the shared Module-level registry. + // This ensures a single memlog.Registry is shared across all SQL MetricSet + // instances, avoiding multiple independent stores operating on the same files. + store, err := m.openCursorStore(base) + if err != nil { + return fmt.Errorf("cursor store initialization failed: %w", err) + } + + // Create cursor manager. + // By default we use the full URI (not just Host) for state key generation + // because Host often strips the database name (for example, "localhost:5432" + // for both postgres://...localhost:5432/db_a and db_b). + // If cursor.state_id is configured, it is used as a stable identity instead. + // The resulting identity is hashed via xxhash so there is no secret leakage + // risk in the stored key. + mgr, err := cursor.NewManager( + m.Config.Cursor, + store, + m.HostData().URI, + m.Config.Query, // use original query for state key + m.Logger(), + ) + if err != nil { + // Cleanup store on error + if closeErr := store.Close(); closeErr != nil { + m.Logger().Warnf("Failed to close store after cursor manager creation error: %v", closeErr) + } + return fmt.Errorf("cursor initialization failed: %w", err) + } + + m.cursorManager = mgr + return nil +} + +// openCursorStore returns a cursor Store handle from the shared Module-level +// statestore registry. The registry must be initialized via sql.ModuleBuilder +// to ensure proper sharing across all SQL module instances. +// +// This method will fail if the module does not implement the sql.Module interface, +// preventing the creation of multiple independent stores that could cause file +// lock conflicts. +func (m *MetricSet) openCursorStore(base mb.BaseMetricSet) (*cursor.Store, error) { + mod, ok := base.Module().(sqlmod.Module) + if !ok { + return nil, fmt.Errorf("cursor requires SQL module to implement registry interface; " + + "ensure module is initialized via sql.ModuleBuilder (not DefaultModuleFactory)") + } + + registry, err := mod.GetCursorRegistry() + if err != nil { + return nil, err + } + + // Debug log to verify registry sharing is working + m.Logger().Debugf("Using shared SQL cursor registry at %p", registry) + + return cursor.NewStoreFromRegistry(registry, m.Logger().Named("cursor")) +} + +// Close implements mb.Closer for proper resource cleanup. +// This is called when the MetricSet is stopped. +func (m *MetricSet) Close() error { + if m.cursorManager != nil { + return m.cursorManager.Close() + } + return nil +} + // queryDBNames returns the query to list databases present in a server // as per the driver name. If the given driver is not supported, queryDBNames // returns an empty query. @@ -115,7 +257,7 @@ func queryDBNames(driver string) string { // NOTE: Add support for other drivers in future as when the need arises. // dbSelector function would also required to be modified in order to add // support for a new driver. - case "mssql": + case "mssql", "sqlserver": return "SELECT [name] FROM sys.databases WITH (NOLOCK) WHERE state = 0 AND HAS_DBACCESS([name]) = 1" // case "mysql": // return "SHOW DATABASES" @@ -139,13 +281,13 @@ func dbSelector(driver, dbName string) string { // queryDBNames function would also required to be modified in order to add // support for a new driver. // - case "mssql": + case "mssql", "sqlserver": return fmt.Sprintf("USE [%s];", dbName) } return "" } -func (m *MetricSet) fetch(ctx context.Context, db *sql.DbClient, reporter mb.ReporterV2, queries []query) (_ bool, fetchErr error) { +func (m *MetricSet) fetch(ctx context.Context, db dbClient, reporter mb.ReporterV2, queries []query) (_ bool, fetchErr error) { defer func() { fetchErr = sql.SanitizeError(fetchErr, m.HostData().URI) }() @@ -220,9 +362,21 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) (fetchErr fetchErr = sql.SanitizeError(fetchErr, m.HostData().URI) }() - db, err := sql.NewDBClient(m.Config.Driver, m.HostData().URI, m.Logger()) + // Handle cursor-enabled case with concurrent execution prevention + if m.cursorManager != nil { + // Try to acquire lock without blocking + if !m.fetchMutex.TryLock() { + m.Logger().Warnf("Previous collection still in progress, skipping this cycle (driver=%s)", m.Config.Driver) + return nil + } + defer m.fetchMutex.Unlock() + + return m.fetchWithCursor(ctx, reporter) + } + + db, err := newDBClient(m.Config.Driver, m.HostData().URI, m.Logger()) if err != nil { - return fmt.Errorf("cannot open connection: %w", err) + return fmt.Errorf("cannot open connection: %w", maybeWrapOracleClientError(m.Config.Driver, err)) } defer db.Close() @@ -306,6 +460,80 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) (fetchErr return nil } +// fetchWithCursor executes the query with cursor-based incremental fetching. +// It uses the cursor manager to track the last fetched row and only retrieves new data. +// +// The context is wrapped with the module's configured timeout to prevent hung queries +// from blocking indefinitely and causing all subsequent collection cycles to be skipped +// via fetchMutex.TryLock(). The timeout defaults to the module's period if not set. +// +// Note: the timeout is applied here (cursor path only) rather than in Fetch() because +// the non-cursor path has never enforced a timeout. Applying it there would be a +// breaking change for existing users whose queries legitimately take longer than their +// configured period. +func (m *MetricSet) fetchWithCursor(ctx context.Context, reporter mb.ReporterV2) error { + timeout := m.Module().Config().Timeout + + // Apply the module's configured timeout (defaults to period) to prevent hung queries. + // Without this, a hung query blocks the goroutine indefinitely and all future + // collection cycles are skipped via fetchMutex.TryLock(). + if timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + + cursorVal := m.cursorManager.CursorValueForQuery() + cursorBefore := m.cursorManager.CursorValueString() + stateKey := m.cursorManager.GetStateKey() + + m.Logger().Debugf("Cursor fetch start: driver=%s timeout=%s state_key=%s cursor=%s", + m.Config.Driver, timeout, stateKey, cursorBefore) + + db, err := newDBClient(m.Config.Driver, m.HostData().URI, m.Logger()) + if err != nil { + return fmt.Errorf("cannot open connection: %w", maybeWrapOracleClientError(m.Config.Driver, err)) + } + defer db.Close() + + // Execute parameterized query with cursor value + rows, err := db.FetchTableModeWithParams(ctx, m.translatedQuery, cursorVal) + if err != nil { + return fmt.Errorf("fetch with cursor failed: %w", err) + } + + m.Logger().Debugf("Query returned %d rows", len(rows)) + + if len(rows) == 0 { + m.Logger().Debugf("Cursor unchanged (state_key=%s, cursor=%s): no rows matched", stateKey, cursorBefore) + return nil + } + + // Report events BEFORE updating cursor (at-least-once delivery). + // If the reporter rejects an event (returns false), stop immediately + // and do NOT advance the cursor so the same rows are re-fetched next cycle. + for _, row := range rows { + if ok := m.reportEvent(row, reporter, m.Config.Query); !ok { + m.Logger().Warnf("Reporter stopped accepting events; cursor unchanged (state_key=%s, cursor=%s) to avoid data loss", + stateKey, cursorBefore) + return nil + } + } + + // Update cursor state — only reached when all events were accepted. + if err := m.cursorManager.UpdateFromResults(rows); err != nil { + m.Logger().Warnf("Failed to save cursor state (state_key=%s, cursor_before=%s, rows=%d): %v", + stateKey, cursorBefore, len(rows), err) + // Don't fail the fetch - events were already emitted. + // Next run will re-fetch some data (duplicates are better than data loss). + return nil + } + m.Logger().Debugf("Cursor fetch completed: state_key=%s cursor_before=%s cursor_after=%s rows=%d", + stateKey, cursorBefore, m.cursorManager.CursorValueString(), len(rows)) + + return nil +} + // reportEvent using 'user' mode with keys under `sql.metrics.*` or using Raw data mode (module and metricset key spaces // provided by the user) func (m *MetricSet) reportEvent(ms mapstr.M, reporter mb.ReporterV2, qry ...string) bool { diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index 39c849c53435..ef9438eea1e3 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -7,6 +7,7 @@ package query import ( + "database/sql" "fmt" "net" "os" @@ -165,7 +166,6 @@ func TestPostgreSQL(t *testing.T) { t.Run("fetch with URL", func(t *testing.T) { testFetch(t, cfg) }) - }) t.Run("table mode", func(t *testing.T) { @@ -189,7 +189,6 @@ func TestPostgreSQL(t *testing.T) { t.Run("fetch with URL", func(t *testing.T) { testFetch(t, cfg) }) - }) t.Run("merged mode", func(t *testing.T) { @@ -197,8 +196,8 @@ func TestPostgreSQL(t *testing.T) { config: config{ Driver: "postgres", Queries: []query{ - query{Query: "SELECT blks_hit FROM pg_stat_database limit 1;", ResponseFormat: "table"}, - query{Query: "SELECT blks_read FROM pg_stat_database limit 1;", ResponseFormat: "table"}, + {Query: "SELECT blks_hit FROM pg_stat_database limit 1;", ResponseFormat: "table"}, + {Query: "SELECT blks_read FROM pg_stat_database limit 1;", ResponseFormat: "table"}, }, ResponseFormat: tableResponseFormat, RawData: rawData{ @@ -221,12 +220,24 @@ func TestPostgreSQL(t *testing.T) { t.Run("fetch with URL", func(t *testing.T) { testFetch(t, cfg) }) - }) }) } func TestOracle(t *testing.T) { + // Skip if Oracle Instant Client is not installed. + // The godror driver requires the Oracle Instant Client library (libclntsh.dylib/so). + // See: https://oracle.github.io/odpi/doc/installation.html + testDB, err := sql.Open("godror", "user/pass@localhost:1521/test") + if err == nil { + err = testDB.Ping() + _ = testDB.Close() + } + if err != nil && containsOracleClientError(err.Error()) { + t.Skip("Skipping Oracle integration tests: Oracle Instant Client not installed. " + + "See https://oracle.github.io/odpi/doc/installation.html") + } + service := compose.EnsureUp(t, "oracle") host, port, _ := net.SplitHostPort(service.Host()) @@ -305,10 +316,19 @@ func assertFieldContainsFloat64(field string, limit float64) func(t *testing.T, } } -func GetOracleConnectionDetails(t *testing.T, host string, port string) string { +// GetOracleConnectionDetails returns the Oracle connection DSN. +// It sets the session timezone to UTC to ensure consistent timestamp handling +// between Go (which uses UTC) and Oracle. +func GetOracleConnectionDetails(t *testing.T, host, port string) string { connectString := GetOracleConnectString(host, port) params, err := godror.ParseDSN(connectString) require.NoError(t, err, "Failed to parse Oracle DSN: %s", connectString) + // Set session timezone to UTC on every connection. + // Without this, the godror driver may convert Go's UTC time.Time values + // to Oracle's session timezone when binding query parameters, causing + // timestamp comparisons to fail if the session TZ differs from UTC. + params.AlterSession = append(params.AlterSession, [2]string{"TIME_ZONE", "UTC"}) + params.Timezone = time.UTC return params.StringWithPassword() } @@ -340,7 +360,7 @@ func GetOracleEnvPassword() string { } // GetOracleConnectString builds the Oracle connection string with proper format -func GetOracleConnectString(host string, port string) string { +func GetOracleConnectString(host, port string) string { connectString := os.Getenv("ORACLE_CONNECT_STRING") if len(connectString) == 0 { // Use the recommended connection string format from godror documentation @@ -356,6 +376,7 @@ func GetOracleConnectString(host string, port string) string { if GetOracleEnvUsername() == "sys" { connectString += "?sysdba=1" } + } return connectString } diff --git a/x-pack/metricbeat/module/sql/query/query_test.go b/x-pack/metricbeat/module/sql/query/query_test.go new file mode 100644 index 000000000000..b22a4cd426fb --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/query_test.go @@ -0,0 +1,870 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !requirefips + +package query + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/beats/v7/libbeat/statestore/backend/memlog" + "github.com/elastic/beats/v7/metricbeat/mb" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql/query/cursor" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/logp/logptest" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/paths" +) + +type fakeDBClient struct { + tableRows []mapstr.M + tableErr error + variableRows mapstr.M + variableErr error + tableWithParamRows []mapstr.M + tableWithParamErr error + + tableQueries []string + variableQueries []string + withParamQueries []string + withParamArgs [][]interface{} + closed bool + + fetchTableFn func(ctx context.Context, query string) ([]mapstr.M, error) + fetchVariableFn func(ctx context.Context, query string) (mapstr.M, error) + fetchTableWithParamFn func(ctx context.Context, query string, args ...interface{}) ([]mapstr.M, error) +} + +func (f *fakeDBClient) FetchTableMode(ctx context.Context, query string) ([]mapstr.M, error) { + f.tableQueries = append(f.tableQueries, query) + if f.fetchTableFn != nil { + return f.fetchTableFn(ctx, query) + } + return f.tableRows, f.tableErr +} + +func (f *fakeDBClient) FetchTableModeWithParams(ctx context.Context, query string, args ...interface{}) ([]mapstr.M, error) { + f.withParamQueries = append(f.withParamQueries, query) + f.withParamArgs = append(f.withParamArgs, args) + if f.fetchTableWithParamFn != nil { + return f.fetchTableWithParamFn(ctx, query, args...) + } + return f.tableWithParamRows, f.tableWithParamErr +} + +func (f *fakeDBClient) FetchVariableMode(ctx context.Context, query string) (mapstr.M, error) { + f.variableQueries = append(f.variableQueries, query) + if f.fetchVariableFn != nil { + return f.fetchVariableFn(ctx, query) + } + return f.variableRows, f.variableErr +} + +func (f *fakeDBClient) Close() error { + f.closed = true + return nil +} + +type stopReporter struct { + events int +} + +func (r *stopReporter) Event(_ mb.Event) bool { + r.events++ + return false +} + +func (r *stopReporter) Error(_ error) bool { + return false +} + +func testMetricSetConfig(queryText string) map[string]interface{} { + return map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{"postgres://user:pass@localhost:5432/mydb?sslmode=disable"}, + "driver": "postgres", + "sql_query": queryText, + "sql_response_format": "table", + } +} + +func newTestMetricSet(t *testing.T, cfg map[string]interface{}) *MetricSet { + t.Helper() + + ms := mbtest.NewMetricSet(t, cfg) + qms, ok := ms.(*MetricSet) + require.Truef(t, ok, "expected *MetricSet, got %T", ms) + return qms +} + +func newTestCursorManager(t *testing.T, defaultValue string) *cursor.Manager { + t.Helper() + + logger := logp.NewNopLogger() + reg, err := memlog.New(logger.Named("memlog"), memlog.Settings{ + Root: t.TempDir(), + FileMode: 0o600, + }) + require.NoError(t, err) + + registry := statestore.NewRegistry(reg) + t.Cleanup(func() { registry.Close() }) + + store, err := cursor.NewStoreFromRegistry(registry, logger.Named("cursor")) + require.NoError(t, err) + + cfg := cursor.Config{ + Enabled: true, + Column: "id", + Type: cursor.CursorTypeInteger, + Default: defaultValue, + Direction: cursor.CursorDirectionAsc, + } + mgr, err := cursor.NewManager(cfg, store, + "postgres://user:pass@localhost:5432/mydb?sslmode=disable", + "SELECT id FROM t WHERE id > :cursor ORDER BY id ASC", + logger, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = mgr.Close() }) + return mgr +} + +func withFakeDBClientFactory(t *testing.T, db dbClient) { + t.Helper() + + original := newDBClient + newDBClient = func(_, _ string, _ *logp.Logger) (dbClient, error) { + return db, nil + } + t.Cleanup(func() { + newDBClient = original + }) +} + +func withTempDataPath(t *testing.T) { + t.Helper() + origData := paths.Paths.Data + paths.Paths.Data = t.TempDir() + t.Cleanup(func() { + paths.Paths.Data = origData + }) +} + +func instantiateMetricSetWithConfig(t *testing.T, cfg map[string]interface{}) error { + t.Helper() + withTempDataPath(t) + + c, err := conf.NewConfigFrom(cfg) + require.NoError(t, err) + _, metricsets, err := mb.NewModule(c, mb.Registry, paths.New(), logptest.NewTestingLogger(t, "")) + if err != nil { + return err + } + + for _, ms := range metricsets { + closer, ok := ms.(mb.Closer) + if !ok { + continue + } + require.NoError(t, closer.Close()) + } + + return nil +} + +func TestFetch_NonCursorPath_ReportsEvents(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t ORDER BY id")) + + fakeDB := &fakeDBClient{ + tableRows: []mapstr.M{ + {"id": int64(1), "name": "a"}, + }, + } + withFakeDBClientFactory(t, fakeDB) + + reporter := &mbtest.CapturingReporterV2{} + err := ms.Fetch(context.Background(), reporter) + require.NoError(t, err) + + assert.True(t, fakeDB.closed, "DB client should be closed after fetch") + assert.Len(t, fakeDB.tableQueries, 1, "table query should be executed once") + assert.Len(t, reporter.GetEvents(), 1, "one event should be reported") +} + +func TestFetch_CursorPath_AdvancesCursorOnSuccessfulReporting(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC")) + ms.cursorManager = newTestCursorManager(t, "0") + ms.translatedQuery = cursor.TranslateQuery(ms.Config.Query, ms.Config.Driver) + + fakeDB := &fakeDBClient{ + tableWithParamRows: []mapstr.M{ + {"id": int64(10)}, + {"id": int64(20)}, + }, + } + withFakeDBClientFactory(t, fakeDB) + + reporter := &mbtest.CapturingReporterV2{} + err := ms.Fetch(context.Background(), reporter) + require.NoError(t, err) + + assert.True(t, fakeDB.closed, "DB client should be closed after fetch") + assert.Len(t, fakeDB.withParamQueries, 1, "parameterized query should be executed once") + assert.Equal(t, ms.translatedQuery, fakeDB.withParamQueries[0]) + require.Len(t, fakeDB.withParamArgs, 1) + require.Len(t, fakeDB.withParamArgs[0], 1) + assert.EqualValues(t, int64(0), fakeDB.withParamArgs[0][0], "default cursor value should be passed as arg") + assert.Len(t, reporter.GetEvents(), 2, "all rows should be reported") + assert.Equal(t, "20", ms.cursorManager.CursorValueString(), "cursor should advance to max id") +} + +func TestFetch_CursorPath_DoesNotAdvanceWhenReporterStops(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC")) + ms.cursorManager = newTestCursorManager(t, "0") + ms.translatedQuery = cursor.TranslateQuery(ms.Config.Query, ms.Config.Driver) + + fakeDB := &fakeDBClient{ + tableWithParamRows: []mapstr.M{ + {"id": int64(10)}, + {"id": int64(20)}, + }, + } + withFakeDBClientFactory(t, fakeDB) + + reporter := &stopReporter{} + err := ms.Fetch(context.Background(), reporter) + require.NoError(t, err) + + assert.Equal(t, 1, reporter.events, "fetch should stop after reporter rejects first event") + assert.Equal(t, "0", ms.cursorManager.CursorValueString(), "cursor must not advance when reporting stops") +} + +func TestFetch_CursorPath_ZeroRowsLeavesCursorUnchanged(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC")) + ms.cursorManager = newTestCursorManager(t, "42") + ms.translatedQuery = cursor.TranslateQuery(ms.Config.Query, ms.Config.Driver) + + fakeDB := &fakeDBClient{tableWithParamRows: []mapstr.M{}} + withFakeDBClientFactory(t, fakeDB) + + reporter := &mbtest.CapturingReporterV2{} + err := ms.Fetch(context.Background(), reporter) + require.NoError(t, err) + assert.Equal(t, "42", ms.cursorManager.CursorValueString()) + assert.Empty(t, reporter.GetEvents()) +} + +func TestFetch_CursorPath_QueryErrorPropagates(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC")) + ms.cursorManager = newTestCursorManager(t, "0") + ms.translatedQuery = cursor.TranslateQuery(ms.Config.Query, ms.Config.Driver) + + fakeDB := &fakeDBClient{ + tableWithParamErr: errors.New("db query failed"), + } + withFakeDBClientFactory(t, fakeDB) + + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "fetch with cursor failed") +} + +func TestFetch_CursorPath_AppliesTimeoutContext(t *testing.T) { + cfg := testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC") + cfg["timeout"] = "50ms" + ms := newTestMetricSet(t, cfg) + ms.cursorManager = newTestCursorManager(t, "0") + ms.translatedQuery = cursor.TranslateQuery(ms.Config.Query, ms.Config.Driver) + + deadlineSeen := false + fakeDB := &fakeDBClient{ + fetchTableWithParamFn: func(ctx context.Context, _ string, _ ...interface{}) ([]mapstr.M, error) { + _, ok := ctx.Deadline() + deadlineSeen = ok + return []mapstr.M{}, nil + }, + } + withFakeDBClientFactory(t, fakeDB) + + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.NoError(t, err) + assert.True(t, deadlineSeen, "cursor fetch path should apply module timeout") +} + +func TestFetch_CursorPath_SkipsWhenPreviousFetchInProgress(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC")) + ms.cursorManager = newTestCursorManager(t, "0") + ms.translatedQuery = cursor.TranslateQuery(ms.Config.Query, ms.Config.Driver) + + fakeDB := &fakeDBClient{ + tableWithParamRows: []mapstr.M{{"id": int64(1)}}, + } + withFakeDBClientFactory(t, fakeDB) + + ms.fetchMutex.Lock() + defer ms.fetchMutex.Unlock() + + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.NoError(t, err) + assert.Empty(t, fakeDB.withParamQueries, "DB should not be called when TryLock fails") +} + +func TestFetch_NonCursorPath_DBOpenError(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t")) + + orig := newDBClient + newDBClient = func(_, _ string, _ *logp.Logger) (dbClient, error) { + return nil, errors.New("open failed") + } + t.Cleanup(func() { newDBClient = orig }) + + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot open connection") +} + +func TestFetch_NonCursorPath_ReporterStopDoesNotError(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t ORDER BY id")) + + fakeDB := &fakeDBClient{ + tableRows: []mapstr.M{{"id": int64(1)}}, + } + withFakeDBClientFactory(t, fakeDB) + + reporter := &stopReporter{} + err := ms.Fetch(context.Background(), reporter) + require.NoError(t, err) + assert.Equal(t, 1, reporter.events) +} + +func TestFetch_MergeResultsRejectsMultipleRowsPerQuery(t *testing.T) { + ms := &MetricSet{ + Config: config{ + MergeResults: true, + }, + } + + fakeDB := &fakeDBClient{ + tableRows: []mapstr.M{ + {"id": int64(1)}, + {"id": int64(2)}, + }, + } + + _, err := ms.fetch(context.Background(), fakeDB, &mbtest.CapturingReporterV2{}, []query{ + {Query: "SELECT id FROM t", ResponseFormat: tableResponseFormat}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot merge query resulting with more than one rows") +} + +func TestFetch_TableAndVariableErrorsAreWrapped(t *testing.T) { + ms := &MetricSet{Config: config{}} + + _, err := ms.fetch(context.Background(), &fakeDBClient{ + tableErr: errors.New("table err"), + }, &mbtest.CapturingReporterV2{}, []query{ + {Query: "SELECT id FROM t", ResponseFormat: tableResponseFormat}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "fetch table mode failed") + + _, err = ms.fetch(context.Background(), &fakeDBClient{ + variableErr: errors.New("var err"), + }, &mbtest.CapturingReporterV2{}, []query{ + {Query: "SHOW STATUS", ResponseFormat: variableResponseFormat}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "fetch variable mode failed") +} + +func TestFetch_MergeResults_OverwritesDuplicateKeys(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t")) + ms.Config.MergeResults = true + ms.Config.Driver = "postgres" + + fakeDB := &fakeDBClient{ + fetchTableFn: func(_ context.Context, query string) ([]mapstr.M, error) { + // one row/query so merge mode accepts it + if strings.Contains(query, "table_a") { + return []mapstr.M{{"dup": int64(1), "a": "x"}}, nil + } + if strings.Contains(query, "table_b") { + return []mapstr.M{{"dup": int64(2), "b": "y"}}, nil + } + return nil, nil + }, + fetchVariableFn: func(_ context.Context, _ string) (mapstr.M, error) { + // duplicate key with table result to exercise overwrite path. + return mapstr.M{"dup": int64(3), "var": "ok"}, nil + }, + } + + reporter := &mbtest.CapturingReporterV2{} + ok, err := ms.fetch(context.Background(), fakeDB, reporter, []query{ + {Query: "SELECT * FROM table_a", ResponseFormat: tableResponseFormat}, + {Query: "SELECT * FROM table_b", ResponseFormat: tableResponseFormat}, + {Query: "SHOW STATUS", ResponseFormat: variableResponseFormat}, + }) + require.NoError(t, err) + assert.True(t, ok) + assert.Len(t, reporter.GetEvents(), 1) +} + +func TestFetch_NonMerge_ReportsEachQueryResult(t *testing.T) { + ms := &MetricSet{ + Config: config{ + MergeResults: false, + Driver: "postgres", + }, + } + + fakeDB := &fakeDBClient{ + fetchTableFn: func(_ context.Context, _ string) ([]mapstr.M, error) { + return []mapstr.M{ + {"id": int64(1)}, + {"id": int64(2)}, + }, nil + }, + fetchVariableFn: func(_ context.Context, _ string) (mapstr.M, error) { + return mapstr.M{"status": "ok"}, nil + }, + } + + reporter := &mbtest.CapturingReporterV2{} + ok, err := ms.fetch(context.Background(), fakeDB, reporter, []query{ + {Query: "SELECT id FROM t", ResponseFormat: tableResponseFormat}, + {Query: "SHOW STATUS", ResponseFormat: variableResponseFormat}, + }) + require.NoError(t, err) + assert.True(t, ok) + assert.Len(t, reporter.GetEvents(), 3, "2 table rows + 1 variable result") +} + +func TestFetch_NonCursorPath_FetchErrorIsSwallowed(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t")) + + withFakeDBClientFactory(t, &fakeDBClient{ + tableErr: errors.New("query failed"), + }) + + // Fetch logs and swallows per-query fetch errors in non-cursor path. + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.NoError(t, err) +} + +func TestFetch_VariableMode_MergeResultsSuccess(t *testing.T) { + ms := &MetricSet{ + Config: config{ + MergeResults: true, + Driver: "postgres", + }, + } + + fakeDB := &fakeDBClient{ + variableRows: mapstr.M{ + "connections": int64(3), + "db_name": "mydb", + }, + } + + reporter := &mbtest.CapturingReporterV2{} + ok, err := ms.fetch(context.Background(), fakeDB, reporter, []query{ + {Query: "SHOW STATUS", ResponseFormat: variableResponseFormat}, + }) + require.NoError(t, err) + assert.True(t, ok) + assert.Len(t, fakeDB.variableQueries, 1) + assert.Len(t, reporter.GetEvents(), 1) +} + +func TestInitCursorAndClose(t *testing.T) { + withTempDataPath(t) + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC")) + ms.Config.Cursor = cursor.Config{ + Enabled: true, + Column: "id", + Type: cursor.CursorTypeInteger, + Default: "0", + } + + err := ms.initCursor(ms.BaseMetricSet) + require.NoError(t, err) + require.NotNil(t, ms.cursorManager) + assert.Contains(t, ms.translatedQuery, "$1") + + require.NoError(t, ms.Close()) +} + +func TestCloseWithoutCursorManager(t *testing.T) { + ms := &MetricSet{} + require.NoError(t, ms.Close()) +} + +func TestReportEvent_RawDataWithAndWithoutQuery(t *testing.T) { + ms := &MetricSet{ + Config: config{ + Driver: "postgres", + RawData: rawData{ + Enabled: true, + }, + }, + } + + reporter := &mbtest.CapturingReporterV2{} + ok := ms.reportEvent(mapstr.M{"x": 1}, reporter, "SELECT 1") + require.True(t, ok) + require.Len(t, reporter.GetEvents(), 1) + + ms2 := &MetricSet{ + Config: config{ + Driver: "postgres", + RawData: rawData{ + Enabled: true, + }, + }, + } + reporter2 := &mbtest.CapturingReporterV2{} + ok = ms2.reportEvent(mapstr.M{"x": 2}, reporter2) + require.True(t, ok) + require.Len(t, reporter2.GetEvents(), 1) +} + +func TestFetch_FetchFromAllDatabases_UnsupportedDriver(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t")) + ms.Config.FetchFromAllDatabases = true + ms.Config.Driver = "postgres" + + fakeDB := &fakeDBClient{} + withFakeDBClientFactory(t, fakeDB) + + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "fetch from all databases feature is not supported for driver: postgres") +} + +func TestFetch_FetchFromAllDatabases_NoDatabaseNames(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t")) + ms.Config.FetchFromAllDatabases = true + ms.Config.Driver = "mssql" + + fakeDB := &fakeDBClient{ + tableRows: []mapstr.M{}, + } + withFakeDBClientFactory(t, fakeDB) + + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no database names found") +} + +func TestFetch_FetchFromAllDatabases_DBNamesQueryError(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM t")) + ms.Config.FetchFromAllDatabases = true + ms.Config.Driver = "mssql" + + withFakeDBClientFactory(t, &fakeDBClient{ + tableErr: errors.New("list dbs failed"), + }) + + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot fetch database names") +} + +func TestFetch_FetchFromAllDatabases_MSSQLSuccess(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM metrics_table")) + ms.Config.FetchFromAllDatabases = true + ms.Config.Driver = "mssql" + + fakeDB := &fakeDBClient{ + fetchTableFn: func(_ context.Context, query string) ([]mapstr.M, error) { + if strings.Contains(query, "sys.databases") { + return []mapstr.M{{"name": "db1"}}, nil + } + if strings.Contains(query, "USE [db1];") { + return []mapstr.M{{"id": int64(1)}}, nil + } + return nil, nil + }, + } + withFakeDBClientFactory(t, fakeDB) + + reporter := &mbtest.CapturingReporterV2{} + err := ms.Fetch(context.Background(), reporter) + require.NoError(t, err) + assert.Len(t, reporter.GetEvents(), 1) +} + +func TestFetch_FetchFromAllDatabases_SkipsRowsWithoutStringName(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM metrics_table")) + ms.Config.FetchFromAllDatabases = true + ms.Config.Driver = "mssql" + + fakeDB := &fakeDBClient{ + fetchTableFn: func(_ context.Context, query string) ([]mapstr.M, error) { + if strings.Contains(query, "sys.databases") { + return []mapstr.M{ + {}, // missing "name" + {"name": int64(1)}, // wrong type + }, nil + } + return nil, nil + }, + } + withFakeDBClientFactory(t, fakeDB) + + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.NoError(t, err) +} + +func TestFetch_FetchFromAllDatabases_InnerFetchErrorStopsCycle(t *testing.T) { + ms := newTestMetricSet(t, testMetricSetConfig("SELECT id FROM metrics_table")) + ms.Config.FetchFromAllDatabases = true + ms.Config.Driver = "mssql" + + fakeDB := &fakeDBClient{ + fetchTableFn: func(_ context.Context, query string) ([]mapstr.M, error) { + if strings.Contains(query, "sys.databases") { + return []mapstr.M{{"name": "db1"}}, nil + } + // Per-database query fails inside m.fetch(). + if strings.Contains(query, "USE [db1];") { + return nil, errors.New("db-specific failure") + } + return nil, nil + }, + } + withFakeDBClientFactory(t, fakeDB) + + // Current behavior: logs warning and returns nil when reporting/fetch fails for a DB. + err := ms.Fetch(context.Background(), &mbtest.CapturingReporterV2{}) + require.NoError(t, err) +} + +func TestOpenCursorStore_RequiresSQLModuleInterface(t *testing.T) { + ms := &MetricSet{} + _, err := ms.openCursorStore(mb.BaseMetricSet{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cursor requires SQL module to implement registry interface") +} + +func TestInitCursor_PropagatesStoreInitError(t *testing.T) { + ms := &MetricSet{ + Config: config{ + ResponseFormat: tableResponseFormat, + Query: "SELECT id FROM t WHERE id > :cursor", + Cursor: cursor.Config{ + Enabled: true, + Column: "id", + Type: cursor.CursorTypeInteger, + Default: "0", + }, + }, + } + + err := ms.initCursor(mb.BaseMetricSet{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cursor store initialization failed") +} + +func TestNew_ConfigValidationErrors(t *testing.T) { + base := testMetricSetConfig("SELECT id FROM t") + + tests := []struct { + name string + overrides map[string]interface{} + wantErr string + }{ + { + name: "invalid response format", + overrides: map[string]interface{}{ + "sql_response_format": "bad", + }, + wantErr: "invalid sql_response_format value: bad", + }, + { + name: "no query inputs", + overrides: map[string]interface{}{ + "sql_query": "", + "sql_queries": []map[string]interface{}{}, + }, + wantErr: "no query input provided, must provide either sql_query or sql_queries", + }, + { + name: "both query and queries", + overrides: map[string]interface{}{ + "sql_query": "SELECT 1", + "sql_queries": []map[string]interface{}{ + {"query": "SELECT 2", "response_format": "table"}, + }, + }, + wantErr: "both query inputs provided, must provide either sql_query or sql_queries", + }, + { + name: "cursor with multiple queries", + overrides: map[string]interface{}{ + "sql_query": "", + "sql_queries": []map[string]interface{}{ + {"query": "SELECT id FROM t", "response_format": "table"}, + }, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": "integer", + "cursor.default": "0", + }, + wantErr: "cursor is not supported with sql_queries (multiple queries)", + }, + { + name: "cursor with fetch from all databases", + overrides: map[string]interface{}{ + "sql_query": "SELECT id FROM t WHERE id > :cursor", + "sql_response_format": "table", + "fetch_from_all_databases": true, + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": "integer", + "cursor.default": "0", + }, + wantErr: "cursor is not supported with fetch_from_all_databases", + }, + { + name: "cursor requires table format", + overrides: map[string]interface{}{ + "sql_query": "SELECT id FROM t WHERE id > :cursor", + "sql_response_format": "variables", + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": "integer", + "cursor.default": "0", + }, + wantErr: "cursor requires sql_response_format: table", + }, + { + name: "invalid response format in sql_queries", + overrides: map[string]interface{}{ + "sql_query": "", + "sql_queries": []map[string]interface{}{ + {"query": "SELECT 1", "response_format": "invalid"}, + }, + }, + wantErr: "invalid sql_response_format value: invalid", + }, + { + name: "cursor query missing placeholder", + overrides: map[string]interface{}{ + "sql_query": "SELECT id FROM t", + "sql_response_format": "table", + "cursor.enabled": true, + "cursor.column": "id", + "cursor.type": "integer", + "cursor.default": "0", + }, + wantErr: "query must contain :cursor placeholder when cursor is enabled", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := make(map[string]interface{}, len(base)+len(tt.overrides)) + for k, v := range base { + cfg[k] = v + } + for k, v := range tt.overrides { + cfg[k] = v + } + + err := instantiateMetricSetWithConfig(t, cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestNew_DefaultsResponseFormatWhenEmpty(t *testing.T) { + cfg := testMetricSetConfig("SELECT id FROM t") + delete(cfg, "sql_response_format") + err := instantiateMetricSetWithConfig(t, cfg) + require.NoError(t, err) +} + +func TestNew_CursorTypeOptional(t *testing.T) { + cfg := testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC") + cfg["cursor.enabled"] = true + cfg["cursor.column"] = "id" + cfg["cursor.default"] = "0" + delete(cfg, "cursor.type") + + err := instantiateMetricSetWithConfig(t, cfg) + require.NoError(t, err) +} + +func TestNew_CursorStateIDOptional(t *testing.T) { + cfg := testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC") + cfg["cursor.enabled"] = true + cfg["cursor.column"] = "id" + cfg["cursor.type"] = "integer" + cfg["cursor.default"] = "0" + cfg["cursor.state_id"] = "payments-prod" + + err := instantiateMetricSetWithConfig(t, cfg) + require.NoError(t, err) +} + +func TestNew_CursorStateIDBlankIsRejected(t *testing.T) { + cfg := testMetricSetConfig("SELECT id FROM t WHERE id > :cursor ORDER BY id ASC") + cfg["cursor.enabled"] = true + cfg["cursor.column"] = "id" + cfg["cursor.type"] = "integer" + cfg["cursor.default"] = "0" + cfg["cursor.state_id"] = " " + + err := instantiateMetricSetWithConfig(t, cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "cursor.state_id cannot be blank") +} + +func TestInferTypeFromMetricsAndDriverHelpers(t *testing.T) { + typed := inferTypeFromMetrics(mapstr.M{ + "i": int64(1), + "f": 3.14, + "s": "hello", + "b": true, + "n": nil, + "misc": time.Second, // default case -> string bucket + }) + + numeric, ok := typed["numeric"].(mapstr.M) + require.True(t, ok) + assert.Contains(t, numeric, "i") + assert.Contains(t, numeric, "f") + + stringVals, ok := typed["string"].(mapstr.M) + require.True(t, ok) + assert.Contains(t, stringVals, "s") + assert.Contains(t, stringVals, "misc") + + boolVals, ok := typed["bool"].(mapstr.M) + require.True(t, ok) + assert.Contains(t, boolVals, "b") + assert.NotContains(t, typed, "n") + + assert.NotEmpty(t, queryDBNames("mssql")) + assert.Empty(t, queryDBNames("postgres")) + assert.Equal(t, "USE [mydb];", dbSelector("sqlserver", "mydb")) + assert.Equal(t, "", dbSelector("postgres", "mydb")) +} diff --git a/x-pack/metricbeat/modules.d/sql.yml.disabled b/x-pack/metricbeat/modules.d/sql.yml.disabled index 289769af3092..f1bf88711b31 100644 --- a/x-pack/metricbeat/modules.d/sql.yml.disabled +++ b/x-pack/metricbeat/modules.d/sql.yml.disabled @@ -16,6 +16,18 @@ sql_query: "select now()" sql_response_format: table + # Cursor-based incremental data fetching. When enabled, the cursor tracks the + # last fetched row value and uses it to retrieve only new data on subsequent + # collection cycles. Requires sql_response_format: table and a single sql_query + # with exactly one :cursor placeholder. + #cursor: + # enabled: true + # column: id + # type: integer + # #state_id: payments-prod # optional: stable identity across DSN changes + # default: "0" + # #direction: asc + # List of root certificates for SSL/TLS server verification # ssl.certificate_authorities: ["/path/to/ca.pem"]